diff options
444 files changed, 23272 insertions, 8852 deletions
diff --git a/Documentation/devicetree/bindings/net/can/microchip,mcp25xxfd.yaml b/Documentation/devicetree/bindings/net/can/microchip,mcp251xfd.yaml index aa2cad14d6d7..2a884c1fe0e0 100644 --- a/Documentation/devicetree/bindings/net/can/microchip,mcp25xxfd.yaml +++ b/Documentation/devicetree/bindings/net/can/microchip,mcp251xfd.yaml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/net/can/microchip,mcp25xxfd.yaml# +$id: http://devicetree.org/schemas/net/can/microchip,mcp251xfd.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# title: @@ -18,13 +18,13 @@ properties: description: for MCP2517FD - const: microchip,mcp2518fd description: for MCP2518FD - - const: microchip,mcp25xxfd + - const: microchip,mcp251xfd description: to autodetect chip variant reg: maxItems: 1 - interrupts-extended: + interrupts: maxItems: 1 clocks: @@ -32,15 +32,13 @@ properties: vdd-supply: description: Regulator that powers the CAN controller. - maxItems: 1 xceiver-supply: description: Regulator that powers the CAN transceiver. - maxItems: 1 microchip,rx-int-gpios: description: - GPIO phandle of GPIO connected to to INT1 pin of the MCP25XXFD, which + GPIO phandle of GPIO connected to to INT1 pin of the MCP251XFD, which signals a pending RX interrupt. maxItems: 1 @@ -52,9 +50,11 @@ properties: required: - compatible - reg - - interrupts-extended + - interrupts - clocks +additionalProperties: false + examples: - | #include <dt-bindings/gpio/gpio.h> @@ -65,7 +65,7 @@ examples: #size-cells = <0>; can@0 { - compatible = "microchip,mcp25xxfd"; + compatible = "microchip,mcp251xfd"; reg = <0>; clocks = <&can0_osc>; pinctrl-names = "default"; diff --git a/Documentation/devicetree/bindings/net/dsa/b53.txt b/Documentation/devicetree/bindings/net/dsa/b53.txt index cfd1afdc6e94..80437b2fc935 100644 --- a/Documentation/devicetree/bindings/net/dsa/b53.txt +++ b/Documentation/devicetree/bindings/net/dsa/b53.txt @@ -106,6 +106,7 @@ Ethernet switch connected via MDIO to the host, CPU port wired to eth0: switch0: ethernet-switch@30 { compatible = "brcm,bcm53125"; + reg = <30>; #address-cells = <1>; #size-cells = <0>; diff --git a/Documentation/devicetree/bindings/net/ethernet-controller.yaml b/Documentation/devicetree/bindings/net/ethernet-controller.yaml index fa2baca8c726..137fbb95c0ea 100644 --- a/Documentation/devicetree/bindings/net/ethernet-controller.yaml +++ b/Documentation/devicetree/bindings/net/ethernet-controller.yaml @@ -120,6 +120,13 @@ properties: and is useful for determining certain configuration settings such as flow control thresholds. + rx-internal-delay-ps: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + RGMII Receive Clock Delay defined in pico seconds. + This is used for controllers that have configurable RX internal delays. + If this property is present then the MAC applies the RX delay. + sfp: $ref: /schemas/types.yaml#definitions/phandle description: @@ -131,6 +138,13 @@ properties: The size of the controller\'s transmit fifo in bytes. This is used for components that can have configurable fifo sizes. + tx-internal-delay-ps: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + RGMII Transmit Clock Delay defined in pico seconds. + This is used for controllers that have configurable TX internal delays. + If this property is present then the MAC applies the TX delay. + managed: description: Specifies the PHY management type. If auto is set and fixed-link diff --git a/Documentation/devicetree/bindings/net/renesas,etheravb.yaml b/Documentation/devicetree/bindings/net/renesas,etheravb.yaml new file mode 100644 index 000000000000..e13653051b23 --- /dev/null +++ b/Documentation/devicetree/bindings/net/renesas,etheravb.yaml @@ -0,0 +1,261 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/renesas,etheravb.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas Ethernet AVB + +maintainers: + - Sergei Shtylyov <sergei.shtylyov@gmail.com> + +properties: + compatible: + oneOf: + - items: + - enum: + - renesas,etheravb-r8a7742 # RZ/G1H + - renesas,etheravb-r8a7743 # RZ/G1M + - renesas,etheravb-r8a7744 # RZ/G1N + - renesas,etheravb-r8a7745 # RZ/G1E + - renesas,etheravb-r8a77470 # RZ/G1C + - renesas,etheravb-r8a7790 # R-Car H2 + - renesas,etheravb-r8a7791 # R-Car M2-W + - renesas,etheravb-r8a7792 # R-Car V2H + - renesas,etheravb-r8a7793 # R-Car M2-N + - renesas,etheravb-r8a7794 # R-Car E2 + - const: renesas,etheravb-rcar-gen2 # R-Car Gen2 and RZ/G1 + + - items: + - enum: + - renesas,etheravb-r8a774a1 # RZ/G2M + - renesas,etheravb-r8a774b1 # RZ/G2N + - renesas,etheravb-r8a774c0 # RZ/G2E + - renesas,etheravb-r8a7795 # R-Car H3 + - renesas,etheravb-r8a7796 # R-Car M3-W + - renesas,etheravb-r8a77961 # R-Car M3-W+ + - renesas,etheravb-r8a77965 # R-Car M3-N + - renesas,etheravb-r8a77970 # R-Car V3M + - renesas,etheravb-r8a77980 # R-Car V3H + - renesas,etheravb-r8a77990 # R-Car E3 + - renesas,etheravb-r8a77995 # R-Car D3 + - const: renesas,etheravb-rcar-gen3 # R-Car Gen3 and RZ/G2 + + reg: true + + interrupts: true + + interrupt-names: true + + clocks: + maxItems: 1 + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + resets: + maxItems: 1 + + phy-mode: true + + phy-handle: true + + '#address-cells': + description: Number of address cells for the MDIO bus. + const: 1 + + '#size-cells': + description: Number of size cells on the MDIO bus. + const: 0 + + renesas,no-ether-link: + type: boolean + description: + Specify when a board does not provide a proper AVB_LINK signal. + + renesas,ether-link-active-low: + type: boolean + description: + Specify when the AVB_LINK signal is active-low instead of normal + active-high. + + rx-internal-delay-ps: + enum: [0, 1800] + + tx-internal-delay-ps: + enum: [0, 2000] + +patternProperties: + "^ethernet-phy@[0-9a-f]$": + type: object + $ref: ethernet-phy.yaml# + +required: + - compatible + - reg + - interrupts + - clocks + - power-domains + - resets + - phy-mode + - phy-handle + - '#address-cells' + - '#size-cells' + +allOf: + - $ref: ethernet-controller.yaml# + + - if: + properties: + compatible: + contains: + enum: + - renesas,etheravb-rcar-gen2 + - renesas,etheravb-r8a7795 + - renesas,etheravb-r8a7796 + - renesas,etheravb-r8a77961 + - renesas,etheravb-r8a77965 + then: + properties: + reg: + items: + - description: MAC register block + - description: Stream buffer + else: + properties: + reg: + items: + - description: MAC register block + + - if: + properties: + compatible: + contains: + const: renesas,etheravb-rcar-gen2 + then: + properties: + interrupts: + maxItems: 1 + interrupt-names: + items: + - const: mux + rx-internal-delay-ps: false + else: + properties: + interrupts: + minItems: 25 + maxItems: 25 + interrupt-names: + items: + pattern: '^ch[0-9]+$' + required: + - interrupt-names + - rx-internal-delay-ps + + - if: + properties: + compatible: + contains: + enum: + - renesas,etheravb-r8a774a1 + - renesas,etheravb-r8a774b1 + - renesas,etheravb-r8a7795 + - renesas,etheravb-r8a7796 + - renesas,etheravb-r8a77961 + - renesas,etheravb-r8a77965 + - renesas,etheravb-r8a77970 + - renesas,etheravb-r8a77980 + then: + required: + - tx-internal-delay-ps + else: + properties: + tx-internal-delay-ps: false + + - if: + properties: + compatible: + contains: + const: renesas,etheravb-r8a77995 + then: + properties: + rx-internal-delay-ps: + const: 1800 + + - if: + properties: + compatible: + contains: + const: renesas,etheravb-r8a77980 + then: + properties: + tx-internal-delay-ps: + const: 2000 + +additionalProperties: false + +examples: + - | + #include <dt-bindings/clock/r8a7795-cpg-mssr.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + #include <dt-bindings/power/r8a7795-sysc.h> + #include <dt-bindings/gpio/gpio.h> + aliases { + ethernet0 = &avb; + }; + + avb: ethernet@e6800000 { + compatible = "renesas,etheravb-r8a7795", + "renesas,etheravb-rcar-gen3"; + reg = <0xe6800000 0x800>, <0xe6a00000 0x10000>; + interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "ch0", "ch1", "ch2", "ch3", "ch4", "ch5", "ch6", + "ch7", "ch8", "ch9", "ch10", "ch11", "ch12", + "ch13", "ch14", "ch15", "ch16", "ch17", "ch18", + "ch19", "ch20", "ch21", "ch22", "ch23", "ch24"; + clocks = <&cpg CPG_MOD 812>; + iommus = <&ipmmu_ds0 16>; + power-domains = <&sysc R8A7795_PD_ALWAYS_ON>; + resets = <&cpg 812>; + phy-mode = "rgmii"; + phy-handle = <&phy0>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <2000>; + #address-cells = <1>; + #size-cells = <0>; + + phy0: ethernet-phy@0 { + rxc-skew-ps = <1500>; + reg = <0>; + interrupt-parent = <&gpio2>; + interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + reset-gpios = <&gpio2 10 GPIO_ACTIVE_LOW>; + }; + }; diff --git a/Documentation/devicetree/bindings/net/renesas,ravb.txt b/Documentation/devicetree/bindings/net/renesas,ravb.txt deleted file mode 100644 index 032b76f14f4f..000000000000 --- a/Documentation/devicetree/bindings/net/renesas,ravb.txt +++ /dev/null @@ -1,134 +0,0 @@ -* Renesas Electronics Ethernet AVB - -This file provides information on what the device node for the Ethernet AVB -interface contains. - -Required properties: -- compatible: Must contain one or more of the following: - - "renesas,etheravb-r8a7742" for the R8A7742 SoC. - - "renesas,etheravb-r8a7743" for the R8A7743 SoC. - - "renesas,etheravb-r8a7744" for the R8A7744 SoC. - - "renesas,etheravb-r8a7745" for the R8A7745 SoC. - - "renesas,etheravb-r8a77470" for the R8A77470 SoC. - - "renesas,etheravb-r8a7790" for the R8A7790 SoC. - - "renesas,etheravb-r8a7791" for the R8A7791 SoC. - - "renesas,etheravb-r8a7792" for the R8A7792 SoC. - - "renesas,etheravb-r8a7793" for the R8A7793 SoC. - - "renesas,etheravb-r8a7794" for the R8A7794 SoC. - - "renesas,etheravb-rcar-gen2" as a fallback for the above - R-Car Gen2 and RZ/G1 devices. - - - "renesas,etheravb-r8a774a1" for the R8A774A1 SoC. - - "renesas,etheravb-r8a774b1" for the R8A774B1 SoC. - - "renesas,etheravb-r8a774c0" for the R8A774C0 SoC. - - "renesas,etheravb-r8a7795" for the R8A7795 SoC. - - "renesas,etheravb-r8a7796" for the R8A77960 SoC. - - "renesas,etheravb-r8a77961" for the R8A77961 SoC. - - "renesas,etheravb-r8a77965" for the R8A77965 SoC. - - "renesas,etheravb-r8a77970" for the R8A77970 SoC. - - "renesas,etheravb-r8a77980" for the R8A77980 SoC. - - "renesas,etheravb-r8a77990" for the R8A77990 SoC. - - "renesas,etheravb-r8a77995" for the R8A77995 SoC. - - "renesas,etheravb-rcar-gen3" as a fallback for the above - R-Car Gen3 and RZ/G2 devices. - - When compatible with the generic version, nodes must list the - SoC-specific version corresponding to the platform first followed by - the generic version. - -- reg: Offset and length of (1) the register block and (2) the stream buffer. - The region for the register block is mandatory. - The region for the stream buffer is optional, as it is only present on - R-Car Gen2 and RZ/G1 SoCs, and on R-Car H3 (R8A7795), M3-W (R8A77960), - M3-W+ (R8A77961), and M3-N (R8A77965). -- interrupts: A list of interrupt-specifiers, one for each entry in - interrupt-names. - If interrupt-names is not present, an interrupt specifier - for a single muxed interrupt. -- phy-mode: see ethernet.txt file in the same directory. -- phy-handle: see ethernet.txt file in the same directory. -- #address-cells: number of address cells for the MDIO bus, must be equal to 1. -- #size-cells: number of size cells on the MDIO bus, must be equal to 0. -- clocks: clock phandle and specifier pair. -- pinctrl-0: phandle, referring to a default pin configuration node. - -Optional properties: -- interrupt-names: A list of interrupt names. - For the R-Car Gen 3 SoCs this property is mandatory; - it should include one entry per channel, named "ch%u", - where %u is the channel number ranging from 0 to 24. - For other SoCs this property is optional; if present - it should contain "mux" for a single muxed interrupt. -- pinctrl-names: pin configuration state name ("default"). -- renesas,no-ether-link: boolean, specify when a board does not provide a proper - AVB_LINK signal. -- renesas,ether-link-active-low: boolean, specify when the AVB_LINK signal is - active-low instead of normal active-high. - -Example: - - ethernet@e6800000 { - compatible = "renesas,etheravb-r8a7795", "renesas,etheravb-rcar-gen3"; - reg = <0 0xe6800000 0 0x800>, <0 0xe6a00000 0 0x10000>; - interrupt-parent = <&gic>; - interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>; - interrupt-names = "ch0", "ch1", "ch2", "ch3", - "ch4", "ch5", "ch6", "ch7", - "ch8", "ch9", "ch10", "ch11", - "ch12", "ch13", "ch14", "ch15", - "ch16", "ch17", "ch18", "ch19", - "ch20", "ch21", "ch22", "ch23", - "ch24"; - clocks = <&cpg CPG_MOD 812>; - power-domains = <&cpg>; - phy-mode = "rgmii-id"; - phy-handle = <&phy0>; - - pinctrl-0 = <ðer_pins>; - pinctrl-names = "default"; - renesas,no-ether-link; - #address-cells = <1>; - #size-cells = <0>; - - phy0: ethernet-phy@0 { - rxc-skew-ps = <900>; - rxdv-skew-ps = <0>; - rxd0-skew-ps = <0>; - rxd1-skew-ps = <0>; - rxd2-skew-ps = <0>; - rxd3-skew-ps = <0>; - txc-skew-ps = <900>; - txen-skew-ps = <0>; - txd0-skew-ps = <0>; - txd1-skew-ps = <0>; - txd2-skew-ps = <0>; - txd3-skew-ps = <0>; - reg = <0>; - interrupt-parent = <&gpio2>; - interrupts = <11 IRQ_TYPE_LEVEL_LOW>; - }; - }; diff --git a/Documentation/networking/caif/index.rst b/Documentation/networking/caif/index.rst index 86e5b7832ec3..ec29b6f4bdb4 100644 --- a/Documentation/networking/caif/index.rst +++ b/Documentation/networking/caif/index.rst @@ -10,4 +10,3 @@ Contents: linux_caif caif - spi_porting diff --git a/Documentation/networking/caif/spi_porting.rst b/Documentation/networking/caif/spi_porting.rst deleted file mode 100644 index d49f874b20ac..000000000000 --- a/Documentation/networking/caif/spi_porting.rst +++ /dev/null @@ -1,229 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -================ -CAIF SPI porting -================ - -CAIF SPI basics -=============== - -Running CAIF over SPI needs some extra setup, owing to the nature of SPI. -Two extra GPIOs have been added in order to negotiate the transfers -between the master and the slave. The minimum requirement for running -CAIF over SPI is a SPI slave chip and two GPIOs (more details below). -Please note that running as a slave implies that you need to keep up -with the master clock. An overrun or underrun event is fatal. - -CAIF SPI framework -================== - -To make porting as easy as possible, the CAIF SPI has been divided in -two parts. The first part (called the interface part) deals with all -generic functionality such as length framing, SPI frame negotiation -and SPI frame delivery and transmission. The other part is the CAIF -SPI slave device part, which is the module that you have to write if -you want to run SPI CAIF on a new hardware. This part takes care of -the physical hardware, both with regard to SPI and to GPIOs. - -- Implementing a CAIF SPI device: - - - Functionality provided by the CAIF SPI slave device: - - In order to implement a SPI device you will, as a minimum, - need to implement the following - functions: - - :: - - int (*init_xfer) (struct cfspi_xfer * xfer, struct cfspi_dev *dev): - - This function is called by the CAIF SPI interface to give - you a chance to set up your hardware to be ready to receive - a stream of data from the master. The xfer structure contains - both physical and logical addresses, as well as the total length - of the transfer in both directions.The dev parameter can be used - to map to different CAIF SPI slave devices. - - :: - - void (*sig_xfer) (bool xfer, struct cfspi_dev *dev): - - This function is called by the CAIF SPI interface when the output - (SPI_INT) GPIO needs to change state. The boolean value of the xfer - variable indicates whether the GPIO should be asserted (HIGH) or - deasserted (LOW). The dev parameter can be used to map to different CAIF - SPI slave devices. - - - Functionality provided by the CAIF SPI interface: - - :: - - void (*ss_cb) (bool assert, struct cfspi_ifc *ifc); - - This function is called by the CAIF SPI slave device in order to - signal a change of state of the input GPIO (SS) to the interface. - Only active edges are mandatory to be reported. - This function can be called from IRQ context (recommended in order - not to introduce latency). The ifc parameter should be the pointer - returned from the platform probe function in the SPI device structure. - - :: - - void (*xfer_done_cb) (struct cfspi_ifc *ifc); - - This function is called by the CAIF SPI slave device in order to - report that a transfer is completed. This function should only be - called once both the transmission and the reception are completed. - This function can be called from IRQ context (recommended in order - not to introduce latency). The ifc parameter should be the pointer - returned from the platform probe function in the SPI device structure. - - - Connecting the bits and pieces: - - - Filling in the SPI slave device structure: - - Connect the necessary callback functions. - - Indicate clock speed (used to calculate toggle delays). - - Chose a suitable name (helps debugging if you use several CAIF - SPI slave devices). - - Assign your private data (can be used to map to your - structure). - - - Filling in the SPI slave platform device structure: - - Add name of driver to connect to ("cfspi_sspi"). - - Assign the SPI slave device structure as platform data. - -Padding -======= - -In order to optimize throughput, a number of SPI padding options are provided. -Padding can be enabled independently for uplink and downlink transfers. -Padding can be enabled for the head, the tail and for the total frame size. -The padding needs to be correctly configured on both sides of the link. -The padding can be changed via module parameters in cfspi_sspi.c or via -the sysfs directory of the cfspi_sspi driver (before device registration). - -- CAIF SPI device template:: - - /* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Daniel Martensson / Daniel.Martensson@stericsson.com - * License terms: GNU General Public License (GPL), version 2. - * - */ - - #include <linux/init.h> - #include <linux/module.h> - #include <linux/device.h> - #include <linux/wait.h> - #include <linux/interrupt.h> - #include <linux/dma-mapping.h> - #include <net/caif/caif_spi.h> - - MODULE_LICENSE("GPL"); - - struct sspi_struct { - struct cfspi_dev sdev; - struct cfspi_xfer *xfer; - }; - - static struct sspi_struct slave; - static struct platform_device slave_device; - - static irqreturn_t sspi_irq(int irq, void *arg) - { - /* You only need to trigger on an edge to the active state of the - * SS signal. Once a edge is detected, the ss_cb() function should be - * called with the parameter assert set to true. It is OK - * (and even advised) to call the ss_cb() function in IRQ context in - * order not to add any delay. */ - - return IRQ_HANDLED; - } - - static void sspi_complete(void *context) - { - /* Normally the DMA or the SPI framework will call you back - * in something similar to this. The only thing you need to - * do is to call the xfer_done_cb() function, providing the pointer - * to the CAIF SPI interface. It is OK to call this function - * from IRQ context. */ - } - - static int sspi_init_xfer(struct cfspi_xfer *xfer, struct cfspi_dev *dev) - { - /* Store transfer info. For a normal implementation you should - * set up your DMA here and make sure that you are ready to - * receive the data from the master SPI. */ - - struct sspi_struct *sspi = (struct sspi_struct *)dev->priv; - - sspi->xfer = xfer; - - return 0; - } - - void sspi_sig_xfer(bool xfer, struct cfspi_dev *dev) - { - /* If xfer is true then you should assert the SPI_INT to indicate to - * the master that you are ready to receive the data from the master - * SPI. If xfer is false then you should de-assert SPI_INT to indicate - * that the transfer is done. - */ - - struct sspi_struct *sspi = (struct sspi_struct *)dev->priv; - } - - static void sspi_release(struct device *dev) - { - /* - * Here you should release your SPI device resources. - */ - } - - static int __init sspi_init(void) - { - /* Here you should initialize your SPI device by providing the - * necessary functions, clock speed, name and private data. Once - * done, you can register your device with the - * platform_device_register() function. This function will return - * with the CAIF SPI interface initialized. This is probably also - * the place where you should set up your GPIOs, interrupts and SPI - * resources. */ - - int res = 0; - - /* Initialize slave device. */ - slave.sdev.init_xfer = sspi_init_xfer; - slave.sdev.sig_xfer = sspi_sig_xfer; - slave.sdev.clk_mhz = 13; - slave.sdev.priv = &slave; - slave.sdev.name = "spi_sspi"; - slave_device.dev.release = sspi_release; - - /* Initialize platform device. */ - slave_device.name = "cfspi_sspi"; - slave_device.dev.platform_data = &slave.sdev; - - /* Register platform device. */ - res = platform_device_register(&slave_device); - if (res) { - printk(KERN_WARNING "sspi_init: failed to register dev.\n"); - return -ENODEV; - } - - return res; - } - - static void __exit sspi_exit(void) - { - platform_device_del(&slave_device); - } - - module_init(sspi_init); - module_exit(sspi_exit); diff --git a/Documentation/networking/vxlan.rst b/Documentation/networking/vxlan.rst index ce239fa01848..2759dc1cc525 100644 --- a/Documentation/networking/vxlan.rst +++ b/Documentation/networking/vxlan.rst @@ -58,3 +58,31 @@ forwarding table using the new bridge command. 3. Show forwarding table:: # bridge fdb show dev vxlan0 + +The following NIC features may indicate support for UDP tunnel-related +offloads (most commonly VXLAN features, but support for a particular +encapsulation protocol is NIC specific): + + - `tx-udp_tnl-segmentation` + - `tx-udp_tnl-csum-segmentation` + ability to perform TCP segmentation offload of UDP encapsulated frames + + - `rx-udp_tunnel-port-offload` + receive side parsing of UDP encapsulated frames which allows NICs to + perform protocol-aware offloads, like checksum validation offload of + inner frames (only needed by NICs without protocol-agnostic offloads) + +For devices supporting `rx-udp_tunnel-port-offload` the list of currently +offloaded ports can be interrogated with `ethtool`:: + + $ ethtool --show-tunnels eth0 + Tunnel information for eth0: + UDP port table 0: + Size: 4 + Types: vxlan + No entries + UDP port table 1: + Size: 4 + Types: geneve, vxlan-gpe + Entries (1): + port 1230, vxlan-gpe diff --git a/MAINTAINERS b/MAINTAINERS index 42c69d2eeece..c2038d47e47f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12065,7 +12065,6 @@ M: Neil Horman <nhorman@tuxdriver.com> L: netdev@vger.kernel.org S: Maintained W: https://fedorahosted.org/dropwatch/ -F: include/net/drop_monitor.h F: include/uapi/linux/net_dropmon.h F: net/core/drop_monitor.c @@ -12159,6 +12158,7 @@ F: net/ipv6/ipcomp6.c F: net/ipv6/xfrm* F: net/key/ F: net/xfrm/ +F: tools/testing/selftests/net/ipsec.c NETWORKING [IPv4/IPv6] M: "David S. Miller" <davem@davemloft.net> @@ -12542,6 +12542,7 @@ F: drivers/net/dsa/ocelot/* F: drivers/net/ethernet/mscc/ F: include/soc/mscc/ocelot* F: net/dsa/tag_ocelot.c +F: tools/testing/selftests/drivers/net/ocelot/* OCXL (Open Coherent Accelerator Processor Interface OpenCAPI) DRIVER M: Frederic Barrat <fbarrat@linux.ibm.com> diff --git a/arch/mips/boot/dts/mscc/ocelot.dtsi b/arch/mips/boot/dts/mscc/ocelot.dtsi index f94e8a02ed06..535a98284dcb 100644 --- a/arch/mips/boot/dts/mscc/ocelot.dtsi +++ b/arch/mips/boot/dts/mscc/ocelot.dtsi @@ -134,11 +134,13 @@ <0x1280000 0x100>, <0x1800000 0x80000>, <0x1880000 0x10000>, + <0x1040000 0x10000>, + <0x1050000 0x10000>, <0x1060000 0x10000>; reg-names = "sys", "rew", "qs", "ptp", "port0", "port1", "port2", "port3", "port4", "port5", "port6", "port7", "port8", "port9", "port10", "qsys", - "ana", "s2"; + "ana", "s0", "s1", "s2"; interrupts = <18 21 22>; interrupt-names = "ptp_rdy", "xtr", "inj"; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 26f43279b78b..796506dcfc42 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -281,7 +281,8 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, EMIT1(0x55); /* push rbp */ EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ /* sub rsp, rounded_stack_depth */ - EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8)); + if (stack_depth) + EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8)); if (tail_call_reachable) EMIT1(0x50); /* push rax */ *pprog = prog; @@ -407,9 +408,9 @@ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, int tcc_off = -4 - round_up(stack_depth, 8); u8 *prog = *pprog; int pop_bytes = 0; - int off1 = 49; - int off2 = 38; - int off3 = 16; + int off1 = 42; + int off2 = 31; + int off3 = 9; int cnt = 0; /* count the additional bytes used for popping callee regs from stack @@ -421,6 +422,12 @@ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, off2 += pop_bytes; off3 += pop_bytes; + if (stack_depth) { + off1 += 7; + off2 += 7; + off3 += 7; + } + /* * rdi - pointer to ctx * rsi - pointer to bpf_array @@ -465,8 +472,9 @@ static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, prog = *pprog; EMIT1(0x58); /* pop rax */ - EMIT3_off32(0x48, 0x81, 0xC4, /* add rsp, sd */ - round_up(stack_depth, 8)); + if (stack_depth) + EMIT3_off32(0x48, 0x81, 0xC4, /* add rsp, sd */ + round_up(stack_depth, 8)); /* goto *(prog->bpf_func + X86_TAIL_CALL_OFFSET); */ EMIT4(0x48, 0x8B, 0x49, /* mov rcx, qword ptr [rcx + 32] */ @@ -491,7 +499,7 @@ static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, int tcc_off = -4 - round_up(stack_depth, 8); u8 *prog = *pprog; int pop_bytes = 0; - int off1 = 27; + int off1 = 20; int poke_off; int cnt = 0; @@ -506,10 +514,14 @@ static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, * total bytes for: * - nop5/ jmpq $off * - pop callee regs - * - sub rsp, $val + * - sub rsp, $val if depth > 0 * - pop rax */ - poke_off = X86_PATCH_SIZE + pop_bytes + 7 + 1; + poke_off = X86_PATCH_SIZE + pop_bytes + 1; + if (stack_depth) { + poke_off += 7; + off1 += 7; + } /* * if (tail_call_cnt > MAX_TAIL_CALL_CNT) @@ -533,7 +545,8 @@ static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, pop_callee_regs(pprog, callee_regs_used); prog = *pprog; EMIT1(0x58); /* pop rax */ - EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8)); + if (stack_depth) + EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8)); memcpy(prog, ideal_nops[NOP_ATOMIC5], X86_PATCH_SIZE); prog += X86_PATCH_SIZE; @@ -1441,8 +1454,6 @@ emit_jmp: /* Update cleanup_addr */ ctx->cleanup_addr = proglen; pop_callee_regs(&prog, callee_regs_used); - if (tail_call_reachable) - EMIT1(0x59); /* pop rcx, get rid of tail_call_cnt */ EMIT1(0xC9); /* leave */ EMIT1(0xC3); /* ret */ break; diff --git a/drivers/atm/atmtcp.c b/drivers/atm/atmtcp.c index 7f814da3c2d0..96bea1ab1ecc 100644 --- a/drivers/atm/atmtcp.c +++ b/drivers/atm/atmtcp.c @@ -327,7 +327,7 @@ done: */ -static struct atmdev_ops atmtcp_v_dev_ops = { +static const struct atmdev_ops atmtcp_v_dev_ops = { .dev_close = atmtcp_v_dev_close, .open = atmtcp_v_open, .close = atmtcp_v_close, diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index 5fa5be3c5598..88ce5f0ffc4b 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -18,7 +18,11 @@ #define VERSION "0.1" -#define BDADDR_INTEL (&(bdaddr_t) {{0x00, 0x8b, 0x9e, 0x19, 0x03, 0x00}}) +#define BDADDR_INTEL (&(bdaddr_t){{0x00, 0x8b, 0x9e, 0x19, 0x03, 0x00}}) +#define RSA_HEADER_LEN 644 +#define CSS_HEADER_OFFSET 8 +#define ECDSA_OFFSET 644 +#define ECDSA_HEADER_LEN 320 int btintel_check_bdaddr(struct hci_dev *hdev) { @@ -360,6 +364,144 @@ int btintel_read_version(struct hci_dev *hdev, struct intel_version *ver) } EXPORT_SYMBOL_GPL(btintel_read_version); +void btintel_version_info_tlv(struct hci_dev *hdev, struct intel_version_tlv *version) +{ + const char *variant; + + switch (version->img_type) { + case 0x01: + variant = "Bootloader"; + bt_dev_info(hdev, "Device revision is %u", version->dev_rev_id); + bt_dev_info(hdev, "Secure boot is %s", + version->secure_boot ? "enabled" : "disabled"); + bt_dev_info(hdev, "OTP lock is %s", + version->otp_lock ? "enabled" : "disabled"); + bt_dev_info(hdev, "API lock is %s", + version->api_lock ? "enabled" : "disabled"); + bt_dev_info(hdev, "Debug lock is %s", + version->debug_lock ? "enabled" : "disabled"); + bt_dev_info(hdev, "Minimum firmware build %u week %u %u", + version->min_fw_build_nn, version->min_fw_build_cw, + 2000 + version->min_fw_build_yy); + break; + case 0x03: + variant = "Firmware"; + break; + default: + bt_dev_err(hdev, "Unsupported image type(%02x)", version->img_type); + goto done; + } + + bt_dev_info(hdev, "%s timestamp %u.%u buildtype %u build %u", variant, + 2000 + (version->timestamp >> 8), version->timestamp & 0xff, + version->build_type, version->build_num); + +done: + return; +} +EXPORT_SYMBOL_GPL(btintel_version_info_tlv); + +int btintel_read_version_tlv(struct hci_dev *hdev, struct intel_version_tlv *version) +{ + struct sk_buff *skb; + const u8 param[1] = { 0xFF }; + + if (!version) + return -EINVAL; + + skb = __hci_cmd_sync(hdev, 0xfc05, 1, param, HCI_CMD_TIMEOUT); + if (IS_ERR(skb)) { + bt_dev_err(hdev, "Reading Intel version information failed (%ld)", + PTR_ERR(skb)); + return PTR_ERR(skb); + } + + if (skb->data[0]) { + bt_dev_err(hdev, "Intel Read Version command failed (%02x)", + skb->data[0]); + kfree_skb(skb); + return -EIO; + } + + /* Consume Command Complete Status field */ + skb_pull(skb, 1); + + /* Event parameters contatin multiple TLVs. Read each of them + * and only keep the required data. Also, it use existing legacy + * version field like hw_platform, hw_variant, and fw_variant + * to keep the existing setup flow + */ + while (skb->len) { + struct intel_tlv *tlv; + + tlv = (struct intel_tlv *)skb->data; + switch (tlv->type) { + case INTEL_TLV_CNVI_TOP: + version->cnvi_top = get_unaligned_le32(tlv->val); + break; + case INTEL_TLV_CNVR_TOP: + version->cnvr_top = get_unaligned_le32(tlv->val); + break; + case INTEL_TLV_CNVI_BT: + version->cnvi_bt = get_unaligned_le32(tlv->val); + break; + case INTEL_TLV_CNVR_BT: + version->cnvr_bt = get_unaligned_le32(tlv->val); + break; + case INTEL_TLV_DEV_REV_ID: + version->dev_rev_id = get_unaligned_le16(tlv->val); + break; + case INTEL_TLV_IMAGE_TYPE: + version->img_type = tlv->val[0]; + break; + case INTEL_TLV_TIME_STAMP: + version->timestamp = get_unaligned_le16(tlv->val); + break; + case INTEL_TLV_BUILD_TYPE: + version->build_type = tlv->val[0]; + break; + case INTEL_TLV_BUILD_NUM: + version->build_num = get_unaligned_le32(tlv->val); + break; + case INTEL_TLV_SECURE_BOOT: + version->secure_boot = tlv->val[0]; + break; + case INTEL_TLV_OTP_LOCK: + version->otp_lock = tlv->val[0]; + break; + case INTEL_TLV_API_LOCK: + version->api_lock = tlv->val[0]; + break; + case INTEL_TLV_DEBUG_LOCK: + version->debug_lock = tlv->val[0]; + break; + case INTEL_TLV_MIN_FW: + version->min_fw_build_nn = tlv->val[0]; + version->min_fw_build_cw = tlv->val[1]; + version->min_fw_build_yy = tlv->val[2]; + break; + case INTEL_TLV_LIMITED_CCE: + version->limited_cce = tlv->val[0]; + break; + case INTEL_TLV_SBE_TYPE: + version->sbe_type = tlv->val[0]; + break; + case INTEL_TLV_OTP_BDADDR: + memcpy(&version->otp_bd_addr, tlv->val, tlv->len); + break; + default: + /* Ignore rest of information */ + break; + } + /* consume the current tlv and move to next*/ + skb_pull(skb, tlv->len + sizeof(*tlv)); + } + + kfree_skb(skb); + return 0; +} +EXPORT_SYMBOL_GPL(btintel_read_version_tlv); + /* ------- REGMAP IBT SUPPORT ------- */ #define IBT_REG_MODE_8BIT 0x00 @@ -626,12 +768,10 @@ int btintel_read_boot_params(struct hci_dev *hdev, } EXPORT_SYMBOL_GPL(btintel_read_boot_params); -int btintel_download_firmware(struct hci_dev *hdev, const struct firmware *fw, - u32 *boot_param) +static int btintel_sfi_rsa_header_secure_send(struct hci_dev *hdev, + const struct firmware *fw) { int err; - const u8 *fw_ptr; - u32 frag_len; /* Start the firmware download transaction with the Init fragment * represented by the 128 bytes of CSS header. @@ -660,8 +800,56 @@ int btintel_download_firmware(struct hci_dev *hdev, const struct firmware *fw, goto done; } - fw_ptr = fw->data + 644; +done: + return err; +} + +static int btintel_sfi_ecdsa_header_secure_send(struct hci_dev *hdev, + const struct firmware *fw) +{ + int err; + + /* Start the firmware download transaction with the Init fragment + * represented by the 128 bytes of CSS header. + */ + err = btintel_secure_send(hdev, 0x00, 128, fw->data + 644); + if (err < 0) { + bt_dev_err(hdev, "Failed to send firmware header (%d)", err); + return err; + } + + /* Send the 96 bytes of public key information from the firmware + * as the PKey fragment. + */ + err = btintel_secure_send(hdev, 0x03, 96, fw->data + 644 + 128); + if (err < 0) { + bt_dev_err(hdev, "Failed to send firmware pkey (%d)", err); + return err; + } + + /* Send the 96 bytes of signature information from the firmware + * as the Sign fragment + */ + err = btintel_secure_send(hdev, 0x02, 96, fw->data + 644 + 224); + if (err < 0) { + bt_dev_err(hdev, "Failed to send firmware signature (%d)", + err); + return err; + } + return 0; +} + +static int btintel_download_firmware_payload(struct hci_dev *hdev, + const struct firmware *fw, + u32 *boot_param, size_t offset) +{ + int err; + const u8 *fw_ptr; + u32 frag_len; + + fw_ptr = fw->data + offset; frag_len = 0; + err = -EINVAL; while (fw_ptr - fw->data < fw->size) { struct hci_command_hdr *cmd = (void *)(fw_ptr + frag_len); @@ -707,8 +895,99 @@ int btintel_download_firmware(struct hci_dev *hdev, const struct firmware *fw, done: return err; } + +int btintel_download_firmware(struct hci_dev *hdev, + const struct firmware *fw, + u32 *boot_param) +{ + int err; + + err = btintel_sfi_rsa_header_secure_send(hdev, fw); + if (err) + return err; + + return btintel_download_firmware_payload(hdev, fw, boot_param, + RSA_HEADER_LEN); +} EXPORT_SYMBOL_GPL(btintel_download_firmware); +int btintel_download_firmware_newgen(struct hci_dev *hdev, + const struct firmware *fw, u32 *boot_param, + u8 hw_variant, u8 sbe_type) +{ + int err; + u32 css_header_ver; + + /* iBT hardware variants 0x0b, 0x0c, 0x11, 0x12, 0x13, 0x14 support + * only RSA secure boot engine. Hence, the corresponding sfi file will + * have RSA header of 644 bytes followed by Command Buffer. + * + * iBT hardware variants 0x17, 0x18 onwards support both RSA and ECDSA + * secure boot engine. As a result, the corresponding sfi file will + * have RSA header of 644, ECDSA header of 320 bytes followed by + * Command Buffer. + * + * CSS Header byte positions 0x08 to 0x0B represent the CSS Header + * version: RSA(0x00010000) , ECDSA (0x00020000) + */ + css_header_ver = get_unaligned_le32(fw->data + CSS_HEADER_OFFSET); + if (css_header_ver != 0x00010000) { + bt_dev_err(hdev, "Invalid CSS Header version"); + return -EINVAL; + } + + if (hw_variant <= 0x14) { + if (sbe_type != 0x00) { + bt_dev_err(hdev, "Invalid SBE type for hardware variant (%d)", + hw_variant); + return -EINVAL; + } + + err = btintel_sfi_rsa_header_secure_send(hdev, fw); + if (err) + return err; + + err = btintel_download_firmware_payload(hdev, fw, boot_param, RSA_HEADER_LEN); + if (err) + return err; + } else if (hw_variant >= 0x17) { + /* Check if CSS header for ECDSA follows the RSA header */ + if (fw->data[ECDSA_OFFSET] != 0x06) + return -EINVAL; + + /* Check if the CSS Header version is ECDSA(0x00020000) */ + css_header_ver = get_unaligned_le32(fw->data + ECDSA_OFFSET + CSS_HEADER_OFFSET); + if (css_header_ver != 0x00020000) { + bt_dev_err(hdev, "Invalid CSS Header version"); + return -EINVAL; + } + + if (sbe_type == 0x00) { + err = btintel_sfi_rsa_header_secure_send(hdev, fw); + if (err) + return err; + + err = btintel_download_firmware_payload(hdev, fw, + boot_param, + RSA_HEADER_LEN + ECDSA_HEADER_LEN); + if (err) + return err; + } else if (sbe_type == 0x01) { + err = btintel_sfi_ecdsa_header_secure_send(hdev, fw); + if (err) + return err; + + err = btintel_download_firmware_payload(hdev, fw, + boot_param, + RSA_HEADER_LEN + ECDSA_HEADER_LEN); + if (err) + return err; + } + } + return 0; +} +EXPORT_SYMBOL_GPL(btintel_download_firmware_newgen); + void btintel_reset_to_bootloader(struct hci_dev *hdev) { struct intel_reset params; diff --git a/drivers/bluetooth/btintel.h b/drivers/bluetooth/btintel.h index 08e20606fb58..09346ae308eb 100644 --- a/drivers/bluetooth/btintel.h +++ b/drivers/bluetooth/btintel.h @@ -6,6 +6,72 @@ * Copyright (C) 2015 Intel Corporation */ +/* List of tlv type */ +enum { + INTEL_TLV_CNVI_TOP = 0x10, + INTEL_TLV_CNVR_TOP, + INTEL_TLV_CNVI_BT, + INTEL_TLV_CNVR_BT, + INTEL_TLV_CNVI_OTP, + INTEL_TLV_CNVR_OTP, + INTEL_TLV_DEV_REV_ID, + INTEL_TLV_USB_VENDOR_ID, + INTEL_TLV_USB_PRODUCT_ID, + INTEL_TLV_PCIE_VENDOR_ID, + INTEL_TLV_PCIE_DEVICE_ID, + INTEL_TLV_PCIE_SUBSYSTEM_ID, + INTEL_TLV_IMAGE_TYPE, + INTEL_TLV_TIME_STAMP, + INTEL_TLV_BUILD_TYPE, + INTEL_TLV_BUILD_NUM, + INTEL_TLV_FW_BUILD_PRODUCT, + INTEL_TLV_FW_BUILD_HW, + INTEL_TLV_FW_STEP, + INTEL_TLV_BT_SPEC, + INTEL_TLV_MFG_NAME, + INTEL_TLV_HCI_REV, + INTEL_TLV_LMP_SUBVER, + INTEL_TLV_OTP_PATCH_VER, + INTEL_TLV_SECURE_BOOT, + INTEL_TLV_KEY_FROM_HDR, + INTEL_TLV_OTP_LOCK, + INTEL_TLV_API_LOCK, + INTEL_TLV_DEBUG_LOCK, + INTEL_TLV_MIN_FW, + INTEL_TLV_LIMITED_CCE, + INTEL_TLV_SBE_TYPE, + INTEL_TLV_OTP_BDADDR, + INTEL_TLV_UNLOCKED_STATE +}; + +struct intel_tlv { + u8 type; + u8 len; + u8 val[0]; +} __packed; + +struct intel_version_tlv { + u32 cnvi_top; + u32 cnvr_top; + u32 cnvi_bt; + u32 cnvr_bt; + u16 dev_rev_id; + u8 img_type; + u16 timestamp; + u8 build_type; + u32 build_num; + u8 secure_boot; + u8 otp_lock; + u8 api_lock; + u8 debug_lock; + u8 min_fw_build_nn; + u8 min_fw_build_cw; + u8 min_fw_build_yy; + u8 limited_cce; + u8 sbe_type; + bdaddr_t otp_bd_addr; +}; + struct intel_version { u8 status; u8 hw_platform; @@ -77,12 +143,14 @@ int btintel_set_diag_mfg(struct hci_dev *hdev, bool enable); void btintel_hw_error(struct hci_dev *hdev, u8 code); void btintel_version_info(struct hci_dev *hdev, struct intel_version *ver); +void btintel_version_info_tlv(struct hci_dev *hdev, struct intel_version_tlv *version); int btintel_secure_send(struct hci_dev *hdev, u8 fragment_type, u32 plen, const void *param); int btintel_load_ddc_config(struct hci_dev *hdev, const char *ddc_name); int btintel_set_event_mask(struct hci_dev *hdev, bool debug); int btintel_set_event_mask_mfg(struct hci_dev *hdev, bool debug); int btintel_read_version(struct hci_dev *hdev, struct intel_version *ver); +int btintel_read_version_tlv(struct hci_dev *hdev, struct intel_version_tlv *ver); struct regmap *btintel_regmap_init(struct hci_dev *hdev, u16 opcode_read, u16 opcode_write); @@ -91,6 +159,10 @@ int btintel_read_boot_params(struct hci_dev *hdev, struct intel_boot_params *params); int btintel_download_firmware(struct hci_dev *dev, const struct firmware *fw, u32 *boot_param); +int btintel_download_firmware_newgen(struct hci_dev *hdev, + const struct firmware *fw, + u32 *boot_param, u8 hw_variant, + u8 sbe_type); void btintel_reset_to_bootloader(struct hci_dev *hdev); int btintel_read_debug_features(struct hci_dev *hdev, struct intel_debug_features *features); @@ -137,6 +209,11 @@ static inline void btintel_version_info(struct hci_dev *hdev, { } +static inline void btintel_version_info_tlv(struct hci_dev *hdev, + struct intel_version_tlv *version) +{ +} + static inline int btintel_secure_send(struct hci_dev *hdev, u8 fragment_type, u32 plen, const void *param) { @@ -165,6 +242,12 @@ static inline int btintel_read_version(struct hci_dev *hdev, return -EOPNOTSUPP; } +static inline int btintel_read_version_tlv(struct hci_dev *hdev, + struct intel_version_tlv *ver) +{ + return -EOPNOTSUPP; +} + static inline struct regmap *btintel_regmap_init(struct hci_dev *hdev, u16 opcode_read, u16 opcode_write) @@ -191,6 +274,14 @@ static inline int btintel_download_firmware(struct hci_dev *dev, return -EOPNOTSUPP; } +static inline int btintel_download_firmware_newgen(struct hci_dev *hdev, + const struct firmware *fw, + u32 *boot_param, + u8 hw_variant, u8 sbe_type) +{ + return -EOPNOTSUPP; +} + static inline void btintel_reset_to_bootloader(struct hci_dev *hdev) { } diff --git a/drivers/bluetooth/btmrvl_sdio.c b/drivers/bluetooth/btmrvl_sdio.c index d15fd5be0216..33d58b30c5ac 100644 --- a/drivers/bluetooth/btmrvl_sdio.c +++ b/drivers/bluetooth/btmrvl_sdio.c @@ -215,30 +215,7 @@ static const struct btmrvl_sdio_card_reg btmrvl_reg_8897 = { .fw_dump_end = 0xea, }; -static const struct btmrvl_sdio_card_reg btmrvl_reg_8977 = { - .cfg = 0x00, - .host_int_mask = 0x08, - .host_intstatus = 0x0c, - .card_status = 0x5c, - .sq_read_base_addr_a0 = 0xf8, - .sq_read_base_addr_a1 = 0xf9, - .card_revision = 0xc8, - .card_fw_status0 = 0xe8, - .card_fw_status1 = 0xe9, - .card_rx_len = 0xea, - .card_rx_unit = 0xeb, - .io_port_0 = 0xe4, - .io_port_1 = 0xe5, - .io_port_2 = 0xe6, - .int_read_to_clear = true, - .host_int_rsr = 0x04, - .card_misc_cfg = 0xD8, - .fw_dump_ctrl = 0xf0, - .fw_dump_start = 0xf1, - .fw_dump_end = 0xf8, -}; - -static const struct btmrvl_sdio_card_reg btmrvl_reg_8987 = { +static const struct btmrvl_sdio_card_reg btmrvl_reg_89xx = { .cfg = 0x00, .host_int_mask = 0x08, .host_intstatus = 0x0c, @@ -261,29 +238,6 @@ static const struct btmrvl_sdio_card_reg btmrvl_reg_8987 = { .fw_dump_end = 0xf8, }; -static const struct btmrvl_sdio_card_reg btmrvl_reg_8997 = { - .cfg = 0x00, - .host_int_mask = 0x08, - .host_intstatus = 0x0c, - .card_status = 0x5c, - .sq_read_base_addr_a0 = 0xf8, - .sq_read_base_addr_a1 = 0xf9, - .card_revision = 0xc8, - .card_fw_status0 = 0xe8, - .card_fw_status1 = 0xe9, - .card_rx_len = 0xea, - .card_rx_unit = 0xeb, - .io_port_0 = 0xe4, - .io_port_1 = 0xe5, - .io_port_2 = 0xe6, - .int_read_to_clear = true, - .host_int_rsr = 0x04, - .card_misc_cfg = 0xD8, - .fw_dump_ctrl = 0xf0, - .fw_dump_start = 0xf1, - .fw_dump_end = 0xf8, -}; - static const struct btmrvl_sdio_device btmrvl_sdio_sd8688 = { .helper = "mrvl/sd8688_helper.bin", .firmware = "mrvl/sd8688.bin", @@ -332,7 +286,7 @@ static const struct btmrvl_sdio_device btmrvl_sdio_sd8897 = { static const struct btmrvl_sdio_device btmrvl_sdio_sd8977 = { .helper = NULL, .firmware = "mrvl/sdsd8977_combo_v2.bin", - .reg = &btmrvl_reg_8977, + .reg = &btmrvl_reg_89xx, .support_pscan_win_report = true, .sd_blksz_fw_dl = 256, .supports_fw_dump = true, @@ -341,7 +295,7 @@ static const struct btmrvl_sdio_device btmrvl_sdio_sd8977 = { static const struct btmrvl_sdio_device btmrvl_sdio_sd8987 = { .helper = NULL, .firmware = "mrvl/sd8987_uapsta.bin", - .reg = &btmrvl_reg_8987, + .reg = &btmrvl_reg_89xx, .support_pscan_win_report = true, .sd_blksz_fw_dl = 256, .supports_fw_dump = true, @@ -350,7 +304,7 @@ static const struct btmrvl_sdio_device btmrvl_sdio_sd8987 = { static const struct btmrvl_sdio_device btmrvl_sdio_sd8997 = { .helper = NULL, .firmware = "mrvl/sdsd8997_combo_v4.bin", - .reg = &btmrvl_reg_8997, + .reg = &btmrvl_reg_89xx, .support_pscan_win_report = true, .sd_blksz_fw_dl = 256, .supports_fw_dump = true, diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c index c7ab7a23bd67..ba45c59bd9f3 100644 --- a/drivers/bluetooth/btmtksdio.c +++ b/drivers/bluetooth/btmtksdio.c @@ -496,7 +496,7 @@ static void btmtksdio_interrupt(struct sdio_func *func) sdio_claim_host(bdev->func); /* Disable interrupt */ - sdio_writel(func, C_INT_EN_CLR, MTK_REG_CHLPCR, 0); + sdio_writel(func, C_INT_EN_CLR, MTK_REG_CHLPCR, NULL); int_status = sdio_readl(func, MTK_REG_CHISR, NULL); @@ -530,7 +530,7 @@ static void btmtksdio_interrupt(struct sdio_func *func) } /* Enable interrupt */ - sdio_writel(func, C_INT_EN_SET, MTK_REG_CHLPCR, 0); + sdio_writel(func, C_INT_EN_SET, MTK_REG_CHLPCR, NULL); pm_runtime_mark_last_busy(bdev->dev); pm_runtime_put_autosuspend(bdev->dev); diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index 8d2608ddfd08..1005b6e8ff74 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -59,6 +59,7 @@ static struct usb_driver btusb_driver; #define BTUSB_MEDIATEK 0x200000 #define BTUSB_WIDEBAND_SPEECH 0x400000 #define BTUSB_VALID_LE_STATES 0x800000 +#define BTUSB_QCA_WCN6855 0x1000000 static const struct usb_device_id btusb_table[] = { /* Generic Bluetooth USB device */ @@ -254,24 +255,46 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x0489, 0xe03c), .driver_info = BTUSB_ATH3012 }, /* QCA ROME chipset */ - { USB_DEVICE(0x0cf3, 0x535b), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x0cf3, 0xe007), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x0cf3, 0xe009), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x0cf3, 0xe010), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x0cf3, 0xe300), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x0cf3, 0xe301), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x0cf3, 0xe360), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x0489, 0xe092), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x0489, 0xe09f), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x0489, 0xe0a2), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x04ca, 0x3011), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x04ca, 0x3015), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x04ca, 0x3016), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x04ca, 0x301a), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x04ca, 0x3021), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x13d3, 0x3491), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x13d3, 0x3496), .driver_info = BTUSB_QCA_ROME }, - { USB_DEVICE(0x13d3, 0x3501), .driver_info = BTUSB_QCA_ROME }, + { USB_DEVICE(0x0cf3, 0x535b), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0cf3, 0xe007), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0cf3, 0xe009), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0cf3, 0xe010), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0cf3, 0xe300), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0cf3, 0xe301), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0cf3, 0xe360), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe092), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe09f), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe0a2), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x04ca, 0x3011), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x04ca, 0x3015), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x04ca, 0x3016), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x04ca, 0x301a), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x04ca, 0x3021), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3491), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3496), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3501), .driver_info = BTUSB_QCA_ROME | + BTUSB_WIDEBAND_SPEECH }, + + /* QCA WCN6855 chipset */ + { USB_DEVICE(0x0cf3, 0xe600), .driver_info = BTUSB_QCA_WCN6855 | + BTUSB_WIDEBAND_SPEECH }, /* Broadcom BCM2035 */ { USB_DEVICE(0x0a5c, 0x2009), .driver_info = BTUSB_BCM92035 }, @@ -2338,10 +2361,10 @@ static bool btusb_setup_intel_new_get_fw_name(struct intel_version *ver, static int btusb_intel_download_firmware(struct hci_dev *hdev, struct intel_version *ver, - struct intel_boot_params *params) + struct intel_boot_params *params, + u32 *boot_param) { const struct firmware *fw; - u32 boot_param; char fwname[64]; int err; struct btusb_data *data = hci_get_drvdata(hdev); @@ -2479,7 +2502,7 @@ static int btusb_intel_download_firmware(struct hci_dev *hdev, set_bit(BTUSB_DOWNLOADING, &data->flags); /* Start firmware downloading and get boot parameter */ - err = btintel_download_firmware(hdev, fw, &boot_param); + err = btintel_download_firmware(hdev, fw, boot_param); if (err < 0) { /* When FW download fails, send Intel Reset to retry * FW download. @@ -2561,7 +2584,7 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) return err; } - err = btusb_intel_download_firmware(hdev, &ver, ¶ms); + err = btusb_intel_download_firmware(hdev, &ver, ¶ms, &boot_param); if (err) return err; @@ -2896,6 +2919,7 @@ static int btusb_mtk_submit_wmt_recv_urb(struct hci_dev *hdev) buf = kmalloc(size, GFP_KERNEL); if (!buf) { kfree(dr); + usb_free_urb(urb); return -ENOMEM; } @@ -3390,6 +3414,27 @@ static int btusb_set_bdaddr_ath3012(struct hci_dev *hdev, return 0; } +static int btusb_set_bdaddr_wcn6855(struct hci_dev *hdev, + const bdaddr_t *bdaddr) +{ + struct sk_buff *skb; + u8 buf[6]; + long ret; + + memcpy(buf, bdaddr, sizeof(bdaddr_t)); + + skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(buf), buf, + HCI_EV_CMD_COMPLETE, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + ret = PTR_ERR(skb); + bt_dev_err(hdev, "Change address command failed (%ld)", ret); + return ret; + } + kfree_skb(skb); + + return 0; +} + #define QCA_DFU_PACKET_LEN 4096 #define QCA_GET_TARGET_VERSION 0x09 @@ -3409,7 +3454,8 @@ struct qca_version { } __packed; struct qca_rampatch_version { - __le16 rom_version; + __le16 rom_version_high; + __le16 rom_version_low; __le16 patch_version; } __packed; @@ -3421,12 +3467,14 @@ struct qca_device_info { }; static const struct qca_device_info qca_devices_table[] = { - { 0x00000100, 20, 4, 10 }, /* Rome 1.0 */ - { 0x00000101, 20, 4, 10 }, /* Rome 1.1 */ - { 0x00000200, 28, 4, 18 }, /* Rome 2.0 */ - { 0x00000201, 28, 4, 18 }, /* Rome 2.1 */ - { 0x00000300, 28, 4, 18 }, /* Rome 3.0 */ - { 0x00000302, 28, 4, 18 }, /* Rome 3.2 */ + { 0x00000100, 20, 4, 8 }, /* Rome 1.0 */ + { 0x00000101, 20, 4, 8 }, /* Rome 1.1 */ + { 0x00000200, 28, 4, 16 }, /* Rome 2.0 */ + { 0x00000201, 28, 4, 16 }, /* Rome 2.1 */ + { 0x00000300, 28, 4, 16 }, /* Rome 3.0 */ + { 0x00000302, 28, 4, 16 }, /* Rome 3.2 */ + { 0x00130100, 40, 4, 16 }, /* WCN6855 1.0 */ + { 0x00130200, 40, 4, 16 }, /* WCN6855 2.0 */ }; static int btusb_qca_send_vendor_req(struct usb_device *udev, u8 request, @@ -3528,8 +3576,8 @@ static int btusb_setup_qca_load_rampatch(struct hci_dev *hdev, { struct qca_rampatch_version *rver; const struct firmware *fw; - u32 ver_rom, ver_patch; - u16 rver_rom, rver_patch; + u32 ver_rom, ver_patch, rver_rom; + u16 rver_rom_low, rver_rom_high, rver_patch; char fwname[64]; int err; @@ -3548,9 +3596,16 @@ static int btusb_setup_qca_load_rampatch(struct hci_dev *hdev, bt_dev_info(hdev, "using rampatch file: %s", fwname); rver = (struct qca_rampatch_version *)(fw->data + info->ver_offset); - rver_rom = le16_to_cpu(rver->rom_version); + rver_rom_low = le16_to_cpu(rver->rom_version_low); rver_patch = le16_to_cpu(rver->patch_version); + if (ver_rom & ~0xffffU) { + rver_rom_high = le16_to_cpu(rver->rom_version_high); + rver_rom = le32_to_cpu(rver_rom_high << 16 | rver_rom_low); + } else { + rver_rom = rver_rom_low; + } + bt_dev_info(hdev, "QCA: patch rome 0x%x build 0x%x, " "firmware rome 0x%x build 0x%x", rver_rom, rver_patch, ver_rom, ver_patch); @@ -3624,9 +3679,6 @@ static int btusb_setup_qca(struct hci_dev *hdev) return err; ver_rom = le32_to_cpu(ver.rom_version); - /* Don't care about high ROM versions */ - if (ver_rom & ~0xffffU) - return 0; for (i = 0; i < ARRAY_SIZE(qca_devices_table); i++) { if (ver_rom == qca_devices_table[i].rom_version) @@ -4062,6 +4114,13 @@ static int btusb_probe(struct usb_interface *intf, btusb_check_needs_reset_resume(intf); } + if (id->driver_info & BTUSB_QCA_WCN6855) { + data->setup_on_usb = btusb_setup_qca; + hdev->set_bdaddr = btusb_set_bdaddr_wcn6855; + hdev->cmd_timeout = btusb_qca_cmd_timeout; + set_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks); + } + if (id->driver_info & BTUSB_AMP) { /* AMP controllers do not support SCO packets */ data->isoc = NULL; diff --git a/drivers/bluetooth/hci_h5.c b/drivers/bluetooth/hci_h5.c index e41854e0d79a..981d96cc7695 100644 --- a/drivers/bluetooth/hci_h5.c +++ b/drivers/bluetooth/hci_h5.c @@ -793,8 +793,6 @@ static int h5_serdev_probe(struct serdev_device *serdev) if (!h5) return -ENOMEM; - set_bit(HCI_UART_RESET_ON_INIT, &h5->serdev_hu.hdev_flags); - h5->hu = &h5->serdev_hu; h5->serdev_hu.serdev = serdev; serdev_device_set_drvdata(serdev, h5); diff --git a/drivers/bluetooth/hci_intel.c b/drivers/bluetooth/hci_intel.c index f1299da6eed8..b20a40fab83e 100644 --- a/drivers/bluetooth/hci_intel.c +++ b/drivers/bluetooth/hci_intel.c @@ -288,7 +288,7 @@ static irqreturn_t intel_irq(int irq, void *dev_id) static int intel_set_power(struct hci_uart *hu, bool powered) { - struct list_head *p; + struct intel_device *idev; int err = -ENODEV; if (!hu->tty->dev) @@ -296,10 +296,7 @@ static int intel_set_power(struct hci_uart *hu, bool powered) mutex_lock(&intel_device_list_lock); - list_for_each(p, &intel_device_list) { - struct intel_device *idev = list_entry(p, struct intel_device, - list); - + list_for_each_entry(idev, &intel_device_list, list) { /* tty device and pdev device should share the same parent * which is the UART port. */ @@ -362,19 +359,16 @@ static int intel_set_power(struct hci_uart *hu, bool powered) static void intel_busy_work(struct work_struct *work) { - struct list_head *p; struct intel_data *intel = container_of(work, struct intel_data, busy_work); + struct intel_device *idev; if (!intel->hu->tty->dev) return; /* Link is busy, delay the suspend */ mutex_lock(&intel_device_list_lock); - list_for_each(p, &intel_device_list) { - struct intel_device *idev = list_entry(p, struct intel_device, - list); - + list_for_each_entry(idev, &intel_device_list, list) { if (intel->hu->tty->dev->parent == idev->pdev->dev.parent) { pm_runtime_get(&idev->pdev->dev); pm_runtime_mark_last_busy(&idev->pdev->dev); @@ -533,7 +527,7 @@ static int intel_setup(struct hci_uart *hu) struct sk_buff *skb; struct intel_version ver; struct intel_boot_params params; - struct list_head *p; + struct intel_device *idev; const struct firmware *fw; char fwname[64]; u32 boot_param; @@ -693,14 +687,11 @@ static int intel_setup(struct hci_uart *hu) case 0x0b: /* SfP */ case 0x0c: /* WsP */ snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.sfi", - le16_to_cpu(ver.hw_variant), - le16_to_cpu(params.dev_revid)); + ver.hw_variant, le16_to_cpu(params.dev_revid)); break; case 0x12: /* ThP */ snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u-%u.sfi", - le16_to_cpu(ver.hw_variant), - le16_to_cpu(ver.hw_revision), - le16_to_cpu(ver.fw_revision)); + ver.hw_variant, ver.hw_revision, ver.fw_revision); break; default: bt_dev_err(hdev, "Unsupported Intel hardware variant (%u)", @@ -722,14 +713,11 @@ static int intel_setup(struct hci_uart *hu) case 0x0b: /* SfP */ case 0x0c: /* WsP */ snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.ddc", - le16_to_cpu(ver.hw_variant), - le16_to_cpu(params.dev_revid)); + ver.hw_variant, le16_to_cpu(params.dev_revid)); break; case 0x12: /* ThP */ snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u-%u.ddc", - le16_to_cpu(ver.hw_variant), - le16_to_cpu(ver.hw_revision), - le16_to_cpu(ver.fw_revision)); + ver.hw_variant, ver.hw_revision, ver.fw_revision); break; default: bt_dev_err(hdev, "Unsupported Intel hardware variant (%u)", @@ -839,13 +827,11 @@ done: * until further LPM TX notification. */ mutex_lock(&intel_device_list_lock); - list_for_each(p, &intel_device_list) { - struct intel_device *dev = list_entry(p, struct intel_device, - list); + list_for_each_entry(idev, &intel_device_list, list) { if (!hu->tty->dev) break; - if (hu->tty->dev->parent == dev->pdev->dev.parent) { - if (device_may_wakeup(&dev->pdev->dev)) { + if (hu->tty->dev->parent == idev->pdev->dev.parent) { + if (device_may_wakeup(&idev->pdev->dev)) { set_bit(STATE_LPM_ENABLED, &intel->flags); set_bit(STATE_TX_ACTIVE, &intel->flags); } @@ -999,7 +985,7 @@ static int intel_recv(struct hci_uart *hu, const void *data, int count) static int intel_enqueue(struct hci_uart *hu, struct sk_buff *skb) { struct intel_data *intel = hu->priv; - struct list_head *p; + struct intel_device *idev; BT_DBG("hu %p skb %p", hu, skb); @@ -1010,10 +996,7 @@ static int intel_enqueue(struct hci_uart *hu, struct sk_buff *skb) * completed before enqueuing any packet. */ mutex_lock(&intel_device_list_lock); - list_for_each(p, &intel_device_list) { - struct intel_device *idev = list_entry(p, struct intel_device, - list); - + list_for_each_entry(idev, &intel_device_list, list) { if (hu->tty->dev->parent == idev->pdev->dev.parent) { pm_runtime_get_sync(&idev->pdev->dev); pm_runtime_mark_last_busy(&idev->pdev->dev); @@ -1076,7 +1059,8 @@ static const struct hci_uart_proto intel_proto = { #ifdef CONFIG_ACPI static const struct acpi_device_id intel_acpi_match[] = { { "INT33E1", 0 }, - { }, + { "INT33E3", 0 }, + { } }; MODULE_DEVICE_TABLE(acpi, intel_acpi_match); #endif @@ -1138,9 +1122,9 @@ static const struct acpi_gpio_params reset_gpios = { 0, 0, false }; static const struct acpi_gpio_params host_wake_gpios = { 1, 0, false }; static const struct acpi_gpio_mapping acpi_hci_intel_gpios[] = { - { "reset-gpios", &reset_gpios, 1 }, - { "host-wake-gpios", &host_wake_gpios, 1 }, - { }, + { "reset-gpios", &reset_gpios, 1, ACPI_GPIO_QUIRK_ONLY_GPIOIO }, + { "host-wake-gpios", &host_wake_gpios, 1, ACPI_GPIO_QUIRK_ONLY_GPIOIO }, + { } }; static int intel_probe(struct platform_device *pdev) diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index 85a30fb9177b..f83d67eafc9f 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -538,6 +538,7 @@ static void hci_uart_tty_close(struct tty_struct *tty) clear_bit(HCI_UART_PROTO_READY, &hu->flags); percpu_up_write(&hu->proto_lock); + cancel_work_sync(&hu->init_ready); cancel_work_sync(&hu->write_work); if (hdev) { diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index 20e1dedbc58c..244b8feba523 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -693,8 +693,6 @@ static int qca_close(struct hci_uart *hu) destroy_workqueue(qca->workqueue); qca->hu = NULL; - qca_power_shutdown(hu); - kfree_skb(qca->rx_skb); hu->priv = NULL; @@ -2007,8 +2005,7 @@ static int qca_serdev_probe(struct serdev_device *serdev) err = hci_uart_register_device(&qcadev->serdev_hu, &qca_proto); if (err) { BT_ERR("Rome serdev registration failed"); - if (qcadev->susclk) - clk_disable_unprepare(qcadev->susclk); + clk_disable_unprepare(qcadev->susclk); return err; } } @@ -2032,8 +2029,9 @@ static int qca_serdev_probe(struct serdev_device *serdev) static void qca_serdev_remove(struct serdev_device *serdev) { struct qca_serdev *qcadev = serdev_device_get_drvdata(serdev); + struct qca_power *power = qcadev->bt_power; - if (qca_is_wcn399x(qcadev->btsoc_type)) + if (qca_is_wcn399x(qcadev->btsoc_type) && power->vregs_on) qca_power_shutdown(&qcadev->serdev_hu); else if (qcadev->susclk) clk_disable_unprepare(qcadev->susclk); diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c index 7b233312e723..ef96ad06fa54 100644 --- a/drivers/bluetooth/hci_serdev.c +++ b/drivers/bluetooth/hci_serdev.c @@ -113,8 +113,22 @@ static int hci_uart_flush(struct hci_dev *hdev) /* Initialize device */ static int hci_uart_open(struct hci_dev *hdev) { + struct hci_uart *hu = hci_get_drvdata(hdev); + int err; + BT_DBG("%s %p", hdev->name, hdev); + /* When Quirk HCI_QUIRK_NON_PERSISTENT_SETUP is set by + * driver, BT SoC is completely turned OFF during + * BT OFF. Upon next BT ON UART port should be opened. + */ + if (!test_bit(HCI_UART_PROTO_READY, &hu->flags)) { + err = serdev_device_open(hu->serdev); + if (err) + return err; + set_bit(HCI_UART_PROTO_READY, &hu->flags); + } + /* Undo clearing this from hci_uart_close() */ hdev->flush = hci_uart_flush; @@ -124,11 +138,25 @@ static int hci_uart_open(struct hci_dev *hdev) /* Close device */ static int hci_uart_close(struct hci_dev *hdev) { + struct hci_uart *hu = hci_get_drvdata(hdev); + BT_DBG("hdev %p", hdev); + if (!test_bit(HCI_UART_PROTO_READY, &hu->flags)) + return 0; + hci_uart_flush(hdev); hdev->flush = NULL; + /* When QUIRK HCI_QUIRK_NON_PERSISTENT_SETUP is set by driver, + * BT SOC is completely powered OFF during BT OFF, holding port + * open may drain the battery. + */ + if (test_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks)) { + clear_bit(HCI_UART_PROTO_READY, &hu->flags); + serdev_device_close(hu->serdev); + } + return 0; } @@ -354,7 +382,7 @@ void hci_uart_unregister_device(struct hci_uart *hu) { struct hci_dev *hdev = hu->hdev; - clear_bit(HCI_UART_PROTO_READY, &hu->flags); + cancel_work_sync(&hu->init_ready); if (test_bit(HCI_UART_REGISTERED, &hu->flags)) hci_unregister_dev(hdev); hci_free_dev(hdev); @@ -362,6 +390,10 @@ void hci_uart_unregister_device(struct hci_uart *hu) cancel_work_sync(&hu->write_work); hu->proto->close(hu); - serdev_device_close(hu->serdev); + + if (test_bit(HCI_UART_PROTO_READY, &hu->flags)) { + clear_bit(HCI_UART_PROTO_READY, &hu->flags); + serdev_device_close(hu->serdev); + } } EXPORT_SYMBOL_GPL(hci_uart_unregister_device); diff --git a/drivers/net/caif/Kconfig b/drivers/net/caif/Kconfig index c245edddb48b..a77124bc1f4b 100644 --- a/drivers/net/caif/Kconfig +++ b/drivers/net/caif/Kconfig @@ -20,25 +20,6 @@ config CAIF_TTY identified as N_CAIF. When this ldisc is opened from user space it will redirect the TTY's traffic into the CAIF stack. -config CAIF_SPI_SLAVE - tristate "CAIF SPI transport driver for slave interface" - depends on CAIF && HAS_DMA - default n - help - The CAIF Link layer SPI Protocol driver for Slave SPI interface. - This driver implements a platform driver to accommodate for a - platform specific SPI device. A sample CAIF SPI Platform device is - provided in <file:Documentation/networking/caif/spi_porting.rst>. - -config CAIF_SPI_SYNC - bool "Next command and length in start of frame" - depends on CAIF_SPI_SLAVE - default n - help - Putting the next command and length in the start of the frame can - help to synchronize to the next transfer in case of over or under-runs. - This option also needs to be enabled on the modem. - config CAIF_HSI tristate "CAIF HSI transport driver" depends on CAIF diff --git a/drivers/net/caif/Makefile b/drivers/net/caif/Makefile index 54ae1165d60a..b1918c8c126c 100644 --- a/drivers/net/caif/Makefile +++ b/drivers/net/caif/Makefile @@ -4,10 +4,6 @@ ccflags-$(CONFIG_CAIF_DEBUG) := -DDEBUG # Serial interface obj-$(CONFIG_CAIF_TTY) += caif_serial.o -# SPI slave physical interfaces module -cfspi_slave-objs := caif_spi.o caif_spi_slave.o -obj-$(CONFIG_CAIF_SPI_SLAVE) += cfspi_slave.o - # HSI interface obj-$(CONFIG_CAIF_HSI) += caif_hsi.o diff --git a/drivers/net/caif/caif_hsi.c b/drivers/net/caif/caif_hsi.c index 4a33ec4fc089..3d63b15bbaa1 100644 --- a/drivers/net/caif/caif_hsi.c +++ b/drivers/net/caif/caif_hsi.c @@ -458,15 +458,7 @@ static int cfhsi_rx_desc(struct cfhsi_desc *desc, struct cfhsi *cfhsi) skb_reset_mac_header(skb); skb->dev = cfhsi->ndev; - /* - * We are in a callback handler and - * unfortunately we don't know what context we're - * running in. - */ - if (in_interrupt()) - netif_rx(skb); - else - netif_rx_ni(skb); + netif_rx_any_context(skb); /* Update network statistics. */ cfhsi->ndev->stats.rx_packets++; @@ -587,14 +579,7 @@ static int cfhsi_rx_pld(struct cfhsi_desc *desc, struct cfhsi *cfhsi) skb_reset_mac_header(skb); skb->dev = cfhsi->ndev; - /* - * We're called in callback from HSI - * and don't know the context we're running in. - */ - if (in_interrupt()) - netif_rx(skb); - else - netif_rx_ni(skb); + netif_rx_any_context(skb); /* Update network statistics. */ cfhsi->ndev->stats.rx_packets++; diff --git a/drivers/net/caif/caif_spi.c b/drivers/net/caif/caif_spi.c deleted file mode 100644 index 7d5899626130..000000000000 --- a/drivers/net/caif/caif_spi.c +++ /dev/null @@ -1,874 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Daniel Martensson - */ - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/device.h> -#include <linux/platform_device.h> -#include <linux/string.h> -#include <linux/workqueue.h> -#include <linux/completion.h> -#include <linux/list.h> -#include <linux/interrupt.h> -#include <linux/dma-mapping.h> -#include <linux/delay.h> -#include <linux/sched.h> -#include <linux/debugfs.h> -#include <linux/if_arp.h> -#include <net/caif/caif_layer.h> -#include <net/caif/caif_spi.h> - -#ifndef CONFIG_CAIF_SPI_SYNC -#define FLAVOR "Flavour: Vanilla.\n" -#else -#define FLAVOR "Flavour: Master CMD&LEN at start.\n" -#endif /* CONFIG_CAIF_SPI_SYNC */ - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Daniel Martensson"); -MODULE_DESCRIPTION("CAIF SPI driver"); - -/* Returns the number of padding bytes for alignment. */ -#define PAD_POW2(x, pow) ((((x)&((pow)-1))==0) ? 0 : (((pow)-((x)&((pow)-1))))) - -static bool spi_loop; -module_param(spi_loop, bool, 0444); -MODULE_PARM_DESC(spi_loop, "SPI running in loopback mode."); - -/* SPI frame alignment. */ -module_param(spi_frm_align, int, 0444); -MODULE_PARM_DESC(spi_frm_align, "SPI frame alignment."); - -/* - * SPI padding options. - * Warning: must be a base of 2 (& operation used) and can not be zero ! - */ -module_param(spi_up_head_align, int, 0444); -MODULE_PARM_DESC(spi_up_head_align, "SPI uplink head alignment."); - -module_param(spi_up_tail_align, int, 0444); -MODULE_PARM_DESC(spi_up_tail_align, "SPI uplink tail alignment."); - -module_param(spi_down_head_align, int, 0444); -MODULE_PARM_DESC(spi_down_head_align, "SPI downlink head alignment."); - -module_param(spi_down_tail_align, int, 0444); -MODULE_PARM_DESC(spi_down_tail_align, "SPI downlink tail alignment."); - -#ifdef CONFIG_ARM -#define BYTE_HEX_FMT "%02X" -#else -#define BYTE_HEX_FMT "%02hhX" -#endif - -#define SPI_MAX_PAYLOAD_SIZE 4096 -/* - * Threshold values for the SPI packet queue. Flowcontrol will be asserted - * when the number of packets exceeds HIGH_WATER_MARK. It will not be - * deasserted before the number of packets drops below LOW_WATER_MARK. - */ -#define LOW_WATER_MARK 100 -#define HIGH_WATER_MARK (LOW_WATER_MARK*5) - -#ifndef CONFIG_HAS_DMA - -/* - * We sometimes use UML for debugging, but it cannot handle - * dma_alloc_coherent so we have to wrap it. - */ -static inline void *dma_alloc(struct cfspi *cfspi, dma_addr_t *daddr) -{ - return kmalloc(SPI_DMA_BUF_LEN, GFP_KERNEL); -} - -static inline void dma_free(struct cfspi *cfspi, void *cpu_addr, - dma_addr_t handle) -{ - kfree(cpu_addr); -} - -#else - -static inline void *dma_alloc(struct cfspi *cfspi, dma_addr_t *daddr) -{ - return dma_alloc_coherent(&cfspi->pdev->dev, SPI_DMA_BUF_LEN, daddr, - GFP_KERNEL); -} - -static inline void dma_free(struct cfspi *cfspi, void *cpu_addr, - dma_addr_t handle) -{ - dma_free_coherent(&cfspi->pdev->dev, SPI_DMA_BUF_LEN, cpu_addr, handle); -} -#endif /* CONFIG_HAS_DMA */ - -#ifdef CONFIG_DEBUG_FS - -#define DEBUGFS_BUF_SIZE 4096 - -static struct dentry *dbgfs_root; - -static inline void driver_debugfs_create(void) -{ - dbgfs_root = debugfs_create_dir(cfspi_spi_driver.driver.name, NULL); -} - -static inline void driver_debugfs_remove(void) -{ - debugfs_remove(dbgfs_root); -} - -static inline void dev_debugfs_rem(struct cfspi *cfspi) -{ - debugfs_remove(cfspi->dbgfs_frame); - debugfs_remove(cfspi->dbgfs_state); - debugfs_remove(cfspi->dbgfs_dir); -} - -static ssize_t dbgfs_state(struct file *file, char __user *user_buf, - size_t count, loff_t *ppos) -{ - char *buf; - int len = 0; - ssize_t size; - struct cfspi *cfspi = file->private_data; - - buf = kzalloc(DEBUGFS_BUF_SIZE, GFP_KERNEL); - if (!buf) - return 0; - - /* Print out debug information. */ - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "CAIF SPI debug information:\n"); - - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), FLAVOR); - - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "STATE: %d\n", cfspi->dbg_state); - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Previous CMD: 0x%x\n", cfspi->pcmd); - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Current CMD: 0x%x\n", cfspi->cmd); - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Previous TX len: %d\n", cfspi->tx_ppck_len); - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Previous RX len: %d\n", cfspi->rx_ppck_len); - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Current TX len: %d\n", cfspi->tx_cpck_len); - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Current RX len: %d\n", cfspi->rx_cpck_len); - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Next TX len: %d\n", cfspi->tx_npck_len); - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Next RX len: %d\n", cfspi->rx_npck_len); - - if (len > DEBUGFS_BUF_SIZE) - len = DEBUGFS_BUF_SIZE; - - size = simple_read_from_buffer(user_buf, count, ppos, buf, len); - kfree(buf); - - return size; -} - -static ssize_t print_frame(char *buf, size_t size, char *frm, - size_t count, size_t cut) -{ - int len = 0; - int i; - for (i = 0; i < count; i++) { - len += scnprintf((buf + len), (size - len), - "[0x" BYTE_HEX_FMT "]", - frm[i]); - if ((i == cut) && (count > (cut * 2))) { - /* Fast forward. */ - i = count - cut; - len += scnprintf((buf + len), (size - len), - "--- %zu bytes skipped ---\n", - count - (cut * 2)); - } - - if ((!(i % 10)) && i) { - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "\n"); - } - } - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), "\n"); - return len; -} - -static ssize_t dbgfs_frame(struct file *file, char __user *user_buf, - size_t count, loff_t *ppos) -{ - char *buf; - int len = 0; - ssize_t size; - struct cfspi *cfspi; - - cfspi = file->private_data; - buf = kzalloc(DEBUGFS_BUF_SIZE, GFP_KERNEL); - if (!buf) - return 0; - - /* Print out debug information. */ - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Current frame:\n"); - - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Tx data (Len: %d):\n", cfspi->tx_cpck_len); - - len += print_frame((buf + len), (DEBUGFS_BUF_SIZE - len), - cfspi->xfer.va_tx[0], - (cfspi->tx_cpck_len + SPI_CMD_SZ), 100); - - len += scnprintf((buf + len), (DEBUGFS_BUF_SIZE - len), - "Rx data (Len: %d):\n", cfspi->rx_cpck_len); - - len += print_frame((buf + len), (DEBUGFS_BUF_SIZE - len), - cfspi->xfer.va_rx, - (cfspi->rx_cpck_len + SPI_CMD_SZ), 100); - - size = simple_read_from_buffer(user_buf, count, ppos, buf, len); - kfree(buf); - - return size; -} - -static const struct file_operations dbgfs_state_fops = { - .open = simple_open, - .read = dbgfs_state, - .owner = THIS_MODULE -}; - -static const struct file_operations dbgfs_frame_fops = { - .open = simple_open, - .read = dbgfs_frame, - .owner = THIS_MODULE -}; - -static inline void dev_debugfs_add(struct cfspi *cfspi) -{ - cfspi->dbgfs_dir = debugfs_create_dir(cfspi->pdev->name, dbgfs_root); - cfspi->dbgfs_state = debugfs_create_file("state", 0444, - cfspi->dbgfs_dir, cfspi, - &dbgfs_state_fops); - cfspi->dbgfs_frame = debugfs_create_file("frame", 0444, - cfspi->dbgfs_dir, cfspi, - &dbgfs_frame_fops); -} - -inline void cfspi_dbg_state(struct cfspi *cfspi, int state) -{ - cfspi->dbg_state = state; -}; -#else - -static inline void driver_debugfs_create(void) -{ -} - -static inline void driver_debugfs_remove(void) -{ -} - -static inline void dev_debugfs_add(struct cfspi *cfspi) -{ -} - -static inline void dev_debugfs_rem(struct cfspi *cfspi) -{ -} - -inline void cfspi_dbg_state(struct cfspi *cfspi, int state) -{ -} -#endif /* CONFIG_DEBUG_FS */ - -static LIST_HEAD(cfspi_list); -static spinlock_t cfspi_list_lock; - -/* SPI uplink head alignment. */ -static ssize_t up_head_align_show(struct device_driver *driver, char *buf) -{ - return sprintf(buf, "%d\n", spi_up_head_align); -} - -static DRIVER_ATTR_RO(up_head_align); - -/* SPI uplink tail alignment. */ -static ssize_t up_tail_align_show(struct device_driver *driver, char *buf) -{ - return sprintf(buf, "%d\n", spi_up_tail_align); -} - -static DRIVER_ATTR_RO(up_tail_align); - -/* SPI downlink head alignment. */ -static ssize_t down_head_align_show(struct device_driver *driver, char *buf) -{ - return sprintf(buf, "%d\n", spi_down_head_align); -} - -static DRIVER_ATTR_RO(down_head_align); - -/* SPI downlink tail alignment. */ -static ssize_t down_tail_align_show(struct device_driver *driver, char *buf) -{ - return sprintf(buf, "%d\n", spi_down_tail_align); -} - -static DRIVER_ATTR_RO(down_tail_align); - -/* SPI frame alignment. */ -static ssize_t frame_align_show(struct device_driver *driver, char *buf) -{ - return sprintf(buf, "%d\n", spi_frm_align); -} - -static DRIVER_ATTR_RO(frame_align); - -int cfspi_xmitfrm(struct cfspi *cfspi, u8 *buf, size_t len) -{ - u8 *dst = buf; - caif_assert(buf); - - if (cfspi->slave && !cfspi->slave_talked) - cfspi->slave_talked = true; - - do { - struct sk_buff *skb; - struct caif_payload_info *info; - int spad = 0; - int epad; - - skb = skb_dequeue(&cfspi->chead); - if (!skb) - break; - - /* - * Calculate length of frame including SPI padding. - * The payload position is found in the control buffer. - */ - info = (struct caif_payload_info *)&skb->cb; - - /* - * Compute head offset i.e. number of bytes to add to - * get the start of the payload aligned. - */ - if (spi_up_head_align > 1) { - spad = 1 + PAD_POW2((info->hdr_len + 1), spi_up_head_align); - *dst = (u8)(spad - 1); - dst += spad; - } - - /* Copy in CAIF frame. */ - skb_copy_bits(skb, 0, dst, skb->len); - dst += skb->len; - cfspi->ndev->stats.tx_packets++; - cfspi->ndev->stats.tx_bytes += skb->len; - - /* - * Compute tail offset i.e. number of bytes to add to - * get the complete CAIF frame aligned. - */ - epad = PAD_POW2((skb->len + spad), spi_up_tail_align); - dst += epad; - - dev_kfree_skb(skb); - - } while ((dst - buf) < len); - - return dst - buf; -} - -int cfspi_xmitlen(struct cfspi *cfspi) -{ - struct sk_buff *skb = NULL; - int frm_len = 0; - int pkts = 0; - - /* - * Decommit previously committed frames. - * skb_queue_splice_tail(&cfspi->chead,&cfspi->qhead) - */ - while (skb_peek(&cfspi->chead)) { - skb = skb_dequeue_tail(&cfspi->chead); - skb_queue_head(&cfspi->qhead, skb); - } - - do { - struct caif_payload_info *info = NULL; - int spad = 0; - int epad = 0; - - skb = skb_dequeue(&cfspi->qhead); - if (!skb) - break; - - /* - * Calculate length of frame including SPI padding. - * The payload position is found in the control buffer. - */ - info = (struct caif_payload_info *)&skb->cb; - - /* - * Compute head offset i.e. number of bytes to add to - * get the start of the payload aligned. - */ - if (spi_up_head_align > 1) - spad = 1 + PAD_POW2((info->hdr_len + 1), spi_up_head_align); - - /* - * Compute tail offset i.e. number of bytes to add to - * get the complete CAIF frame aligned. - */ - epad = PAD_POW2((skb->len + spad), spi_up_tail_align); - - if ((skb->len + spad + epad + frm_len) <= CAIF_MAX_SPI_FRAME) { - skb_queue_tail(&cfspi->chead, skb); - pkts++; - frm_len += skb->len + spad + epad; - } else { - /* Put back packet. */ - skb_queue_head(&cfspi->qhead, skb); - break; - } - } while (pkts <= CAIF_MAX_SPI_PKTS); - - /* - * Send flow on if previously sent flow off - * and now go below the low water mark - */ - if (cfspi->flow_off_sent && cfspi->qhead.qlen < cfspi->qd_low_mark && - cfspi->cfdev.flowctrl) { - cfspi->flow_off_sent = 0; - cfspi->cfdev.flowctrl(cfspi->ndev, 1); - } - - return frm_len; -} - -static void cfspi_ss_cb(bool assert, struct cfspi_ifc *ifc) -{ - struct cfspi *cfspi = (struct cfspi *)ifc->priv; - - /* - * The slave device is the master on the link. Interrupts before the - * slave has transmitted are considered spurious. - */ - if (cfspi->slave && !cfspi->slave_talked) { - printk(KERN_WARNING "CFSPI: Spurious SS interrupt.\n"); - return; - } - - if (!in_interrupt()) - spin_lock(&cfspi->lock); - if (assert) { - set_bit(SPI_SS_ON, &cfspi->state); - set_bit(SPI_XFER, &cfspi->state); - } else { - set_bit(SPI_SS_OFF, &cfspi->state); - } - if (!in_interrupt()) - spin_unlock(&cfspi->lock); - - /* Wake up the xfer thread. */ - if (assert) - wake_up_interruptible(&cfspi->wait); -} - -static void cfspi_xfer_done_cb(struct cfspi_ifc *ifc) -{ - struct cfspi *cfspi = (struct cfspi *)ifc->priv; - - /* Transfer done, complete work queue */ - complete(&cfspi->comp); -} - -static netdev_tx_t cfspi_xmit(struct sk_buff *skb, struct net_device *dev) -{ - struct cfspi *cfspi = NULL; - unsigned long flags; - if (!dev) - return -EINVAL; - - cfspi = netdev_priv(dev); - - skb_queue_tail(&cfspi->qhead, skb); - - spin_lock_irqsave(&cfspi->lock, flags); - if (!test_and_set_bit(SPI_XFER, &cfspi->state)) { - /* Wake up xfer thread. */ - wake_up_interruptible(&cfspi->wait); - } - spin_unlock_irqrestore(&cfspi->lock, flags); - - /* Send flow off if number of bytes is above high water mark */ - if (!cfspi->flow_off_sent && - cfspi->qhead.qlen > cfspi->qd_high_mark && - cfspi->cfdev.flowctrl) { - cfspi->flow_off_sent = 1; - cfspi->cfdev.flowctrl(cfspi->ndev, 0); - } - - return NETDEV_TX_OK; -} - -int cfspi_rxfrm(struct cfspi *cfspi, u8 *buf, size_t len) -{ - u8 *src = buf; - - caif_assert(buf != NULL); - - do { - int res; - struct sk_buff *skb = NULL; - int spad = 0; - int epad = 0; - int pkt_len = 0; - - /* - * Compute head offset i.e. number of bytes added to - * get the start of the payload aligned. - */ - if (spi_down_head_align > 1) { - spad = 1 + *src; - src += spad; - } - - /* Read length of CAIF frame (little endian). */ - pkt_len = *src; - pkt_len |= ((*(src+1)) << 8) & 0xFF00; - pkt_len += 2; /* Add FCS fields. */ - - /* Get a suitable caif packet and copy in data. */ - - skb = netdev_alloc_skb(cfspi->ndev, pkt_len + 1); - caif_assert(skb != NULL); - - skb_put_data(skb, src, pkt_len); - src += pkt_len; - - skb->protocol = htons(ETH_P_CAIF); - skb_reset_mac_header(skb); - - /* - * Push received packet up the stack. - */ - if (!spi_loop) - res = netif_rx_ni(skb); - else - res = cfspi_xmit(skb, cfspi->ndev); - - if (!res) { - cfspi->ndev->stats.rx_packets++; - cfspi->ndev->stats.rx_bytes += pkt_len; - } else - cfspi->ndev->stats.rx_dropped++; - - /* - * Compute tail offset i.e. number of bytes added to - * get the complete CAIF frame aligned. - */ - epad = PAD_POW2((pkt_len + spad), spi_down_tail_align); - src += epad; - } while ((src - buf) < len); - - return src - buf; -} - -static int cfspi_open(struct net_device *dev) -{ - netif_wake_queue(dev); - return 0; -} - -static int cfspi_close(struct net_device *dev) -{ - netif_stop_queue(dev); - return 0; -} - -static int cfspi_init(struct net_device *dev) -{ - int res = 0; - struct cfspi *cfspi = netdev_priv(dev); - - /* Set flow info. */ - cfspi->flow_off_sent = 0; - cfspi->qd_low_mark = LOW_WATER_MARK; - cfspi->qd_high_mark = HIGH_WATER_MARK; - - /* Set slave info. */ - if (!strncmp(cfspi_spi_driver.driver.name, "cfspi_sspi", 10)) { - cfspi->slave = true; - cfspi->slave_talked = false; - } else { - cfspi->slave = false; - cfspi->slave_talked = false; - } - - /* Allocate DMA buffers. */ - cfspi->xfer.va_tx[0] = dma_alloc(cfspi, &cfspi->xfer.pa_tx[0]); - if (!cfspi->xfer.va_tx[0]) { - res = -ENODEV; - goto err_dma_alloc_tx_0; - } - - cfspi->xfer.va_rx = dma_alloc(cfspi, &cfspi->xfer.pa_rx); - - if (!cfspi->xfer.va_rx) { - res = -ENODEV; - goto err_dma_alloc_rx; - } - - /* Initialize the work queue. */ - INIT_WORK(&cfspi->work, cfspi_xfer); - - /* Initialize spin locks. */ - spin_lock_init(&cfspi->lock); - - /* Initialize flow control state. */ - cfspi->flow_stop = false; - - /* Initialize wait queue. */ - init_waitqueue_head(&cfspi->wait); - - /* Create work thread. */ - cfspi->wq = create_singlethread_workqueue(dev->name); - if (!cfspi->wq) { - printk(KERN_WARNING "CFSPI: failed to create work queue.\n"); - res = -ENODEV; - goto err_create_wq; - } - - /* Initialize work queue. */ - init_completion(&cfspi->comp); - - /* Create debugfs entries. */ - dev_debugfs_add(cfspi); - - /* Set up the ifc. */ - cfspi->ifc.ss_cb = cfspi_ss_cb; - cfspi->ifc.xfer_done_cb = cfspi_xfer_done_cb; - cfspi->ifc.priv = cfspi; - - /* Add CAIF SPI device to list. */ - spin_lock(&cfspi_list_lock); - list_add_tail(&cfspi->list, &cfspi_list); - spin_unlock(&cfspi_list_lock); - - /* Schedule the work queue. */ - queue_work(cfspi->wq, &cfspi->work); - - return 0; - - err_create_wq: - dma_free(cfspi, cfspi->xfer.va_rx, cfspi->xfer.pa_rx); - err_dma_alloc_rx: - dma_free(cfspi, cfspi->xfer.va_tx[0], cfspi->xfer.pa_tx[0]); - err_dma_alloc_tx_0: - return res; -} - -static void cfspi_uninit(struct net_device *dev) -{ - struct cfspi *cfspi = netdev_priv(dev); - - /* Remove from list. */ - spin_lock(&cfspi_list_lock); - list_del(&cfspi->list); - spin_unlock(&cfspi_list_lock); - - cfspi->ndev = NULL; - /* Free DMA buffers. */ - dma_free(cfspi, cfspi->xfer.va_rx, cfspi->xfer.pa_rx); - dma_free(cfspi, cfspi->xfer.va_tx[0], cfspi->xfer.pa_tx[0]); - set_bit(SPI_TERMINATE, &cfspi->state); - wake_up_interruptible(&cfspi->wait); - destroy_workqueue(cfspi->wq); - /* Destroy debugfs directory and files. */ - dev_debugfs_rem(cfspi); - return; -} - -static const struct net_device_ops cfspi_ops = { - .ndo_open = cfspi_open, - .ndo_stop = cfspi_close, - .ndo_init = cfspi_init, - .ndo_uninit = cfspi_uninit, - .ndo_start_xmit = cfspi_xmit -}; - -static void cfspi_setup(struct net_device *dev) -{ - struct cfspi *cfspi = netdev_priv(dev); - dev->features = 0; - dev->netdev_ops = &cfspi_ops; - dev->type = ARPHRD_CAIF; - dev->flags = IFF_NOARP | IFF_POINTOPOINT; - dev->priv_flags |= IFF_NO_QUEUE; - dev->mtu = SPI_MAX_PAYLOAD_SIZE; - dev->needs_free_netdev = true; - skb_queue_head_init(&cfspi->qhead); - skb_queue_head_init(&cfspi->chead); - cfspi->cfdev.link_select = CAIF_LINK_HIGH_BANDW; - cfspi->cfdev.use_frag = false; - cfspi->cfdev.use_stx = false; - cfspi->cfdev.use_fcs = false; - cfspi->ndev = dev; -} - -int cfspi_spi_probe(struct platform_device *pdev) -{ - struct cfspi *cfspi = NULL; - struct net_device *ndev; - struct cfspi_dev *dev; - int res; - dev = (struct cfspi_dev *)pdev->dev.platform_data; - - if (!dev) - return -ENODEV; - - ndev = alloc_netdev(sizeof(struct cfspi), "cfspi%d", - NET_NAME_UNKNOWN, cfspi_setup); - if (!ndev) - return -ENOMEM; - - cfspi = netdev_priv(ndev); - netif_stop_queue(ndev); - cfspi->ndev = ndev; - cfspi->pdev = pdev; - - /* Assign the SPI device. */ - cfspi->dev = dev; - /* Assign the device ifc to this SPI interface. */ - dev->ifc = &cfspi->ifc; - - /* Register network device. */ - res = register_netdev(ndev); - if (res) { - printk(KERN_ERR "CFSPI: Reg. error: %d.\n", res); - goto err_net_reg; - } - return res; - - err_net_reg: - free_netdev(ndev); - - return res; -} - -int cfspi_spi_remove(struct platform_device *pdev) -{ - /* Everything is done in cfspi_uninit(). */ - return 0; -} - -static void __exit cfspi_exit_module(void) -{ - struct list_head *list_node; - struct list_head *n; - struct cfspi *cfspi = NULL; - - list_for_each_safe(list_node, n, &cfspi_list) { - cfspi = list_entry(list_node, struct cfspi, list); - unregister_netdev(cfspi->ndev); - } - - /* Destroy sysfs files. */ - driver_remove_file(&cfspi_spi_driver.driver, - &driver_attr_up_head_align); - driver_remove_file(&cfspi_spi_driver.driver, - &driver_attr_up_tail_align); - driver_remove_file(&cfspi_spi_driver.driver, - &driver_attr_down_head_align); - driver_remove_file(&cfspi_spi_driver.driver, - &driver_attr_down_tail_align); - driver_remove_file(&cfspi_spi_driver.driver, &driver_attr_frame_align); - /* Unregister platform driver. */ - platform_driver_unregister(&cfspi_spi_driver); - /* Destroy debugfs root directory. */ - driver_debugfs_remove(); -} - -static int __init cfspi_init_module(void) -{ - int result; - - /* Initialize spin lock. */ - spin_lock_init(&cfspi_list_lock); - - /* Register platform driver. */ - result = platform_driver_register(&cfspi_spi_driver); - if (result) { - printk(KERN_ERR "Could not register platform SPI driver.\n"); - goto err_dev_register; - } - - /* Create sysfs files. */ - result = - driver_create_file(&cfspi_spi_driver.driver, - &driver_attr_up_head_align); - if (result) { - printk(KERN_ERR "Sysfs creation failed 1.\n"); - goto err_create_up_head_align; - } - - result = - driver_create_file(&cfspi_spi_driver.driver, - &driver_attr_up_tail_align); - if (result) { - printk(KERN_ERR "Sysfs creation failed 2.\n"); - goto err_create_up_tail_align; - } - - result = - driver_create_file(&cfspi_spi_driver.driver, - &driver_attr_down_head_align); - if (result) { - printk(KERN_ERR "Sysfs creation failed 3.\n"); - goto err_create_down_head_align; - } - - result = - driver_create_file(&cfspi_spi_driver.driver, - &driver_attr_down_tail_align); - if (result) { - printk(KERN_ERR "Sysfs creation failed 4.\n"); - goto err_create_down_tail_align; - } - - result = - driver_create_file(&cfspi_spi_driver.driver, - &driver_attr_frame_align); - if (result) { - printk(KERN_ERR "Sysfs creation failed 5.\n"); - goto err_create_frame_align; - } - driver_debugfs_create(); - return result; - - err_create_frame_align: - driver_remove_file(&cfspi_spi_driver.driver, - &driver_attr_down_tail_align); - err_create_down_tail_align: - driver_remove_file(&cfspi_spi_driver.driver, - &driver_attr_down_head_align); - err_create_down_head_align: - driver_remove_file(&cfspi_spi_driver.driver, - &driver_attr_up_tail_align); - err_create_up_tail_align: - driver_remove_file(&cfspi_spi_driver.driver, - &driver_attr_up_head_align); - err_create_up_head_align: - platform_driver_unregister(&cfspi_spi_driver); - err_dev_register: - return result; -} - -module_init(cfspi_init_module); -module_exit(cfspi_exit_module); diff --git a/drivers/net/caif/caif_spi_slave.c b/drivers/net/caif/caif_spi_slave.c deleted file mode 100644 index bb776d33dd8f..000000000000 --- a/drivers/net/caif/caif_spi_slave.c +++ /dev/null @@ -1,254 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Daniel Martensson - */ -#include <linux/module.h> -#include <linux/device.h> -#include <linux/platform_device.h> -#include <linux/string.h> -#include <linux/semaphore.h> -#include <linux/workqueue.h> -#include <linux/completion.h> -#include <linux/list.h> -#include <linux/interrupt.h> -#include <linux/dma-mapping.h> -#include <linux/delay.h> -#include <linux/sched.h> -#include <linux/debugfs.h> -#include <net/caif/caif_spi.h> - -#ifndef CONFIG_CAIF_SPI_SYNC -#define SPI_DATA_POS 0 -static inline int forward_to_spi_cmd(struct cfspi *cfspi) -{ - return cfspi->rx_cpck_len; -} -#else -#define SPI_DATA_POS SPI_CMD_SZ -static inline int forward_to_spi_cmd(struct cfspi *cfspi) -{ - return 0; -} -#endif - -int spi_frm_align = 2; - -/* - * SPI padding options. - * Warning: must be a base of 2 (& operation used) and can not be zero ! - */ -int spi_up_head_align = 1 << 1; -int spi_up_tail_align = 1 << 0; -int spi_down_head_align = 1 << 2; -int spi_down_tail_align = 1 << 1; - -#ifdef CONFIG_DEBUG_FS -static inline void debugfs_store_prev(struct cfspi *cfspi) -{ - /* Store previous command for debugging reasons.*/ - cfspi->pcmd = cfspi->cmd; - /* Store previous transfer. */ - cfspi->tx_ppck_len = cfspi->tx_cpck_len; - cfspi->rx_ppck_len = cfspi->rx_cpck_len; -} -#else -static inline void debugfs_store_prev(struct cfspi *cfspi) -{ -} -#endif - -void cfspi_xfer(struct work_struct *work) -{ - struct cfspi *cfspi; - u8 *ptr = NULL; - unsigned long flags; - int ret; - cfspi = container_of(work, struct cfspi, work); - - /* Initialize state. */ - cfspi->cmd = SPI_CMD_EOT; - - for (;;) { - - cfspi_dbg_state(cfspi, CFSPI_STATE_WAITING); - - /* Wait for master talk or transmit event. */ - wait_event_interruptible(cfspi->wait, - test_bit(SPI_XFER, &cfspi->state) || - test_bit(SPI_TERMINATE, &cfspi->state)); - - if (test_bit(SPI_TERMINATE, &cfspi->state)) - return; - -#if CFSPI_DBG_PREFILL - /* Prefill buffers for easier debugging. */ - memset(cfspi->xfer.va_tx, 0xFF, SPI_DMA_BUF_LEN); - memset(cfspi->xfer.va_rx, 0xFF, SPI_DMA_BUF_LEN); -#endif /* CFSPI_DBG_PREFILL */ - - cfspi_dbg_state(cfspi, CFSPI_STATE_AWAKE); - - /* Check whether we have a committed frame. */ - if (cfspi->tx_cpck_len) { - int len; - - cfspi_dbg_state(cfspi, CFSPI_STATE_FETCH_PKT); - - /* Copy committed SPI frames after the SPI indication. */ - ptr = (u8 *) cfspi->xfer.va_tx; - ptr += SPI_IND_SZ; - len = cfspi_xmitfrm(cfspi, ptr, cfspi->tx_cpck_len); - WARN_ON(len != cfspi->tx_cpck_len); - } - - cfspi_dbg_state(cfspi, CFSPI_STATE_GET_NEXT); - - /* Get length of next frame to commit. */ - cfspi->tx_npck_len = cfspi_xmitlen(cfspi); - - WARN_ON(cfspi->tx_npck_len > SPI_DMA_BUF_LEN); - - /* - * Add indication and length at the beginning of the frame, - * using little endian. - */ - ptr = (u8 *) cfspi->xfer.va_tx; - *ptr++ = SPI_CMD_IND; - *ptr++ = (SPI_CMD_IND & 0xFF00) >> 8; - *ptr++ = cfspi->tx_npck_len & 0x00FF; - *ptr++ = (cfspi->tx_npck_len & 0xFF00) >> 8; - - /* Calculate length of DMAs. */ - cfspi->xfer.tx_dma_len = cfspi->tx_cpck_len + SPI_IND_SZ; - cfspi->xfer.rx_dma_len = cfspi->rx_cpck_len + SPI_CMD_SZ; - - /* Add SPI TX frame alignment padding, if necessary. */ - if (cfspi->tx_cpck_len && - (cfspi->xfer.tx_dma_len % spi_frm_align)) { - - cfspi->xfer.tx_dma_len += spi_frm_align - - (cfspi->xfer.tx_dma_len % spi_frm_align); - } - - /* Add SPI RX frame alignment padding, if necessary. */ - if (cfspi->rx_cpck_len && - (cfspi->xfer.rx_dma_len % spi_frm_align)) { - - cfspi->xfer.rx_dma_len += spi_frm_align - - (cfspi->xfer.rx_dma_len % spi_frm_align); - } - - cfspi_dbg_state(cfspi, CFSPI_STATE_INIT_XFER); - - /* Start transfer. */ - ret = cfspi->dev->init_xfer(&cfspi->xfer, cfspi->dev); - WARN_ON(ret); - - cfspi_dbg_state(cfspi, CFSPI_STATE_WAIT_ACTIVE); - - /* - * TODO: We might be able to make an assumption if this is the - * first loop. Make sure that minimum toggle time is respected. - */ - udelay(MIN_TRANSITION_TIME_USEC); - - cfspi_dbg_state(cfspi, CFSPI_STATE_SIG_ACTIVE); - - /* Signal that we are ready to receive data. */ - cfspi->dev->sig_xfer(true, cfspi->dev); - - cfspi_dbg_state(cfspi, CFSPI_STATE_WAIT_XFER_DONE); - - /* Wait for transfer completion. */ - wait_for_completion(&cfspi->comp); - - cfspi_dbg_state(cfspi, CFSPI_STATE_XFER_DONE); - - if (cfspi->cmd == SPI_CMD_EOT) { - /* - * Clear the master talk bit. A xfer is always at - * least two bursts. - */ - clear_bit(SPI_SS_ON, &cfspi->state); - } - - cfspi_dbg_state(cfspi, CFSPI_STATE_WAIT_INACTIVE); - - /* Make sure that the minimum toggle time is respected. */ - if (SPI_XFER_TIME_USEC(cfspi->xfer.tx_dma_len, - cfspi->dev->clk_mhz) < - MIN_TRANSITION_TIME_USEC) { - - udelay(MIN_TRANSITION_TIME_USEC - - SPI_XFER_TIME_USEC - (cfspi->xfer.tx_dma_len, cfspi->dev->clk_mhz)); - } - - cfspi_dbg_state(cfspi, CFSPI_STATE_SIG_INACTIVE); - - /* De-assert transfer signal. */ - cfspi->dev->sig_xfer(false, cfspi->dev); - - /* Check whether we received a CAIF packet. */ - if (cfspi->rx_cpck_len) { - int len; - - cfspi_dbg_state(cfspi, CFSPI_STATE_DELIVER_PKT); - - /* Parse SPI frame. */ - ptr = ((u8 *)(cfspi->xfer.va_rx + SPI_DATA_POS)); - - len = cfspi_rxfrm(cfspi, ptr, cfspi->rx_cpck_len); - WARN_ON(len != cfspi->rx_cpck_len); - } - - /* Check the next SPI command and length. */ - ptr = (u8 *) cfspi->xfer.va_rx; - - ptr += forward_to_spi_cmd(cfspi); - - cfspi->cmd = *ptr++; - cfspi->cmd |= ((*ptr++) << 8) & 0xFF00; - cfspi->rx_npck_len = *ptr++; - cfspi->rx_npck_len |= ((*ptr++) << 8) & 0xFF00; - - WARN_ON(cfspi->rx_npck_len > SPI_DMA_BUF_LEN); - WARN_ON(cfspi->cmd > SPI_CMD_EOT); - - debugfs_store_prev(cfspi); - - /* Check whether the master issued an EOT command. */ - if (cfspi->cmd == SPI_CMD_EOT) { - /* Reset state. */ - cfspi->tx_cpck_len = 0; - cfspi->rx_cpck_len = 0; - } else { - /* Update state. */ - cfspi->tx_cpck_len = cfspi->tx_npck_len; - cfspi->rx_cpck_len = cfspi->rx_npck_len; - } - - /* - * Check whether we need to clear the xfer bit. - * Spin lock needed for packet insertion. - * Test and clear of different bits - * are not supported. - */ - spin_lock_irqsave(&cfspi->lock, flags); - if (cfspi->cmd == SPI_CMD_EOT && !cfspi_xmitlen(cfspi) - && !test_bit(SPI_SS_ON, &cfspi->state)) - clear_bit(SPI_XFER, &cfspi->state); - - spin_unlock_irqrestore(&cfspi->lock, flags); - } -} - -struct platform_driver cfspi_spi_driver = { - .probe = cfspi_spi_probe, - .remove = cfspi_spi_remove, - .driver = { - .name = "cfspi_sspi", - .owner = THIS_MODULE, - }, -}; diff --git a/drivers/net/caif/caif_virtio.c b/drivers/net/caif/caif_virtio.c index 80ea2e913c2b..47a6d62b7511 100644 --- a/drivers/net/caif/caif_virtio.c +++ b/drivers/net/caif/caif_virtio.c @@ -652,7 +652,7 @@ static int cfv_probe(struct virtio_device *vdev) const char *cfv_netdev_name = "cfvrt"; struct net_device *netdev; struct cfv_info *cfv; - int err = -EINVAL; + int err; netdev = alloc_netdev(sizeof(struct cfv_info), cfv_netdev_name, NET_NAME_UNKNOWN, cfv_netdev_setup); diff --git a/drivers/net/can/flexcan.c b/drivers/net/can/flexcan.c index e86925134009..fbdd9a8c9374 100644 --- a/drivers/net/can/flexcan.c +++ b/drivers/net/can/flexcan.c @@ -214,6 +214,7 @@ * MX53 FlexCAN2 03.00.00.00 yes no no no no no * MX6s FlexCAN3 10.00.12.00 yes yes no no yes no * MX8QM FlexCAN3 03.00.23.00 yes yes no no yes yes + * MX8MP FlexCAN3 03.00.17.01 yes yes no yes yes yes * VF610 FlexCAN3 ? no yes no yes yes? no * LS1021A FlexCAN2 03.00.04.00 no yes no no yes no * LX2160A FlexCAN3 03.00.23.00 no yes no no yes yes @@ -239,6 +240,8 @@ #define FLEXCAN_QUIRK_SETUP_STOP_MODE BIT(8) /* Support CAN-FD mode */ #define FLEXCAN_QUIRK_SUPPORT_FD BIT(9) +/* support memory detection and correction */ +#define FLEXCAN_QUIRK_SUPPORT_ECC BIT(10) /* Structure of the message buffer */ struct flexcan_mb { @@ -292,7 +295,16 @@ struct flexcan_regs { u32 rximr[64]; /* 0x880 - Not affected by Soft Reset */ u32 _reserved5[24]; /* 0x980 */ u32 gfwr_mx6; /* 0x9e0 - MX6 */ - u32 _reserved6[63]; /* 0x9e4 */ + u32 _reserved6[39]; /* 0x9e4 */ + u32 _rxfir[6]; /* 0xa80 */ + u32 _reserved8[2]; /* 0xa98 */ + u32 _rxmgmask; /* 0xaa0 */ + u32 _rxfgmask; /* 0xaa4 */ + u32 _rx14mask; /* 0xaa8 */ + u32 _rx15mask; /* 0xaac */ + u32 tx_smb[4]; /* 0xab0 */ + u32 rx_smb0[4]; /* 0xac0 */ + u32 rx_smb1[4]; /* 0xad0 */ u32 mecr; /* 0xae0 */ u32 erriar; /* 0xae4 */ u32 erridpr; /* 0xae8 */ @@ -305,9 +317,13 @@ struct flexcan_regs { u32 fdctrl; /* 0xc00 - Not affected by Soft Reset */ u32 fdcbt; /* 0xc04 - Not affected by Soft Reset */ u32 fdcrc; /* 0xc08 */ + u32 _reserved9[199]; /* 0xc0c */ + u32 tx_smb_fd[18]; /* 0xf28 */ + u32 rx_smb0_fd[18]; /* 0xf70 */ + u32 rx_smb1_fd[18]; /* 0xfb8 */ }; -static_assert(sizeof(struct flexcan_regs) == 0x4 + 0xc08); +static_assert(sizeof(struct flexcan_regs) == 0x4 * 18 + 0xfb8); struct flexcan_devtype_data { u32 quirks; /* quirks needed for different IP cores */ @@ -376,6 +392,13 @@ static const struct flexcan_devtype_data fsl_imx8qm_devtype_data = { FLEXCAN_QUIRK_SUPPORT_FD, }; +static struct flexcan_devtype_data fsl_imx8mp_devtype_data = { + .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS | + FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_OFF_TIMESTAMP | + FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_SETUP_STOP_MODE | + FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SUPPORT_ECC, +}; + static const struct flexcan_devtype_data fsl_vf610_devtype_data = { .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS | FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_OFF_TIMESTAMP | @@ -1292,6 +1315,37 @@ static void flexcan_set_bittiming(struct net_device *dev) return flexcan_set_bittiming_ctrl(dev); } +static void flexcan_ram_init(struct net_device *dev) +{ + struct flexcan_priv *priv = netdev_priv(dev); + struct flexcan_regs __iomem *regs = priv->regs; + u32 reg_ctrl2; + + /* 11.8.3.13 Detection and correction of memory errors: + * CTRL2[WRMFRZ] grants write access to all memory positions + * that require initialization, ranging from 0x080 to 0xADF + * and from 0xF28 to 0xFFF when the CAN FD feature is enabled. + * The RXMGMASK, RX14MASK, RX15MASK, and RXFGMASK registers + * need to be initialized as well. MCR[RFEN] must not be set + * during memory initialization. + */ + reg_ctrl2 = priv->read(®s->ctrl2); + reg_ctrl2 |= FLEXCAN_CTRL2_WRMFRZ; + priv->write(reg_ctrl2, ®s->ctrl2); + + memset_io(®s->mb[0][0], 0, + offsetof(struct flexcan_regs, rx_smb1[3]) - + offsetof(struct flexcan_regs, mb[0][0]) + 0x4); + + if (priv->can.ctrlmode & CAN_CTRLMODE_FD) + memset_io(®s->tx_smb_fd[0], 0, + offsetof(struct flexcan_regs, rx_smb1_fd[17]) - + offsetof(struct flexcan_regs, tx_smb_fd[0]) + 0x4); + + reg_ctrl2 &= ~FLEXCAN_CTRL2_WRMFRZ; + priv->write(reg_ctrl2, ®s->ctrl2); +} + /* flexcan_chip_start * * this functions is entered with clocks enabled @@ -1316,6 +1370,9 @@ static int flexcan_chip_start(struct net_device *dev) if (err) goto out_chip_disable; + if (priv->devtype_data->quirks & FLEXCAN_QUIRK_SUPPORT_ECC) + flexcan_ram_init(dev); + flexcan_set_bittiming(dev); /* MCR @@ -1845,6 +1902,7 @@ out_put_node: static const struct of_device_id flexcan_of_match[] = { { .compatible = "fsl,imx8qm-flexcan", .data = &fsl_imx8qm_devtype_data, }, + { .compatible = "fsl,imx8mp-flexcan", .data = &fsl_imx8mp_devtype_data, }, { .compatible = "fsl,imx6q-flexcan", .data = &fsl_imx6q_devtype_data, }, { .compatible = "fsl,imx28-flexcan", .data = &fsl_imx28_devtype_data, }, { .compatible = "fsl,imx53-flexcan", .data = &fsl_imx25_devtype_data, }, @@ -1999,6 +2057,8 @@ static int flexcan_probe(struct platform_device *pdev) return 0; failed_register: + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); free_candev(dev); return err; } diff --git a/drivers/net/can/spi/Kconfig b/drivers/net/can/spi/Kconfig index a82240628c33..f45449210047 100644 --- a/drivers/net/can/spi/Kconfig +++ b/drivers/net/can/spi/Kconfig @@ -13,6 +13,6 @@ config CAN_MCP251X Driver for the Microchip MCP251x and MCP25625 SPI CAN controllers. -source "drivers/net/can/spi/mcp25xxfd/Kconfig" +source "drivers/net/can/spi/mcp251xfd/Kconfig" endmenu diff --git a/drivers/net/can/spi/Makefile b/drivers/net/can/spi/Makefile index 20c18ac96b1c..33e3f60bbc10 100644 --- a/drivers/net/can/spi/Makefile +++ b/drivers/net/can/spi/Makefile @@ -6,4 +6,4 @@ obj-$(CONFIG_CAN_HI311X) += hi311x.o obj-$(CONFIG_CAN_MCP251X) += mcp251x.o -obj-y += mcp25xxfd/ +obj-y += mcp251xfd/ diff --git a/drivers/net/can/spi/mcp25xxfd/Kconfig b/drivers/net/can/spi/mcp251xfd/Kconfig index 9eb596019a58..f5a147a92cb2 100644 --- a/drivers/net/can/spi/mcp25xxfd/Kconfig +++ b/drivers/net/can/spi/mcp251xfd/Kconfig @@ -1,14 +1,14 @@ # SPDX-License-Identifier: GPL-2.0-only -config CAN_MCP25XXFD - tristate "Microchip MCP25xxFD SPI CAN controllers" +config CAN_MCP251XFD + tristate "Microchip MCP251xFD SPI CAN controllers" select REGMAP help - Driver for the Microchip MCP25XXFD SPI FD-CAN controller + Driver for the Microchip MCP251XFD SPI FD-CAN controller family. -config CAN_MCP25XXFD_SANITY - depends on CAN_MCP25XXFD +config CAN_MCP251XFD_SANITY + depends on CAN_MCP251XFD bool "Additional Sanity Checks" help This option enables additional sanity checks in the driver, diff --git a/drivers/net/can/spi/mcp251xfd/Makefile b/drivers/net/can/spi/mcp251xfd/Makefile new file mode 100644 index 000000000000..cb71244cbe89 --- /dev/null +++ b/drivers/net/can/spi/mcp251xfd/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_CAN_MCP251XFD) += mcp251xfd.o + +mcp251xfd-objs := +mcp251xfd-objs += mcp251xfd-core.o +mcp251xfd-objs += mcp251xfd-crc16.o +mcp251xfd-objs += mcp251xfd-regmap.o diff --git a/drivers/net/can/spi/mcp25xxfd/mcp25xxfd-core.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c index bd2ba981ae36..c3f49543ff26 100644 --- a/drivers/net/can/spi/mcp25xxfd/mcp25xxfd-core.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 // -// mcp25xxfd - Microchip MCP25xxFD Family CAN controller driver +// mcp251xfd - Microchip MCP251xFD Family CAN controller driver // // Copyright (c) 2019, 2020 Pengutronix, // Marc Kleine-Budde <kernel@pengutronix.de> @@ -23,31 +23,31 @@ #include <asm/unaligned.h> -#include "mcp25xxfd.h" +#include "mcp251xfd.h" -#define DEVICE_NAME "mcp25xxfd" +#define DEVICE_NAME "mcp251xfd" -static const struct mcp25xxfd_devtype_data mcp25xxfd_devtype_data_mcp2517fd = { - .quirks = MCP25XXFD_QUIRK_MAB_NO_WARN | MCP25XXFD_QUIRK_CRC_REG | - MCP25XXFD_QUIRK_CRC_RX | MCP25XXFD_QUIRK_CRC_TX | - MCP25XXFD_QUIRK_ECC, - .model = MCP25XXFD_MODEL_MCP2517FD, +static const struct mcp251xfd_devtype_data mcp251xfd_devtype_data_mcp2517fd = { + .quirks = MCP251XFD_QUIRK_MAB_NO_WARN | MCP251XFD_QUIRK_CRC_REG | + MCP251XFD_QUIRK_CRC_RX | MCP251XFD_QUIRK_CRC_TX | + MCP251XFD_QUIRK_ECC, + .model = MCP251XFD_MODEL_MCP2517FD, }; -static const struct mcp25xxfd_devtype_data mcp25xxfd_devtype_data_mcp2518fd = { - .quirks = MCP25XXFD_QUIRK_CRC_REG | MCP25XXFD_QUIRK_CRC_RX | - MCP25XXFD_QUIRK_CRC_TX | MCP25XXFD_QUIRK_ECC, - .model = MCP25XXFD_MODEL_MCP2518FD, +static const struct mcp251xfd_devtype_data mcp251xfd_devtype_data_mcp2518fd = { + .quirks = MCP251XFD_QUIRK_CRC_REG | MCP251XFD_QUIRK_CRC_RX | + MCP251XFD_QUIRK_CRC_TX | MCP251XFD_QUIRK_ECC, + .model = MCP251XFD_MODEL_MCP2518FD, }; /* Autodetect model, start with CRC enabled. */ -static const struct mcp25xxfd_devtype_data mcp25xxfd_devtype_data_mcp25xxfd = { - .quirks = MCP25XXFD_QUIRK_CRC_REG | MCP25XXFD_QUIRK_CRC_RX | - MCP25XXFD_QUIRK_CRC_TX | MCP25XXFD_QUIRK_ECC, - .model = MCP25XXFD_MODEL_MCP25XXFD, +static const struct mcp251xfd_devtype_data mcp251xfd_devtype_data_mcp251xfd = { + .quirks = MCP251XFD_QUIRK_CRC_REG | MCP251XFD_QUIRK_CRC_RX | + MCP251XFD_QUIRK_CRC_TX | MCP251XFD_QUIRK_ECC, + .model = MCP251XFD_MODEL_MCP251XFD, }; -static const struct can_bittiming_const mcp25xxfd_bittiming_const = { +static const struct can_bittiming_const mcp251xfd_bittiming_const = { .name = DEVICE_NAME, .tseg1_min = 2, .tseg1_max = 256, @@ -59,7 +59,7 @@ static const struct can_bittiming_const mcp25xxfd_bittiming_const = { .brp_inc = 1, }; -static const struct can_bittiming_const mcp25xxfd_data_bittiming_const = { +static const struct can_bittiming_const mcp251xfd_data_bittiming_const = { .name = DEVICE_NAME, .tseg1_min = 1, .tseg1_max = 32, @@ -71,51 +71,51 @@ static const struct can_bittiming_const mcp25xxfd_data_bittiming_const = { .brp_inc = 1, }; -static const char *__mcp25xxfd_get_model_str(enum mcp25xxfd_model model) +static const char *__mcp251xfd_get_model_str(enum mcp251xfd_model model) { switch (model) { - case MCP25XXFD_MODEL_MCP2517FD: + case MCP251XFD_MODEL_MCP2517FD: return "MCP2517FD"; break; - case MCP25XXFD_MODEL_MCP2518FD: + case MCP251XFD_MODEL_MCP2518FD: return "MCP2518FD"; break; - case MCP25XXFD_MODEL_MCP25XXFD: - return "MCP25xxFD"; break; + case MCP251XFD_MODEL_MCP251XFD: + return "MCP251xFD"; break; } return "<unknown>"; } static inline const char * -mcp25xxfd_get_model_str(const struct mcp25xxfd_priv *priv) +mcp251xfd_get_model_str(const struct mcp251xfd_priv *priv) { - return __mcp25xxfd_get_model_str(priv->devtype_data.model); + return __mcp251xfd_get_model_str(priv->devtype_data.model); } -static const char *mcp25xxfd_get_mode_str(const u8 mode) +static const char *mcp251xfd_get_mode_str(const u8 mode) { switch (mode) { - case MCP25XXFD_REG_CON_MODE_MIXED: + case MCP251XFD_REG_CON_MODE_MIXED: return "Mixed (CAN FD/CAN 2.0)"; break; - case MCP25XXFD_REG_CON_MODE_SLEEP: + case MCP251XFD_REG_CON_MODE_SLEEP: return "Sleep"; break; - case MCP25XXFD_REG_CON_MODE_INT_LOOPBACK: + case MCP251XFD_REG_CON_MODE_INT_LOOPBACK: return "Internal Loopback"; break; - case MCP25XXFD_REG_CON_MODE_LISTENONLY: + case MCP251XFD_REG_CON_MODE_LISTENONLY: return "Listen Only"; break; - case MCP25XXFD_REG_CON_MODE_CONFIG: + case MCP251XFD_REG_CON_MODE_CONFIG: return "Configuration"; break; - case MCP25XXFD_REG_CON_MODE_EXT_LOOPBACK: + case MCP251XFD_REG_CON_MODE_EXT_LOOPBACK: return "External Loopback"; break; - case MCP25XXFD_REG_CON_MODE_CAN2_0: + case MCP251XFD_REG_CON_MODE_CAN2_0: return "CAN 2.0"; break; - case MCP25XXFD_REG_CON_MODE_RESTRICTED: + case MCP251XFD_REG_CON_MODE_RESTRICTED: return "Restricted Operation"; break; } return "<unknown>"; } -static inline int mcp25xxfd_vdd_enable(const struct mcp25xxfd_priv *priv) +static inline int mcp251xfd_vdd_enable(const struct mcp251xfd_priv *priv) { if (!priv->reg_vdd) return 0; @@ -123,7 +123,7 @@ static inline int mcp25xxfd_vdd_enable(const struct mcp25xxfd_priv *priv) return regulator_enable(priv->reg_vdd); } -static inline int mcp25xxfd_vdd_disable(const struct mcp25xxfd_priv *priv) +static inline int mcp251xfd_vdd_disable(const struct mcp251xfd_priv *priv) { if (!priv->reg_vdd) return 0; @@ -132,7 +132,7 @@ static inline int mcp25xxfd_vdd_disable(const struct mcp25xxfd_priv *priv) } static inline int -mcp25xxfd_transceiver_enable(const struct mcp25xxfd_priv *priv) +mcp251xfd_transceiver_enable(const struct mcp251xfd_priv *priv) { if (!priv->reg_xceiver) return 0; @@ -141,7 +141,7 @@ mcp25xxfd_transceiver_enable(const struct mcp25xxfd_priv *priv) } static inline int -mcp25xxfd_transceiver_disable(const struct mcp25xxfd_priv *priv) +mcp251xfd_transceiver_disable(const struct mcp251xfd_priv *priv) { if (!priv->reg_xceiver) return 0; @@ -149,7 +149,7 @@ mcp25xxfd_transceiver_disable(const struct mcp25xxfd_priv *priv) return regulator_disable(priv->reg_xceiver); } -static int mcp25xxfd_clks_and_vdd_enable(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_clks_and_vdd_enable(const struct mcp251xfd_priv *priv) { int err; @@ -157,22 +157,22 @@ static int mcp25xxfd_clks_and_vdd_enable(const struct mcp25xxfd_priv *priv) if (err) return err; - err = mcp25xxfd_vdd_enable(priv); + err = mcp251xfd_vdd_enable(priv); if (err) clk_disable_unprepare(priv->clk); /* Wait for oscillator stabilisation time after power up */ - usleep_range(MCP25XXFD_OSC_STAB_SLEEP_US, - 2 * MCP25XXFD_OSC_STAB_SLEEP_US); + usleep_range(MCP251XFD_OSC_STAB_SLEEP_US, + 2 * MCP251XFD_OSC_STAB_SLEEP_US); return err; } -static int mcp25xxfd_clks_and_vdd_disable(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_clks_and_vdd_disable(const struct mcp251xfd_priv *priv) { int err; - err = mcp25xxfd_vdd_disable(priv); + err = mcp251xfd_vdd_disable(priv); if (err) return err; @@ -182,30 +182,30 @@ static int mcp25xxfd_clks_and_vdd_disable(const struct mcp25xxfd_priv *priv) } static inline u8 -mcp25xxfd_cmd_prepare_write_reg(const struct mcp25xxfd_priv *priv, - union mcp25xxfd_write_reg_buf *write_reg_buf, +mcp251xfd_cmd_prepare_write_reg(const struct mcp251xfd_priv *priv, + union mcp251xfd_write_reg_buf *write_reg_buf, const u16 reg, const u32 mask, const u32 val) { u8 first_byte, last_byte, len; u8 *data; __le32 val_le32; - first_byte = mcp25xxfd_first_byte_set(mask); - last_byte = mcp25xxfd_last_byte_set(mask); + first_byte = mcp251xfd_first_byte_set(mask); + last_byte = mcp251xfd_last_byte_set(mask); len = last_byte - first_byte + 1; - data = mcp25xxfd_spi_cmd_write(priv, write_reg_buf, reg + first_byte); + data = mcp251xfd_spi_cmd_write(priv, write_reg_buf, reg + first_byte); val_le32 = cpu_to_le32(val >> BITS_PER_BYTE * first_byte); memcpy(data, &val_le32, len); - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_REG) { + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_REG) { u16 crc; - mcp25xxfd_spi_cmd_crc_set_len_in_reg(&write_reg_buf->crc.cmd, + mcp251xfd_spi_cmd_crc_set_len_in_reg(&write_reg_buf->crc.cmd, len); /* CRC */ len += sizeof(write_reg_buf->crc.cmd); - crc = mcp25xxfd_crc16_compute(&write_reg_buf->crc, len); + crc = mcp251xfd_crc16_compute(&write_reg_buf->crc, len); put_unaligned_be16(crc, (void *)write_reg_buf + len); /* Total length */ @@ -218,80 +218,80 @@ mcp25xxfd_cmd_prepare_write_reg(const struct mcp25xxfd_priv *priv, } static inline int -mcp25xxfd_tef_tail_get_from_chip(const struct mcp25xxfd_priv *priv, +mcp251xfd_tef_tail_get_from_chip(const struct mcp251xfd_priv *priv, u8 *tef_tail) { u32 tef_ua; int err; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_TEFUA, &tef_ua); + err = regmap_read(priv->map_reg, MCP251XFD_REG_TEFUA, &tef_ua); if (err) return err; - *tef_tail = tef_ua / sizeof(struct mcp25xxfd_hw_tef_obj); + *tef_tail = tef_ua / sizeof(struct mcp251xfd_hw_tef_obj); return 0; } static inline int -mcp25xxfd_tx_tail_get_from_chip(const struct mcp25xxfd_priv *priv, +mcp251xfd_tx_tail_get_from_chip(const struct mcp251xfd_priv *priv, u8 *tx_tail) { u32 fifo_sta; int err; err = regmap_read(priv->map_reg, - MCP25XXFD_REG_FIFOSTA(MCP25XXFD_TX_FIFO), + MCP251XFD_REG_FIFOSTA(MCP251XFD_TX_FIFO), &fifo_sta); if (err) return err; - *tx_tail = FIELD_GET(MCP25XXFD_REG_FIFOSTA_FIFOCI_MASK, fifo_sta); + *tx_tail = FIELD_GET(MCP251XFD_REG_FIFOSTA_FIFOCI_MASK, fifo_sta); return 0; } static inline int -mcp25xxfd_rx_head_get_from_chip(const struct mcp25xxfd_priv *priv, - const struct mcp25xxfd_rx_ring *ring, +mcp251xfd_rx_head_get_from_chip(const struct mcp251xfd_priv *priv, + const struct mcp251xfd_rx_ring *ring, u8 *rx_head) { u32 fifo_sta; int err; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_FIFOSTA(ring->fifo_nr), + err = regmap_read(priv->map_reg, MCP251XFD_REG_FIFOSTA(ring->fifo_nr), &fifo_sta); if (err) return err; - *rx_head = FIELD_GET(MCP25XXFD_REG_FIFOSTA_FIFOCI_MASK, fifo_sta); + *rx_head = FIELD_GET(MCP251XFD_REG_FIFOSTA_FIFOCI_MASK, fifo_sta); return 0; } static inline int -mcp25xxfd_rx_tail_get_from_chip(const struct mcp25xxfd_priv *priv, - const struct mcp25xxfd_rx_ring *ring, +mcp251xfd_rx_tail_get_from_chip(const struct mcp251xfd_priv *priv, + const struct mcp251xfd_rx_ring *ring, u8 *rx_tail) { u32 fifo_ua; int err; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_FIFOUA(ring->fifo_nr), + err = regmap_read(priv->map_reg, MCP251XFD_REG_FIFOUA(ring->fifo_nr), &fifo_ua); if (err) return err; - fifo_ua -= ring->base - MCP25XXFD_RAM_START; + fifo_ua -= ring->base - MCP251XFD_RAM_START; *rx_tail = fifo_ua / ring->obj_size; return 0; } static void -mcp25xxfd_tx_ring_init_tx_obj(const struct mcp25xxfd_priv *priv, - const struct mcp25xxfd_tx_ring *ring, - struct mcp25xxfd_tx_obj *tx_obj, +mcp251xfd_tx_ring_init_tx_obj(const struct mcp251xfd_priv *priv, + const struct mcp251xfd_tx_ring *ring, + struct mcp251xfd_tx_obj *tx_obj, const u8 rts_buf_len, const u8 n) { @@ -299,12 +299,12 @@ mcp25xxfd_tx_ring_init_tx_obj(const struct mcp25xxfd_priv *priv, u16 addr; /* FIFO load */ - addr = mcp25xxfd_get_tx_obj_addr(ring, n); - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_TX) - mcp25xxfd_spi_cmd_write_crc_set_addr(&tx_obj->buf.crc.cmd, + addr = mcp251xfd_get_tx_obj_addr(ring, n); + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_TX) + mcp251xfd_spi_cmd_write_crc_set_addr(&tx_obj->buf.crc.cmd, addr); else - mcp25xxfd_spi_cmd_write_nocrc(&tx_obj->buf.nocrc.cmd, + mcp251xfd_spi_cmd_write_nocrc(&tx_obj->buf.nocrc.cmd, addr); xfer = &tx_obj->xfer[0]; @@ -324,11 +324,11 @@ mcp25xxfd_tx_ring_init_tx_obj(const struct mcp25xxfd_priv *priv, ARRAY_SIZE(tx_obj->xfer)); } -static void mcp25xxfd_ring_init(struct mcp25xxfd_priv *priv) +static void mcp251xfd_ring_init(struct mcp251xfd_priv *priv) { - struct mcp25xxfd_tx_ring *tx_ring; - struct mcp25xxfd_rx_ring *rx_ring, *prev_rx_ring = NULL; - struct mcp25xxfd_tx_obj *tx_obj; + struct mcp251xfd_tx_ring *tx_ring; + struct mcp251xfd_rx_ring *rx_ring, *prev_rx_ring = NULL; + struct mcp251xfd_tx_obj *tx_obj; u32 val; u16 addr; u8 len; @@ -342,27 +342,27 @@ static void mcp25xxfd_ring_init(struct mcp25xxfd_priv *priv) tx_ring = priv->tx; tx_ring->head = 0; tx_ring->tail = 0; - tx_ring->base = mcp25xxfd_get_tef_obj_addr(tx_ring->obj_num); + tx_ring->base = mcp251xfd_get_tef_obj_addr(tx_ring->obj_num); /* FIFO request to send */ - addr = MCP25XXFD_REG_FIFOCON(MCP25XXFD_TX_FIFO); - val = MCP25XXFD_REG_FIFOCON_TXREQ | MCP25XXFD_REG_FIFOCON_UINC; - len = mcp25xxfd_cmd_prepare_write_reg(priv, &tx_ring->rts_buf, + addr = MCP251XFD_REG_FIFOCON(MCP251XFD_TX_FIFO); + val = MCP251XFD_REG_FIFOCON_TXREQ | MCP251XFD_REG_FIFOCON_UINC; + len = mcp251xfd_cmd_prepare_write_reg(priv, &tx_ring->rts_buf, addr, val, val); - mcp25xxfd_for_each_tx_obj(tx_ring, tx_obj, i) - mcp25xxfd_tx_ring_init_tx_obj(priv, tx_ring, tx_obj, len, i); + mcp251xfd_for_each_tx_obj(tx_ring, tx_obj, i) + mcp251xfd_tx_ring_init_tx_obj(priv, tx_ring, tx_obj, len, i); /* RX */ - mcp25xxfd_for_each_rx_ring(priv, rx_ring, i) { + mcp251xfd_for_each_rx_ring(priv, rx_ring, i) { rx_ring->head = 0; rx_ring->tail = 0; rx_ring->nr = i; - rx_ring->fifo_nr = MCP25XXFD_RX_FIFO(i); + rx_ring->fifo_nr = MCP251XFD_RX_FIFO(i); if (!prev_rx_ring) rx_ring->base = - mcp25xxfd_get_tx_obj_addr(tx_ring, + mcp251xfd_get_tx_obj_addr(tx_ring, tx_ring->obj_num); else rx_ring->base = prev_rx_ring->base + @@ -373,41 +373,41 @@ static void mcp25xxfd_ring_init(struct mcp25xxfd_priv *priv) } } -static void mcp25xxfd_ring_free(struct mcp25xxfd_priv *priv) +static void mcp251xfd_ring_free(struct mcp251xfd_priv *priv) { int i; - for (i = ARRAY_SIZE(priv->rx) - 1; i > 0; i--) { + for (i = ARRAY_SIZE(priv->rx) - 1; i >= 0; i--) { kfree(priv->rx[i]); priv->rx[i] = NULL; } } -static int mcp25xxfd_ring_alloc(struct mcp25xxfd_priv *priv) +static int mcp251xfd_ring_alloc(struct mcp251xfd_priv *priv) { - struct mcp25xxfd_tx_ring *tx_ring; - struct mcp25xxfd_rx_ring *rx_ring; + struct mcp251xfd_tx_ring *tx_ring; + struct mcp251xfd_rx_ring *rx_ring; int tef_obj_size, tx_obj_size, rx_obj_size; int tx_obj_num; int ram_free, i; - tef_obj_size = sizeof(struct mcp25xxfd_hw_tef_obj); + tef_obj_size = sizeof(struct mcp251xfd_hw_tef_obj); /* listen-only mode works like FD mode */ if (priv->can.ctrlmode & (CAN_CTRLMODE_LISTENONLY | CAN_CTRLMODE_FD)) { - tx_obj_num = MCP25XXFD_TX_OBJ_NUM_CANFD; - tx_obj_size = sizeof(struct mcp25xxfd_hw_tx_obj_canfd); - rx_obj_size = sizeof(struct mcp25xxfd_hw_rx_obj_canfd); + tx_obj_num = MCP251XFD_TX_OBJ_NUM_CANFD; + tx_obj_size = sizeof(struct mcp251xfd_hw_tx_obj_canfd); + rx_obj_size = sizeof(struct mcp251xfd_hw_rx_obj_canfd); } else { - tx_obj_num = MCP25XXFD_TX_OBJ_NUM_CAN; - tx_obj_size = sizeof(struct mcp25xxfd_hw_tx_obj_can); - rx_obj_size = sizeof(struct mcp25xxfd_hw_rx_obj_can); + tx_obj_num = MCP251XFD_TX_OBJ_NUM_CAN; + tx_obj_size = sizeof(struct mcp251xfd_hw_tx_obj_can); + rx_obj_size = sizeof(struct mcp251xfd_hw_rx_obj_can); } tx_ring = priv->tx; tx_ring->obj_num = tx_obj_num; tx_ring->obj_size = tx_obj_size; - ram_free = MCP25XXFD_RAM_SIZE - tx_obj_num * + ram_free = MCP251XFD_RAM_SIZE - tx_obj_num * (tef_obj_size + tx_obj_size); for (i = 0; @@ -421,7 +421,7 @@ static int mcp25xxfd_ring_alloc(struct mcp25xxfd_priv *priv) rx_ring = kzalloc(sizeof(*rx_ring) + rx_obj_size * rx_obj_num, GFP_KERNEL); if (!rx_ring) { - mcp25xxfd_ring_free(priv); + mcp251xfd_ring_free(priv); return -ENOMEM; } rx_ring->obj_num = rx_obj_num; @@ -437,7 +437,7 @@ static int mcp25xxfd_ring_alloc(struct mcp25xxfd_priv *priv) tx_obj_num, tef_obj_size, tef_obj_size * tx_obj_num, tx_obj_num, tx_obj_size, tx_obj_size * tx_obj_num); - mcp25xxfd_for_each_rx_ring(priv, rx_ring, i) { + mcp251xfd_for_each_rx_ring(priv, rx_ring, i) { netdev_dbg(priv->ndev, "FIFO setup: RX-%d: %d*%d bytes = %d bytes\n", i, rx_ring->obj_num, rx_ring->obj_size, @@ -452,48 +452,48 @@ static int mcp25xxfd_ring_alloc(struct mcp25xxfd_priv *priv) } static inline int -mcp25xxfd_chip_get_mode(const struct mcp25xxfd_priv *priv, u8 *mode) +mcp251xfd_chip_get_mode(const struct mcp251xfd_priv *priv, u8 *mode) { u32 val; int err; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_CON, &val); + err = regmap_read(priv->map_reg, MCP251XFD_REG_CON, &val); if (err) return err; - *mode = FIELD_GET(MCP25XXFD_REG_CON_OPMOD_MASK, val); + *mode = FIELD_GET(MCP251XFD_REG_CON_OPMOD_MASK, val); return 0; } static int -__mcp25xxfd_chip_set_mode(const struct mcp25xxfd_priv *priv, +__mcp251xfd_chip_set_mode(const struct mcp251xfd_priv *priv, const u8 mode_req, bool nowait) { u32 con, con_reqop; int err; - con_reqop = FIELD_PREP(MCP25XXFD_REG_CON_REQOP_MASK, mode_req); - err = regmap_update_bits(priv->map_reg, MCP25XXFD_REG_CON, - MCP25XXFD_REG_CON_REQOP_MASK, con_reqop); + con_reqop = FIELD_PREP(MCP251XFD_REG_CON_REQOP_MASK, mode_req); + err = regmap_update_bits(priv->map_reg, MCP251XFD_REG_CON, + MCP251XFD_REG_CON_REQOP_MASK, con_reqop); if (err) return err; - if (mode_req == MCP25XXFD_REG_CON_MODE_SLEEP || nowait) + if (mode_req == MCP251XFD_REG_CON_MODE_SLEEP || nowait) return 0; - err = regmap_read_poll_timeout(priv->map_reg, MCP25XXFD_REG_CON, con, - FIELD_GET(MCP25XXFD_REG_CON_OPMOD_MASK, + err = regmap_read_poll_timeout(priv->map_reg, MCP251XFD_REG_CON, con, + FIELD_GET(MCP251XFD_REG_CON_OPMOD_MASK, con) == mode_req, - MCP25XXFD_POLL_SLEEP_US, - MCP25XXFD_POLL_TIMEOUT_US); + MCP251XFD_POLL_SLEEP_US, + MCP251XFD_POLL_TIMEOUT_US); if (err) { - u8 mode = FIELD_GET(MCP25XXFD_REG_CON_OPMOD_MASK, con); + u8 mode = FIELD_GET(MCP251XFD_REG_CON_OPMOD_MASK, con); netdev_err(priv->ndev, "Controller failed to enter mode %s Mode (%u) and stays in %s Mode (%u).\n", - mcp25xxfd_get_mode_str(mode_req), mode_req, - mcp25xxfd_get_mode_str(mode), mode); + mcp251xfd_get_mode_str(mode_req), mode_req, + mcp251xfd_get_mode_str(mode), mode); return err; } @@ -501,25 +501,25 @@ __mcp25xxfd_chip_set_mode(const struct mcp25xxfd_priv *priv, } static inline int -mcp25xxfd_chip_set_mode(const struct mcp25xxfd_priv *priv, +mcp251xfd_chip_set_mode(const struct mcp251xfd_priv *priv, const u8 mode_req) { - return __mcp25xxfd_chip_set_mode(priv, mode_req, false); + return __mcp251xfd_chip_set_mode(priv, mode_req, false); } static inline int -mcp25xxfd_chip_set_mode_nowait(const struct mcp25xxfd_priv *priv, +mcp251xfd_chip_set_mode_nowait(const struct mcp251xfd_priv *priv, const u8 mode_req) { - return __mcp25xxfd_chip_set_mode(priv, mode_req, true); + return __mcp251xfd_chip_set_mode(priv, mode_req, true); } -static inline bool mcp25xxfd_osc_invalid(u32 reg) +static inline bool mcp251xfd_osc_invalid(u32 reg) { return reg == 0x0 || reg == 0xffffffff; } -static int mcp25xxfd_chip_clock_enable(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_clock_enable(const struct mcp251xfd_priv *priv) { u32 osc, osc_reference, osc_mask; int err; @@ -527,10 +527,10 @@ static int mcp25xxfd_chip_clock_enable(const struct mcp25xxfd_priv *priv) /* Set Power On Defaults for "Clock Output Divisor" and remove * "Oscillator Disable" bit. */ - osc = FIELD_PREP(MCP25XXFD_REG_OSC_CLKODIV_MASK, - MCP25XXFD_REG_OSC_CLKODIV_10); - osc_reference = MCP25XXFD_REG_OSC_OSCRDY; - osc_mask = MCP25XXFD_REG_OSC_OSCRDY | MCP25XXFD_REG_OSC_PLLRDY; + osc = FIELD_PREP(MCP251XFD_REG_OSC_CLKODIV_MASK, + MCP251XFD_REG_OSC_CLKODIV_10); + osc_reference = MCP251XFD_REG_OSC_OSCRDY; + osc_mask = MCP251XFD_REG_OSC_OSCRDY | MCP251XFD_REG_OSC_PLLRDY; /* Note: * @@ -538,19 +538,19 @@ static int mcp25xxfd_chip_clock_enable(const struct mcp25xxfd_priv *priv) * removes the "Oscillator Disable" bit and powers it up. All * other bits are unaffected. */ - err = regmap_write(priv->map_reg, MCP25XXFD_REG_OSC, osc); + err = regmap_write(priv->map_reg, MCP251XFD_REG_OSC, osc); if (err) return err; /* Wait for "Oscillator Ready" bit */ - err = regmap_read_poll_timeout(priv->map_reg, MCP25XXFD_REG_OSC, osc, + err = regmap_read_poll_timeout(priv->map_reg, MCP251XFD_REG_OSC, osc, (osc & osc_mask) == osc_reference, - MCP25XXFD_OSC_STAB_SLEEP_US, - MCP25XXFD_OSC_STAB_TIMEOUT_US); - if (mcp25xxfd_osc_invalid(osc)) { + MCP251XFD_OSC_STAB_SLEEP_US, + MCP251XFD_OSC_STAB_TIMEOUT_US); + if (mcp251xfd_osc_invalid(osc)) { netdev_err(priv->ndev, "Failed to detect %s (osc=0x%08x).\n", - mcp25xxfd_get_model_str(priv), osc); + mcp251xfd_get_model_str(priv), osc); return -ENODEV; } else if (err == -ETIMEDOUT) { netdev_err(priv->ndev, @@ -564,19 +564,19 @@ static int mcp25xxfd_chip_clock_enable(const struct mcp25xxfd_priv *priv) return 0; } -static int mcp25xxfd_chip_softreset_do(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_softreset_do(const struct mcp251xfd_priv *priv) { - const __be16 cmd = mcp25xxfd_cmd_reset(); + const __be16 cmd = mcp251xfd_cmd_reset(); int err; /* The Set Mode and SPI Reset command only seems to works if * the controller is not in Sleep Mode. */ - err = mcp25xxfd_chip_clock_enable(priv); + err = mcp251xfd_chip_clock_enable(priv); if (err) return err; - err = mcp25xxfd_chip_set_mode(priv, MCP25XXFD_REG_CON_MODE_CONFIG); + err = mcp251xfd_chip_set_mode(priv, MCP251XFD_REG_CON_MODE_CONFIG); if (err) return err; @@ -584,29 +584,29 @@ static int mcp25xxfd_chip_softreset_do(const struct mcp25xxfd_priv *priv) return spi_write_then_read(priv->spi, &cmd, sizeof(cmd), NULL, 0); } -static int mcp25xxfd_chip_softreset_check(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_softreset_check(const struct mcp251xfd_priv *priv) { u32 osc, osc_reference; u8 mode; int err; - err = mcp25xxfd_chip_get_mode(priv, &mode); + err = mcp251xfd_chip_get_mode(priv, &mode); if (err) return err; - if (mode != MCP25XXFD_REG_CON_MODE_CONFIG) { + if (mode != MCP251XFD_REG_CON_MODE_CONFIG) { netdev_info(priv->ndev, "Controller not in Config Mode after reset, but in %s Mode (%u).\n", - mcp25xxfd_get_mode_str(mode), mode); + mcp251xfd_get_mode_str(mode), mode); return -ETIMEDOUT; } - osc_reference = MCP25XXFD_REG_OSC_OSCRDY | - FIELD_PREP(MCP25XXFD_REG_OSC_CLKODIV_MASK, - MCP25XXFD_REG_OSC_CLKODIV_10); + osc_reference = MCP251XFD_REG_OSC_OSCRDY | + FIELD_PREP(MCP251XFD_REG_OSC_CLKODIV_MASK, + MCP251XFD_REG_OSC_CLKODIV_10); /* check reset defaults of OSC reg */ - err = regmap_read(priv->map_reg, MCP25XXFD_REG_OSC, &osc); + err = regmap_read(priv->map_reg, MCP251XFD_REG_OSC, &osc); if (err) return err; @@ -620,22 +620,22 @@ static int mcp25xxfd_chip_softreset_check(const struct mcp25xxfd_priv *priv) return 0; } -static int mcp25xxfd_chip_softreset(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_softreset(const struct mcp251xfd_priv *priv) { int err, i; - for (i = 0; i < MCP25XXFD_SOFTRESET_RETRIES_MAX; i++) { + for (i = 0; i < MCP251XFD_SOFTRESET_RETRIES_MAX; i++) { if (i) netdev_info(priv->ndev, "Retrying to reset Controller.\n"); - err = mcp25xxfd_chip_softreset_do(priv); + err = mcp251xfd_chip_softreset_do(priv); if (err == -ETIMEDOUT) continue; if (err) return err; - err = mcp25xxfd_chip_softreset_check(priv); + err = mcp251xfd_chip_softreset_check(priv); if (err == -ETIMEDOUT) continue; if (err) @@ -650,7 +650,7 @@ static int mcp25xxfd_chip_softreset(const struct mcp25xxfd_priv *priv) return -ETIMEDOUT; } -static int mcp25xxfd_chip_clock_init(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_clock_init(const struct mcp251xfd_priv *priv) { u32 osc; int err; @@ -659,10 +659,10 @@ static int mcp25xxfd_chip_clock_init(const struct mcp25xxfd_priv *priv) * works on the MCP2518FD. The MCP2517FD will go into normal * Sleep Mode instead. */ - osc = MCP25XXFD_REG_OSC_LPMEN | - FIELD_PREP(MCP25XXFD_REG_OSC_CLKODIV_MASK, - MCP25XXFD_REG_OSC_CLKODIV_10); - err = regmap_write(priv->map_reg, MCP25XXFD_REG_OSC, osc); + osc = MCP251XFD_REG_OSC_LPMEN | + FIELD_PREP(MCP251XFD_REG_OSC_CLKODIV_MASK, + MCP251XFD_REG_OSC_CLKODIV_10); + err = regmap_write(priv->map_reg, MCP251XFD_REG_OSC, osc); if (err) return err; @@ -671,11 +671,11 @@ static int mcp25xxfd_chip_clock_init(const struct mcp25xxfd_priv *priv) * This means an overflow of the 32 bit Time Base Counter * register at 40 MHz every 107 seconds. */ - return regmap_write(priv->map_reg, MCP25XXFD_REG_TSCON, - MCP25XXFD_REG_TSCON_TBCEN); + return regmap_write(priv->map_reg, MCP251XFD_REG_TSCON, + MCP251XFD_REG_TSCON_TBCEN); } -static int mcp25xxfd_set_bittiming(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_set_bittiming(const struct mcp251xfd_priv *priv) { const struct can_bittiming *bt = &priv->can.bittiming; const struct can_bittiming *dbt = &priv->can.data_bittiming; @@ -699,32 +699,32 @@ static int mcp25xxfd_set_bittiming(const struct mcp25xxfd_priv *priv) * - protocol exception is treated as a form error * - Do not compare data bytes */ - val = FIELD_PREP(MCP25XXFD_REG_CON_REQOP_MASK, - MCP25XXFD_REG_CON_MODE_CONFIG) | - MCP25XXFD_REG_CON_STEF | - MCP25XXFD_REG_CON_ESIGM | - MCP25XXFD_REG_CON_RTXAT | - FIELD_PREP(MCP25XXFD_REG_CON_WFT_MASK, - MCP25XXFD_REG_CON_WFT_T11FILTER) | - MCP25XXFD_REG_CON_WAKFIL | - MCP25XXFD_REG_CON_PXEDIS; + val = FIELD_PREP(MCP251XFD_REG_CON_REQOP_MASK, + MCP251XFD_REG_CON_MODE_CONFIG) | + MCP251XFD_REG_CON_STEF | + MCP251XFD_REG_CON_ESIGM | + MCP251XFD_REG_CON_RTXAT | + FIELD_PREP(MCP251XFD_REG_CON_WFT_MASK, + MCP251XFD_REG_CON_WFT_T11FILTER) | + MCP251XFD_REG_CON_WAKFIL | + MCP251XFD_REG_CON_PXEDIS; if (!(priv->can.ctrlmode & CAN_CTRLMODE_FD_NON_ISO)) - val |= MCP25XXFD_REG_CON_ISOCRCEN; + val |= MCP251XFD_REG_CON_ISOCRCEN; - err = regmap_write(priv->map_reg, MCP25XXFD_REG_CON, val); + err = regmap_write(priv->map_reg, MCP251XFD_REG_CON, val); if (err) return err; /* Nominal Bit Time */ - val = FIELD_PREP(MCP25XXFD_REG_NBTCFG_BRP_MASK, bt->brp - 1) | - FIELD_PREP(MCP25XXFD_REG_NBTCFG_TSEG1_MASK, + val = FIELD_PREP(MCP251XFD_REG_NBTCFG_BRP_MASK, bt->brp - 1) | + FIELD_PREP(MCP251XFD_REG_NBTCFG_TSEG1_MASK, bt->prop_seg + bt->phase_seg1 - 1) | - FIELD_PREP(MCP25XXFD_REG_NBTCFG_TSEG2_MASK, + FIELD_PREP(MCP251XFD_REG_NBTCFG_TSEG2_MASK, bt->phase_seg2 - 1) | - FIELD_PREP(MCP25XXFD_REG_NBTCFG_SJW_MASK, bt->sjw - 1); + FIELD_PREP(MCP251XFD_REG_NBTCFG_SJW_MASK, bt->sjw - 1); - err = regmap_write(priv->map_reg, MCP25XXFD_REG_NBTCFG, val); + err = regmap_write(priv->map_reg, MCP251XFD_REG_NBTCFG, val); if (err) return err; @@ -732,28 +732,28 @@ static int mcp25xxfd_set_bittiming(const struct mcp25xxfd_priv *priv) return 0; /* Data Bit Time */ - val = FIELD_PREP(MCP25XXFD_REG_DBTCFG_BRP_MASK, dbt->brp - 1) | - FIELD_PREP(MCP25XXFD_REG_DBTCFG_TSEG1_MASK, + val = FIELD_PREP(MCP251XFD_REG_DBTCFG_BRP_MASK, dbt->brp - 1) | + FIELD_PREP(MCP251XFD_REG_DBTCFG_TSEG1_MASK, dbt->prop_seg + dbt->phase_seg1 - 1) | - FIELD_PREP(MCP25XXFD_REG_DBTCFG_TSEG2_MASK, + FIELD_PREP(MCP251XFD_REG_DBTCFG_TSEG2_MASK, dbt->phase_seg2 - 1) | - FIELD_PREP(MCP25XXFD_REG_DBTCFG_SJW_MASK, dbt->sjw - 1); + FIELD_PREP(MCP251XFD_REG_DBTCFG_SJW_MASK, dbt->sjw - 1); - err = regmap_write(priv->map_reg, MCP25XXFD_REG_DBTCFG, val); + err = regmap_write(priv->map_reg, MCP251XFD_REG_DBTCFG, val); if (err) return err; /* Transmitter Delay Compensation */ tdco = clamp_t(int, dbt->brp * (dbt->prop_seg + dbt->phase_seg1), -64, 63); - val = FIELD_PREP(MCP25XXFD_REG_TDC_TDCMOD_MASK, - MCP25XXFD_REG_TDC_TDCMOD_AUTO) | - FIELD_PREP(MCP25XXFD_REG_TDC_TDCO_MASK, tdco); + val = FIELD_PREP(MCP251XFD_REG_TDC_TDCMOD_MASK, + MCP251XFD_REG_TDC_TDCMOD_AUTO) | + FIELD_PREP(MCP251XFD_REG_TDC_TDCO_MASK, tdco); - return regmap_write(priv->map_reg, MCP25XXFD_REG_TDC, val); + return regmap_write(priv->map_reg, MCP251XFD_REG_TDC, val); } -static int mcp25xxfd_chip_rx_int_enable(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_rx_int_enable(const struct mcp251xfd_priv *priv) { u32 val; @@ -769,12 +769,12 @@ static int mcp25xxfd_chip_rx_int_enable(const struct mcp25xxfd_priv *priv) * (in the first byte of the SPI transfer) and configuring the * PIN as interrupt (in the last byte of the SPI transfer). */ - val = MCP25XXFD_REG_IOCON_PM0 | MCP25XXFD_REG_IOCON_TRIS1 | - MCP25XXFD_REG_IOCON_TRIS0; - return regmap_write(priv->map_reg, MCP25XXFD_REG_IOCON, val); + val = MCP251XFD_REG_IOCON_PM0 | MCP251XFD_REG_IOCON_TRIS1 | + MCP251XFD_REG_IOCON_TRIS0; + return regmap_write(priv->map_reg, MCP251XFD_REG_IOCON, val); } -static int mcp25xxfd_chip_rx_int_disable(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_rx_int_disable(const struct mcp251xfd_priv *priv) { u32 val; @@ -785,14 +785,14 @@ static int mcp25xxfd_chip_rx_int_disable(const struct mcp25xxfd_priv *priv) * - PIN0: GPIO Input * - PIN1: GPIO Input */ - val = MCP25XXFD_REG_IOCON_PM1 | MCP25XXFD_REG_IOCON_PM0 | - MCP25XXFD_REG_IOCON_TRIS1 | MCP25XXFD_REG_IOCON_TRIS0; - return regmap_write(priv->map_reg, MCP25XXFD_REG_IOCON, val); + val = MCP251XFD_REG_IOCON_PM1 | MCP251XFD_REG_IOCON_PM0 | + MCP251XFD_REG_IOCON_TRIS1 | MCP251XFD_REG_IOCON_TRIS0; + return regmap_write(priv->map_reg, MCP251XFD_REG_IOCON, val); } static int -mcp25xxfd_chip_rx_fifo_init_one(const struct mcp25xxfd_priv *priv, - const struct mcp25xxfd_rx_ring *ring) +mcp251xfd_chip_rx_fifo_init_one(const struct mcp251xfd_priv *priv, + const struct mcp251xfd_rx_ring *ring) { u32 fifo_con; @@ -802,89 +802,89 @@ mcp25xxfd_chip_rx_fifo_init_one(const struct mcp25xxfd_priv *priv, * generate a RXOVIF, use this to properly detect RX MAB * overflows. */ - fifo_con = FIELD_PREP(MCP25XXFD_REG_FIFOCON_FSIZE_MASK, + fifo_con = FIELD_PREP(MCP251XFD_REG_FIFOCON_FSIZE_MASK, ring->obj_num - 1) | - MCP25XXFD_REG_FIFOCON_RXTSEN | - MCP25XXFD_REG_FIFOCON_RXOVIE | - MCP25XXFD_REG_FIFOCON_TFNRFNIE; + MCP251XFD_REG_FIFOCON_RXTSEN | + MCP251XFD_REG_FIFOCON_RXOVIE | + MCP251XFD_REG_FIFOCON_TFNRFNIE; if (priv->can.ctrlmode & (CAN_CTRLMODE_LISTENONLY | CAN_CTRLMODE_FD)) - fifo_con |= FIELD_PREP(MCP25XXFD_REG_FIFOCON_PLSIZE_MASK, - MCP25XXFD_REG_FIFOCON_PLSIZE_64); + fifo_con |= FIELD_PREP(MCP251XFD_REG_FIFOCON_PLSIZE_MASK, + MCP251XFD_REG_FIFOCON_PLSIZE_64); else - fifo_con |= FIELD_PREP(MCP25XXFD_REG_FIFOCON_PLSIZE_MASK, - MCP25XXFD_REG_FIFOCON_PLSIZE_8); + fifo_con |= FIELD_PREP(MCP251XFD_REG_FIFOCON_PLSIZE_MASK, + MCP251XFD_REG_FIFOCON_PLSIZE_8); return regmap_write(priv->map_reg, - MCP25XXFD_REG_FIFOCON(ring->fifo_nr), fifo_con); + MCP251XFD_REG_FIFOCON(ring->fifo_nr), fifo_con); } static int -mcp25xxfd_chip_rx_filter_init_one(const struct mcp25xxfd_priv *priv, - const struct mcp25xxfd_rx_ring *ring) +mcp251xfd_chip_rx_filter_init_one(const struct mcp251xfd_priv *priv, + const struct mcp251xfd_rx_ring *ring) { u32 fltcon; - fltcon = MCP25XXFD_REG_FLTCON_FLTEN(ring->nr) | - MCP25XXFD_REG_FLTCON_FBP(ring->nr, ring->fifo_nr); + fltcon = MCP251XFD_REG_FLTCON_FLTEN(ring->nr) | + MCP251XFD_REG_FLTCON_FBP(ring->nr, ring->fifo_nr); return regmap_update_bits(priv->map_reg, - MCP25XXFD_REG_FLTCON(ring->nr >> 2), - MCP25XXFD_REG_FLTCON_FLT_MASK(ring->nr), + MCP251XFD_REG_FLTCON(ring->nr >> 2), + MCP251XFD_REG_FLTCON_FLT_MASK(ring->nr), fltcon); } -static int mcp25xxfd_chip_fifo_init(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_fifo_init(const struct mcp251xfd_priv *priv) { - const struct mcp25xxfd_tx_ring *tx_ring = priv->tx; - const struct mcp25xxfd_rx_ring *rx_ring; + const struct mcp251xfd_tx_ring *tx_ring = priv->tx; + const struct mcp251xfd_rx_ring *rx_ring; u32 val; int err, n; /* TEF */ - val = FIELD_PREP(MCP25XXFD_REG_TEFCON_FSIZE_MASK, + val = FIELD_PREP(MCP251XFD_REG_TEFCON_FSIZE_MASK, tx_ring->obj_num - 1) | - MCP25XXFD_REG_TEFCON_TEFTSEN | - MCP25XXFD_REG_TEFCON_TEFOVIE | - MCP25XXFD_REG_TEFCON_TEFNEIE; + MCP251XFD_REG_TEFCON_TEFTSEN | + MCP251XFD_REG_TEFCON_TEFOVIE | + MCP251XFD_REG_TEFCON_TEFNEIE; - err = regmap_write(priv->map_reg, MCP25XXFD_REG_TEFCON, val); + err = regmap_write(priv->map_reg, MCP251XFD_REG_TEFCON, val); if (err) return err; /* FIFO 1 - TX */ - val = FIELD_PREP(MCP25XXFD_REG_FIFOCON_FSIZE_MASK, + val = FIELD_PREP(MCP251XFD_REG_FIFOCON_FSIZE_MASK, tx_ring->obj_num - 1) | - MCP25XXFD_REG_FIFOCON_TXEN | - MCP25XXFD_REG_FIFOCON_TXATIE; + MCP251XFD_REG_FIFOCON_TXEN | + MCP251XFD_REG_FIFOCON_TXATIE; if (priv->can.ctrlmode & (CAN_CTRLMODE_LISTENONLY | CAN_CTRLMODE_FD)) - val |= FIELD_PREP(MCP25XXFD_REG_FIFOCON_PLSIZE_MASK, - MCP25XXFD_REG_FIFOCON_PLSIZE_64); + val |= FIELD_PREP(MCP251XFD_REG_FIFOCON_PLSIZE_MASK, + MCP251XFD_REG_FIFOCON_PLSIZE_64); else - val |= FIELD_PREP(MCP25XXFD_REG_FIFOCON_PLSIZE_MASK, - MCP25XXFD_REG_FIFOCON_PLSIZE_8); + val |= FIELD_PREP(MCP251XFD_REG_FIFOCON_PLSIZE_MASK, + MCP251XFD_REG_FIFOCON_PLSIZE_8); if (priv->can.ctrlmode & CAN_CTRLMODE_ONE_SHOT) - val |= FIELD_PREP(MCP25XXFD_REG_FIFOCON_TXAT_MASK, - MCP25XXFD_REG_FIFOCON_TXAT_ONE_SHOT); + val |= FIELD_PREP(MCP251XFD_REG_FIFOCON_TXAT_MASK, + MCP251XFD_REG_FIFOCON_TXAT_ONE_SHOT); else - val |= FIELD_PREP(MCP25XXFD_REG_FIFOCON_TXAT_MASK, - MCP25XXFD_REG_FIFOCON_TXAT_UNLIMITED); + val |= FIELD_PREP(MCP251XFD_REG_FIFOCON_TXAT_MASK, + MCP251XFD_REG_FIFOCON_TXAT_UNLIMITED); err = regmap_write(priv->map_reg, - MCP25XXFD_REG_FIFOCON(MCP25XXFD_TX_FIFO), + MCP251XFD_REG_FIFOCON(MCP251XFD_TX_FIFO), val); if (err) return err; /* RX FIFOs */ - mcp25xxfd_for_each_rx_ring(priv, rx_ring, n) { - err = mcp25xxfd_chip_rx_fifo_init_one(priv, rx_ring); + mcp251xfd_for_each_rx_ring(priv, rx_ring, n) { + err = mcp251xfd_chip_rx_fifo_init_one(priv, rx_ring); if (err) return err; - err = mcp25xxfd_chip_rx_filter_init_one(priv, rx_ring); + err = mcp251xfd_chip_rx_filter_init_one(priv, rx_ring); if (err) return err; } @@ -892,195 +892,195 @@ static int mcp25xxfd_chip_fifo_init(const struct mcp25xxfd_priv *priv) return 0; } -static int mcp25xxfd_chip_ecc_init(struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_ecc_init(struct mcp251xfd_priv *priv) { - struct mcp25xxfd_ecc *ecc = &priv->ecc; + struct mcp251xfd_ecc *ecc = &priv->ecc; void *ram; u32 val = 0; int err; ecc->ecc_stat = 0; - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_ECC) - val = MCP25XXFD_REG_ECCCON_ECCEN; + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_ECC) + val = MCP251XFD_REG_ECCCON_ECCEN; - err = regmap_update_bits(priv->map_reg, MCP25XXFD_REG_ECCCON, - MCP25XXFD_REG_ECCCON_ECCEN, val); + err = regmap_update_bits(priv->map_reg, MCP251XFD_REG_ECCCON, + MCP251XFD_REG_ECCCON_ECCEN, val); if (err) return err; - ram = kzalloc(MCP25XXFD_RAM_SIZE, GFP_KERNEL); + ram = kzalloc(MCP251XFD_RAM_SIZE, GFP_KERNEL); if (!ram) return -ENOMEM; - err = regmap_raw_write(priv->map_reg, MCP25XXFD_RAM_START, ram, - MCP25XXFD_RAM_SIZE); + err = regmap_raw_write(priv->map_reg, MCP251XFD_RAM_START, ram, + MCP251XFD_RAM_SIZE); kfree(ram); return err; } -static inline void mcp25xxfd_ecc_tefif_successful(struct mcp25xxfd_priv *priv) +static inline void mcp251xfd_ecc_tefif_successful(struct mcp251xfd_priv *priv) { - struct mcp25xxfd_ecc *ecc = &priv->ecc; + struct mcp251xfd_ecc *ecc = &priv->ecc; ecc->ecc_stat = 0; } -static u8 mcp25xxfd_get_normal_mode(const struct mcp25xxfd_priv *priv) +static u8 mcp251xfd_get_normal_mode(const struct mcp251xfd_priv *priv) { u8 mode; if (priv->can.ctrlmode & CAN_CTRLMODE_LISTENONLY) - mode = MCP25XXFD_REG_CON_MODE_LISTENONLY; + mode = MCP251XFD_REG_CON_MODE_LISTENONLY; else if (priv->can.ctrlmode & CAN_CTRLMODE_FD) - mode = MCP25XXFD_REG_CON_MODE_MIXED; + mode = MCP251XFD_REG_CON_MODE_MIXED; else - mode = MCP25XXFD_REG_CON_MODE_CAN2_0; + mode = MCP251XFD_REG_CON_MODE_CAN2_0; return mode; } static int -__mcp25xxfd_chip_set_normal_mode(const struct mcp25xxfd_priv *priv, +__mcp251xfd_chip_set_normal_mode(const struct mcp251xfd_priv *priv, bool nowait) { u8 mode; - mode = mcp25xxfd_get_normal_mode(priv); + mode = mcp251xfd_get_normal_mode(priv); - return __mcp25xxfd_chip_set_mode(priv, mode, nowait); + return __mcp251xfd_chip_set_mode(priv, mode, nowait); } static inline int -mcp25xxfd_chip_set_normal_mode(const struct mcp25xxfd_priv *priv) +mcp251xfd_chip_set_normal_mode(const struct mcp251xfd_priv *priv) { - return __mcp25xxfd_chip_set_normal_mode(priv, false); + return __mcp251xfd_chip_set_normal_mode(priv, false); } static inline int -mcp25xxfd_chip_set_normal_mode_nowait(const struct mcp25xxfd_priv *priv) +mcp251xfd_chip_set_normal_mode_nowait(const struct mcp251xfd_priv *priv) { - return __mcp25xxfd_chip_set_normal_mode(priv, true); + return __mcp251xfd_chip_set_normal_mode(priv, true); } -static int mcp25xxfd_chip_interrupts_enable(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_interrupts_enable(const struct mcp251xfd_priv *priv) { u32 val; int err; - val = MCP25XXFD_REG_CRC_FERRIE | MCP25XXFD_REG_CRC_CRCERRIE; - err = regmap_write(priv->map_reg, MCP25XXFD_REG_CRC, val); + val = MCP251XFD_REG_CRC_FERRIE | MCP251XFD_REG_CRC_CRCERRIE; + err = regmap_write(priv->map_reg, MCP251XFD_REG_CRC, val); if (err) return err; - val = MCP25XXFD_REG_ECCCON_DEDIE | MCP25XXFD_REG_ECCCON_SECIE; - err = regmap_update_bits(priv->map_reg, MCP25XXFD_REG_ECCCON, val, val); + val = MCP251XFD_REG_ECCCON_DEDIE | MCP251XFD_REG_ECCCON_SECIE; + err = regmap_update_bits(priv->map_reg, MCP251XFD_REG_ECCCON, val, val); if (err) return err; - val = MCP25XXFD_REG_INT_CERRIE | - MCP25XXFD_REG_INT_SERRIE | - MCP25XXFD_REG_INT_RXOVIE | - MCP25XXFD_REG_INT_TXATIE | - MCP25XXFD_REG_INT_SPICRCIE | - MCP25XXFD_REG_INT_ECCIE | - MCP25XXFD_REG_INT_TEFIE | - MCP25XXFD_REG_INT_MODIE | - MCP25XXFD_REG_INT_RXIE; + val = MCP251XFD_REG_INT_CERRIE | + MCP251XFD_REG_INT_SERRIE | + MCP251XFD_REG_INT_RXOVIE | + MCP251XFD_REG_INT_TXATIE | + MCP251XFD_REG_INT_SPICRCIE | + MCP251XFD_REG_INT_ECCIE | + MCP251XFD_REG_INT_TEFIE | + MCP251XFD_REG_INT_MODIE | + MCP251XFD_REG_INT_RXIE; if (priv->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING) - val |= MCP25XXFD_REG_INT_IVMIE; + val |= MCP251XFD_REG_INT_IVMIE; - return regmap_write(priv->map_reg, MCP25XXFD_REG_INT, val); + return regmap_write(priv->map_reg, MCP251XFD_REG_INT, val); } -static int mcp25xxfd_chip_interrupts_disable(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_interrupts_disable(const struct mcp251xfd_priv *priv) { int err; u32 mask; - err = regmap_write(priv->map_reg, MCP25XXFD_REG_INT, 0); + err = regmap_write(priv->map_reg, MCP251XFD_REG_INT, 0); if (err) return err; - mask = MCP25XXFD_REG_ECCCON_DEDIE | MCP25XXFD_REG_ECCCON_SECIE; - err = regmap_update_bits(priv->map_reg, MCP25XXFD_REG_ECCCON, + mask = MCP251XFD_REG_ECCCON_DEDIE | MCP251XFD_REG_ECCCON_SECIE; + err = regmap_update_bits(priv->map_reg, MCP251XFD_REG_ECCCON, mask, 0x0); if (err) return err; - return regmap_write(priv->map_reg, MCP25XXFD_REG_CRC, 0); + return regmap_write(priv->map_reg, MCP251XFD_REG_CRC, 0); } -static int mcp25xxfd_chip_stop(struct mcp25xxfd_priv *priv, +static int mcp251xfd_chip_stop(struct mcp251xfd_priv *priv, const enum can_state state) { priv->can.state = state; - mcp25xxfd_chip_interrupts_disable(priv); - mcp25xxfd_chip_rx_int_disable(priv); - return mcp25xxfd_chip_set_mode(priv, MCP25XXFD_REG_CON_MODE_SLEEP); + mcp251xfd_chip_interrupts_disable(priv); + mcp251xfd_chip_rx_int_disable(priv); + return mcp251xfd_chip_set_mode(priv, MCP251XFD_REG_CON_MODE_SLEEP); } -static int mcp25xxfd_chip_start(struct mcp25xxfd_priv *priv) +static int mcp251xfd_chip_start(struct mcp251xfd_priv *priv) { int err; - err = mcp25xxfd_chip_softreset(priv); + err = mcp251xfd_chip_softreset(priv); if (err) goto out_chip_stop; - err = mcp25xxfd_chip_clock_init(priv); + err = mcp251xfd_chip_clock_init(priv); if (err) goto out_chip_stop; - err = mcp25xxfd_set_bittiming(priv); + err = mcp251xfd_set_bittiming(priv); if (err) goto out_chip_stop; - err = mcp25xxfd_chip_rx_int_enable(priv); + err = mcp251xfd_chip_rx_int_enable(priv); if (err) return err; - err = mcp25xxfd_chip_ecc_init(priv); + err = mcp251xfd_chip_ecc_init(priv); if (err) goto out_chip_stop; - mcp25xxfd_ring_init(priv); + mcp251xfd_ring_init(priv); - err = mcp25xxfd_chip_fifo_init(priv); + err = mcp251xfd_chip_fifo_init(priv); if (err) goto out_chip_stop; priv->can.state = CAN_STATE_ERROR_ACTIVE; - err = mcp25xxfd_chip_set_normal_mode(priv); + err = mcp251xfd_chip_set_normal_mode(priv); if (err) goto out_chip_stop; return 0; out_chip_stop: - mcp25xxfd_chip_stop(priv, CAN_STATE_STOPPED); + mcp251xfd_chip_stop(priv, CAN_STATE_STOPPED); return err; } -static int mcp25xxfd_set_mode(struct net_device *ndev, enum can_mode mode) +static int mcp251xfd_set_mode(struct net_device *ndev, enum can_mode mode) { - struct mcp25xxfd_priv *priv = netdev_priv(ndev); + struct mcp251xfd_priv *priv = netdev_priv(ndev); int err; switch (mode) { case CAN_MODE_START: - err = mcp25xxfd_chip_start(priv); + err = mcp251xfd_chip_start(priv); if (err) return err; - err = mcp25xxfd_chip_interrupts_enable(priv); + err = mcp251xfd_chip_interrupts_enable(priv); if (err) { - mcp25xxfd_chip_stop(priv, CAN_STATE_STOPPED); + mcp251xfd_chip_stop(priv, CAN_STATE_STOPPED); return err; } @@ -1094,30 +1094,30 @@ static int mcp25xxfd_set_mode(struct net_device *ndev, enum can_mode mode) return 0; } -static int __mcp25xxfd_get_berr_counter(const struct net_device *ndev, +static int __mcp251xfd_get_berr_counter(const struct net_device *ndev, struct can_berr_counter *bec) { - const struct mcp25xxfd_priv *priv = netdev_priv(ndev); + const struct mcp251xfd_priv *priv = netdev_priv(ndev); u32 trec; int err; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_TREC, &trec); + err = regmap_read(priv->map_reg, MCP251XFD_REG_TREC, &trec); if (err) return err; - if (trec & MCP25XXFD_REG_TREC_TXBO) + if (trec & MCP251XFD_REG_TREC_TXBO) bec->txerr = 256; else - bec->txerr = FIELD_GET(MCP25XXFD_REG_TREC_TEC_MASK, trec); - bec->rxerr = FIELD_GET(MCP25XXFD_REG_TREC_REC_MASK, trec); + bec->txerr = FIELD_GET(MCP251XFD_REG_TREC_TEC_MASK, trec); + bec->rxerr = FIELD_GET(MCP251XFD_REG_TREC_REC_MASK, trec); return 0; } -static int mcp25xxfd_get_berr_counter(const struct net_device *ndev, +static int mcp251xfd_get_berr_counter(const struct net_device *ndev, struct can_berr_counter *bec) { - const struct mcp25xxfd_priv *priv = netdev_priv(ndev); + const struct mcp251xfd_priv *priv = netdev_priv(ndev); /* Avoid waking up the controller if the interface is down */ if (!(ndev->flags & IFF_UP)) @@ -1131,22 +1131,22 @@ static int mcp25xxfd_get_berr_counter(const struct net_device *ndev, return 0; } - return __mcp25xxfd_get_berr_counter(ndev, bec); + return __mcp251xfd_get_berr_counter(ndev, bec); } -static int mcp25xxfd_check_tef_tail(const struct mcp25xxfd_priv *priv) +static int mcp251xfd_check_tef_tail(const struct mcp251xfd_priv *priv) { u8 tef_tail_chip, tef_tail; int err; - if (!IS_ENABLED(CONFIG_CAN_MCP25XXFD_SANITY)) + if (!IS_ENABLED(CONFIG_CAN_MCP251XFD_SANITY)) return 0; - err = mcp25xxfd_tef_tail_get_from_chip(priv, &tef_tail_chip); + err = mcp251xfd_tef_tail_get_from_chip(priv, &tef_tail_chip); if (err) return err; - tef_tail = mcp25xxfd_get_tef_tail(priv); + tef_tail = mcp251xfd_get_tef_tail(priv); if (tef_tail_chip != tef_tail) { netdev_err(priv->ndev, "TEF tail of chip (0x%02x) and ours (0x%08x) inconsistent.\n", @@ -1158,20 +1158,20 @@ static int mcp25xxfd_check_tef_tail(const struct mcp25xxfd_priv *priv) } static int -mcp25xxfd_check_rx_tail(const struct mcp25xxfd_priv *priv, - const struct mcp25xxfd_rx_ring *ring) +mcp251xfd_check_rx_tail(const struct mcp251xfd_priv *priv, + const struct mcp251xfd_rx_ring *ring) { u8 rx_tail_chip, rx_tail; int err; - if (!IS_ENABLED(CONFIG_CAN_MCP25XXFD_SANITY)) + if (!IS_ENABLED(CONFIG_CAN_MCP251XFD_SANITY)) return 0; - err = mcp25xxfd_rx_tail_get_from_chip(priv, ring, &rx_tail_chip); + err = mcp251xfd_rx_tail_get_from_chip(priv, ring, &rx_tail_chip); if (err) return err; - rx_tail = mcp25xxfd_get_rx_tail(ring); + rx_tail = mcp251xfd_get_rx_tail(ring); if (rx_tail_chip != rx_tail) { netdev_err(priv->ndev, "RX tail of chip (%d) and ours (%d) inconsistent.\n", @@ -1183,17 +1183,17 @@ mcp25xxfd_check_rx_tail(const struct mcp25xxfd_priv *priv, } static int -mcp25xxfd_handle_tefif_recover(const struct mcp25xxfd_priv *priv, const u32 seq) +mcp251xfd_handle_tefif_recover(const struct mcp251xfd_priv *priv, const u32 seq) { - const struct mcp25xxfd_tx_ring *tx_ring = priv->tx; + const struct mcp251xfd_tx_ring *tx_ring = priv->tx; u32 tef_sta; int err; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_TEFSTA, &tef_sta); + err = regmap_read(priv->map_reg, MCP251XFD_REG_TEFSTA, &tef_sta); if (err) return err; - if (tef_sta & MCP25XXFD_REG_TEFSTA_TEFOVIF) { + if (tef_sta & MCP251XFD_REG_TEFSTA_TEFOVIF) { netdev_err(priv->ndev, "Transmit Event FIFO buffer overflow.\n"); return -ENOBUFS; @@ -1201,8 +1201,8 @@ mcp25xxfd_handle_tefif_recover(const struct mcp25xxfd_priv *priv, const u32 seq) netdev_info(priv->ndev, "Transmit Event FIFO buffer %s. (seq=0x%08x, tef_tail=0x%08x, tef_head=0x%08x, tx_head=0x%08x)\n", - tef_sta & MCP25XXFD_REG_TEFSTA_TEFFIF ? - "full" : tef_sta & MCP25XXFD_REG_TEFSTA_TEFNEIF ? + tef_sta & MCP251XFD_REG_TEFSTA_TEFFIF ? + "full" : tef_sta & MCP251XFD_REG_TEFSTA_TEFNEIF ? "not empty" : "empty", seq, priv->tef.tail, priv->tef.head, tx_ring->head); @@ -1211,15 +1211,15 @@ mcp25xxfd_handle_tefif_recover(const struct mcp25xxfd_priv *priv, const u32 seq) } static int -mcp25xxfd_handle_tefif_one(struct mcp25xxfd_priv *priv, - const struct mcp25xxfd_hw_tef_obj *hw_tef_obj) +mcp251xfd_handle_tefif_one(struct mcp251xfd_priv *priv, + const struct mcp251xfd_hw_tef_obj *hw_tef_obj) { - struct mcp25xxfd_tx_ring *tx_ring = priv->tx; + struct mcp251xfd_tx_ring *tx_ring = priv->tx; struct net_device_stats *stats = &priv->ndev->stats; u32 seq, seq_masked, tef_tail_masked; int err; - seq = FIELD_GET(MCP25XXFD_OBJ_FLAGS_SEQ_MCP2518FD_MASK, + seq = FIELD_GET(MCP251XFD_OBJ_FLAGS_SEQ_MCP2518FD_MASK, hw_tef_obj->flags); /* Use the MCP2517FD mask on the MCP2518FD, too. We only @@ -1227,39 +1227,39 @@ mcp25xxfd_handle_tefif_one(struct mcp25xxfd_priv *priv, * net-yet-completed, i.e. old TEF objects. */ seq_masked = seq & - field_mask(MCP25XXFD_OBJ_FLAGS_SEQ_MCP2517FD_MASK); + field_mask(MCP251XFD_OBJ_FLAGS_SEQ_MCP2517FD_MASK); tef_tail_masked = priv->tef.tail & - field_mask(MCP25XXFD_OBJ_FLAGS_SEQ_MCP2517FD_MASK); + field_mask(MCP251XFD_OBJ_FLAGS_SEQ_MCP2517FD_MASK); if (seq_masked != tef_tail_masked) - return mcp25xxfd_handle_tefif_recover(priv, seq); + return mcp251xfd_handle_tefif_recover(priv, seq); stats->tx_bytes += can_rx_offload_get_echo_skb(&priv->offload, - mcp25xxfd_get_tef_tail(priv), + mcp251xfd_get_tef_tail(priv), hw_tef_obj->ts); stats->tx_packets++; /* finally increment the TEF pointer */ - err = regmap_update_bits(priv->map_reg, MCP25XXFD_REG_TEFCON, + err = regmap_update_bits(priv->map_reg, MCP251XFD_REG_TEFCON, GENMASK(15, 8), - MCP25XXFD_REG_TEFCON_UINC); + MCP251XFD_REG_TEFCON_UINC); if (err) return err; priv->tef.tail++; tx_ring->tail++; - return mcp25xxfd_check_tef_tail(priv); + return mcp251xfd_check_tef_tail(priv); } -static int mcp25xxfd_tef_ring_update(struct mcp25xxfd_priv *priv) +static int mcp251xfd_tef_ring_update(struct mcp251xfd_priv *priv) { - const struct mcp25xxfd_tx_ring *tx_ring = priv->tx; + const struct mcp251xfd_tx_ring *tx_ring = priv->tx; unsigned int new_head; u8 chip_tx_tail; int err; - err = mcp25xxfd_tx_tail_get_from_chip(priv, &chip_tx_tail); + err = mcp251xfd_tx_tail_get_from_chip(priv, &chip_tx_tail); if (err) return err; @@ -1273,17 +1273,17 @@ static int mcp25xxfd_tef_ring_update(struct mcp25xxfd_priv *priv) /* ... but it cannot exceed the TX head. */ priv->tef.head = min(new_head, tx_ring->head); - return mcp25xxfd_check_tef_tail(priv); + return mcp251xfd_check_tef_tail(priv); } static inline int -mcp25xxfd_tef_obj_read(const struct mcp25xxfd_priv *priv, - struct mcp25xxfd_hw_tef_obj *hw_tef_obj, +mcp251xfd_tef_obj_read(const struct mcp251xfd_priv *priv, + struct mcp251xfd_hw_tef_obj *hw_tef_obj, const u8 offset, const u8 len) { - const struct mcp25xxfd_tx_ring *tx_ring = priv->tx; + const struct mcp251xfd_tx_ring *tx_ring = priv->tx; - if (IS_ENABLED(CONFIG_CAN_MCP25XXFD_SANITY) && + if (IS_ENABLED(CONFIG_CAN_MCP251XFD_SANITY) && (offset > tx_ring->obj_num || len > tx_ring->obj_num || offset + len > tx_ring->obj_num)) { @@ -1294,36 +1294,36 @@ mcp25xxfd_tef_obj_read(const struct mcp25xxfd_priv *priv, } return regmap_bulk_read(priv->map_rx, - mcp25xxfd_get_tef_obj_addr(offset), + mcp251xfd_get_tef_obj_addr(offset), hw_tef_obj, sizeof(*hw_tef_obj) / sizeof(u32) * len); } -static int mcp25xxfd_handle_tefif(struct mcp25xxfd_priv *priv) +static int mcp251xfd_handle_tefif(struct mcp251xfd_priv *priv) { - struct mcp25xxfd_hw_tef_obj hw_tef_obj[MCP25XXFD_TX_OBJ_NUM_MAX]; + struct mcp251xfd_hw_tef_obj hw_tef_obj[MCP251XFD_TX_OBJ_NUM_MAX]; u8 tef_tail, len, l; int err, i; - err = mcp25xxfd_tef_ring_update(priv); + err = mcp251xfd_tef_ring_update(priv); if (err) return err; - tef_tail = mcp25xxfd_get_tef_tail(priv); - len = mcp25xxfd_get_tef_len(priv); - l = mcp25xxfd_get_tef_linear_len(priv); - err = mcp25xxfd_tef_obj_read(priv, hw_tef_obj, tef_tail, l); + tef_tail = mcp251xfd_get_tef_tail(priv); + len = mcp251xfd_get_tef_len(priv); + l = mcp251xfd_get_tef_linear_len(priv); + err = mcp251xfd_tef_obj_read(priv, hw_tef_obj, tef_tail, l); if (err) return err; if (l < len) { - err = mcp25xxfd_tef_obj_read(priv, &hw_tef_obj[l], 0, len - l); + err = mcp251xfd_tef_obj_read(priv, &hw_tef_obj[l], 0, len - l); if (err) return err; } for (i = 0; i < len; i++) { - err = mcp25xxfd_handle_tefif_one(priv, &hw_tef_obj[i]); + err = mcp251xfd_handle_tefif_one(priv, &hw_tef_obj[i]); /* -EAGAIN means the Sequence Number in the TEF * doesn't match our tef_tail. This can happen if we * read the TEF objects too early. Leave loop let the @@ -1336,9 +1336,9 @@ static int mcp25xxfd_handle_tefif(struct mcp25xxfd_priv *priv) } out_netif_wake_queue: - mcp25xxfd_ecc_tefif_successful(priv); + mcp251xfd_ecc_tefif_successful(priv); - if (mcp25xxfd_get_tx_free(priv->tx)) { + if (mcp251xfd_get_tx_free(priv->tx)) { /* Make sure that anybody stopping the queue after * this sees the new tx_ring->tail. */ @@ -1350,14 +1350,14 @@ static int mcp25xxfd_handle_tefif(struct mcp25xxfd_priv *priv) } static int -mcp25xxfd_rx_ring_update(const struct mcp25xxfd_priv *priv, - struct mcp25xxfd_rx_ring *ring) +mcp251xfd_rx_ring_update(const struct mcp251xfd_priv *priv, + struct mcp251xfd_rx_ring *ring) { u32 new_head; u8 chip_rx_head; int err; - err = mcp25xxfd_rx_head_get_from_chip(priv, ring, &chip_rx_head); + err = mcp251xfd_rx_head_get_from_chip(priv, ring, &chip_rx_head); if (err) return err; @@ -1370,47 +1370,47 @@ mcp25xxfd_rx_ring_update(const struct mcp25xxfd_priv *priv, ring->head = new_head; - return mcp25xxfd_check_rx_tail(priv, ring); + return mcp251xfd_check_rx_tail(priv, ring); } static void -mcp25xxfd_hw_rx_obj_to_skb(const struct mcp25xxfd_priv *priv, - const struct mcp25xxfd_hw_rx_obj_canfd *hw_rx_obj, +mcp251xfd_hw_rx_obj_to_skb(const struct mcp251xfd_priv *priv, + const struct mcp251xfd_hw_rx_obj_canfd *hw_rx_obj, struct sk_buff *skb) { struct canfd_frame *cfd = (struct canfd_frame *)skb->data; - if (hw_rx_obj->flags & MCP25XXFD_OBJ_FLAGS_IDE) { + if (hw_rx_obj->flags & MCP251XFD_OBJ_FLAGS_IDE) { u32 sid, eid; - eid = FIELD_GET(MCP25XXFD_OBJ_ID_EID_MASK, hw_rx_obj->id); - sid = FIELD_GET(MCP25XXFD_OBJ_ID_SID_MASK, hw_rx_obj->id); + eid = FIELD_GET(MCP251XFD_OBJ_ID_EID_MASK, hw_rx_obj->id); + sid = FIELD_GET(MCP251XFD_OBJ_ID_SID_MASK, hw_rx_obj->id); cfd->can_id = CAN_EFF_FLAG | - FIELD_PREP(MCP25XXFD_REG_FRAME_EFF_EID_MASK, eid) | - FIELD_PREP(MCP25XXFD_REG_FRAME_EFF_SID_MASK, sid); + FIELD_PREP(MCP251XFD_REG_FRAME_EFF_EID_MASK, eid) | + FIELD_PREP(MCP251XFD_REG_FRAME_EFF_SID_MASK, sid); } else { - cfd->can_id = FIELD_GET(MCP25XXFD_OBJ_ID_SID_MASK, + cfd->can_id = FIELD_GET(MCP251XFD_OBJ_ID_SID_MASK, hw_rx_obj->id); } /* CANFD */ - if (hw_rx_obj->flags & MCP25XXFD_OBJ_FLAGS_FDF) { + if (hw_rx_obj->flags & MCP251XFD_OBJ_FLAGS_FDF) { u8 dlc; - if (hw_rx_obj->flags & MCP25XXFD_OBJ_FLAGS_ESI) + if (hw_rx_obj->flags & MCP251XFD_OBJ_FLAGS_ESI) cfd->flags |= CANFD_ESI; - if (hw_rx_obj->flags & MCP25XXFD_OBJ_FLAGS_BRS) + if (hw_rx_obj->flags & MCP251XFD_OBJ_FLAGS_BRS) cfd->flags |= CANFD_BRS; - dlc = FIELD_GET(MCP25XXFD_OBJ_FLAGS_DLC, hw_rx_obj->flags); + dlc = FIELD_GET(MCP251XFD_OBJ_FLAGS_DLC, hw_rx_obj->flags); cfd->len = can_dlc2len(get_canfd_dlc(dlc)); } else { - if (hw_rx_obj->flags & MCP25XXFD_OBJ_FLAGS_RTR) + if (hw_rx_obj->flags & MCP251XFD_OBJ_FLAGS_RTR) cfd->can_id |= CAN_RTR_FLAG; - cfd->len = get_can_dlc(FIELD_GET(MCP25XXFD_OBJ_FLAGS_DLC, + cfd->len = get_can_dlc(FIELD_GET(MCP251XFD_OBJ_FLAGS_DLC, hw_rx_obj->flags)); } @@ -1418,16 +1418,16 @@ mcp25xxfd_hw_rx_obj_to_skb(const struct mcp25xxfd_priv *priv, } static int -mcp25xxfd_handle_rxif_one(struct mcp25xxfd_priv *priv, - struct mcp25xxfd_rx_ring *ring, - const struct mcp25xxfd_hw_rx_obj_canfd *hw_rx_obj) +mcp251xfd_handle_rxif_one(struct mcp251xfd_priv *priv, + struct mcp251xfd_rx_ring *ring, + const struct mcp251xfd_hw_rx_obj_canfd *hw_rx_obj) { struct net_device_stats *stats = &priv->ndev->stats; struct sk_buff *skb; struct canfd_frame *cfd; int err; - if (hw_rx_obj->flags & MCP25XXFD_OBJ_FLAGS_FDF) + if (hw_rx_obj->flags & MCP251XFD_OBJ_FLAGS_FDF) skb = alloc_canfd_skb(priv->ndev, &cfd); else skb = alloc_can_skb(priv->ndev, (struct can_frame **)&cfd); @@ -1437,7 +1437,7 @@ mcp25xxfd_handle_rxif_one(struct mcp25xxfd_priv *priv, return 0; } - mcp25xxfd_hw_rx_obj_to_skb(priv, hw_rx_obj, skb); + mcp251xfd_hw_rx_obj_to_skb(priv, hw_rx_obj, skb); err = can_rx_offload_queue_sorted(&priv->offload, skb, hw_rx_obj->ts); if (err) stats->rx_fifo_errors++; @@ -1446,21 +1446,21 @@ mcp25xxfd_handle_rxif_one(struct mcp25xxfd_priv *priv, /* finally increment the RX pointer */ return regmap_update_bits(priv->map_reg, - MCP25XXFD_REG_FIFOCON(ring->fifo_nr), + MCP251XFD_REG_FIFOCON(ring->fifo_nr), GENMASK(15, 8), - MCP25XXFD_REG_FIFOCON_UINC); + MCP251XFD_REG_FIFOCON_UINC); } static inline int -mcp25xxfd_rx_obj_read(const struct mcp25xxfd_priv *priv, - const struct mcp25xxfd_rx_ring *ring, - struct mcp25xxfd_hw_rx_obj_canfd *hw_rx_obj, +mcp251xfd_rx_obj_read(const struct mcp251xfd_priv *priv, + const struct mcp251xfd_rx_ring *ring, + struct mcp251xfd_hw_rx_obj_canfd *hw_rx_obj, const u8 offset, const u8 len) { int err; err = regmap_bulk_read(priv->map_rx, - mcp25xxfd_get_rx_obj_addr(ring, offset), + mcp251xfd_get_rx_obj_addr(ring, offset), hw_rx_obj, len * ring->obj_size / sizeof(u32)); @@ -1468,27 +1468,27 @@ mcp25xxfd_rx_obj_read(const struct mcp25xxfd_priv *priv, } static int -mcp25xxfd_handle_rxif_ring(struct mcp25xxfd_priv *priv, - struct mcp25xxfd_rx_ring *ring) +mcp251xfd_handle_rxif_ring(struct mcp251xfd_priv *priv, + struct mcp251xfd_rx_ring *ring) { - struct mcp25xxfd_hw_rx_obj_canfd *hw_rx_obj = ring->obj; + struct mcp251xfd_hw_rx_obj_canfd *hw_rx_obj = ring->obj; u8 rx_tail, len; int err, i; - err = mcp25xxfd_rx_ring_update(priv, ring); + err = mcp251xfd_rx_ring_update(priv, ring); if (err) return err; - while ((len = mcp25xxfd_get_rx_linear_len(ring))) { - rx_tail = mcp25xxfd_get_rx_tail(ring); + while ((len = mcp251xfd_get_rx_linear_len(ring))) { + rx_tail = mcp251xfd_get_rx_tail(ring); - err = mcp25xxfd_rx_obj_read(priv, ring, hw_rx_obj, + err = mcp251xfd_rx_obj_read(priv, ring, hw_rx_obj, rx_tail, len); if (err) return err; for (i = 0; i < len; i++) { - err = mcp25xxfd_handle_rxif_one(priv, ring, + err = mcp251xfd_handle_rxif_one(priv, ring, (void *)hw_rx_obj + i * ring->obj_size); if (err) @@ -1499,13 +1499,13 @@ mcp25xxfd_handle_rxif_ring(struct mcp25xxfd_priv *priv, return 0; } -static int mcp25xxfd_handle_rxif(struct mcp25xxfd_priv *priv) +static int mcp251xfd_handle_rxif(struct mcp251xfd_priv *priv) { - struct mcp25xxfd_rx_ring *ring; + struct mcp251xfd_rx_ring *ring; int err, n; - mcp25xxfd_for_each_rx_ring(priv, ring, n) { - err = mcp25xxfd_handle_rxif_ring(priv, ring); + mcp251xfd_for_each_rx_ring(priv, ring, n) { + err = mcp251xfd_handle_rxif_ring(priv, ring); if (err) return err; } @@ -1513,29 +1513,29 @@ static int mcp25xxfd_handle_rxif(struct mcp25xxfd_priv *priv) return 0; } -static inline int mcp25xxfd_get_timestamp(const struct mcp25xxfd_priv *priv, +static inline int mcp251xfd_get_timestamp(const struct mcp251xfd_priv *priv, u32 *timestamp) { - return regmap_read(priv->map_reg, MCP25XXFD_REG_TBC, timestamp); + return regmap_read(priv->map_reg, MCP251XFD_REG_TBC, timestamp); } static struct sk_buff * -mcp25xxfd_alloc_can_err_skb(const struct mcp25xxfd_priv *priv, +mcp251xfd_alloc_can_err_skb(const struct mcp251xfd_priv *priv, struct can_frame **cf, u32 *timestamp) { int err; - err = mcp25xxfd_get_timestamp(priv, timestamp); + err = mcp251xfd_get_timestamp(priv, timestamp); if (err) return NULL; return alloc_can_err_skb(priv->ndev, cf); } -static int mcp25xxfd_handle_rxovif(struct mcp25xxfd_priv *priv) +static int mcp251xfd_handle_rxovif(struct mcp251xfd_priv *priv) { struct net_device_stats *stats = &priv->ndev->stats; - struct mcp25xxfd_rx_ring *ring; + struct mcp251xfd_rx_ring *ring; struct sk_buff *skb; struct can_frame *cf; u32 timestamp, rxovif; @@ -1544,16 +1544,16 @@ static int mcp25xxfd_handle_rxovif(struct mcp25xxfd_priv *priv) stats->rx_over_errors++; stats->rx_errors++; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_RXOVIF, &rxovif); + err = regmap_read(priv->map_reg, MCP251XFD_REG_RXOVIF, &rxovif); if (err) return err; - mcp25xxfd_for_each_rx_ring(priv, ring, i) { + mcp251xfd_for_each_rx_ring(priv, ring, i) { if (!(rxovif & BIT(ring->fifo_nr))) continue; /* If SERRIF is active, there was a RX MAB overflow. */ - if (priv->regs_status.intf & MCP25XXFD_REG_INT_SERRIF) { + if (priv->regs_status.intf & MCP251XFD_REG_INT_SERRIF) { netdev_info(priv->ndev, "RX-%d: MAB overflow detected.\n", ring->nr); @@ -1563,14 +1563,14 @@ static int mcp25xxfd_handle_rxovif(struct mcp25xxfd_priv *priv) } err = regmap_update_bits(priv->map_reg, - MCP25XXFD_REG_FIFOSTA(ring->fifo_nr), - MCP25XXFD_REG_FIFOSTA_RXOVIF, + MCP251XFD_REG_FIFOSTA(ring->fifo_nr), + MCP251XFD_REG_FIFOSTA_RXOVIF, 0x0); if (err) return err; } - skb = mcp25xxfd_alloc_can_err_skb(priv, &cf, ×tamp); + skb = mcp251xfd_alloc_can_err_skb(priv, &cf, ×tamp); if (!skb) return 0; @@ -1584,14 +1584,14 @@ static int mcp25xxfd_handle_rxovif(struct mcp25xxfd_priv *priv) return 0; } -static int mcp25xxfd_handle_txatif(struct mcp25xxfd_priv *priv) +static int mcp251xfd_handle_txatif(struct mcp251xfd_priv *priv) { netdev_info(priv->ndev, "%s\n", __func__); return 0; } -static int mcp25xxfd_handle_ivmif(struct mcp25xxfd_priv *priv) +static int mcp251xfd_handle_ivmif(struct mcp251xfd_priv *priv) { struct net_device_stats *stats = &priv->ndev->stats; u32 bdiag1, timestamp; @@ -1599,18 +1599,18 @@ static int mcp25xxfd_handle_ivmif(struct mcp25xxfd_priv *priv) struct can_frame *cf = NULL; int err; - err = mcp25xxfd_get_timestamp(priv, ×tamp); + err = mcp251xfd_get_timestamp(priv, ×tamp); if (err) return err; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_BDIAG1, &bdiag1); + err = regmap_read(priv->map_reg, MCP251XFD_REG_BDIAG1, &bdiag1); if (err) return err; /* Write 0s to clear error bits, don't write 1s to non active * bits, as they will be set. */ - err = regmap_write(priv->map_reg, MCP25XXFD_REG_BDIAG1, 0x0); + err = regmap_write(priv->map_reg, MCP251XFD_REG_BDIAG1, 0x0); if (err) return err; @@ -1621,29 +1621,29 @@ static int mcp25xxfd_handle_ivmif(struct mcp25xxfd_priv *priv) cf->can_id |= CAN_ERR_PROT | CAN_ERR_BUSERROR; /* Controller misconfiguration */ - if (WARN_ON(bdiag1 & MCP25XXFD_REG_BDIAG1_DLCMM)) + if (WARN_ON(bdiag1 & MCP251XFD_REG_BDIAG1_DLCMM)) netdev_err(priv->ndev, "recv'd DLC is larger than PLSIZE of FIFO element."); /* RX errors */ - if (bdiag1 & (MCP25XXFD_REG_BDIAG1_DCRCERR | - MCP25XXFD_REG_BDIAG1_NCRCERR)) { + if (bdiag1 & (MCP251XFD_REG_BDIAG1_DCRCERR | + MCP251XFD_REG_BDIAG1_NCRCERR)) { netdev_dbg(priv->ndev, "CRC error\n"); stats->rx_errors++; if (cf) cf->data[3] |= CAN_ERR_PROT_LOC_CRC_SEQ; } - if (bdiag1 & (MCP25XXFD_REG_BDIAG1_DSTUFERR | - MCP25XXFD_REG_BDIAG1_NSTUFERR)) { + if (bdiag1 & (MCP251XFD_REG_BDIAG1_DSTUFERR | + MCP251XFD_REG_BDIAG1_NSTUFERR)) { netdev_dbg(priv->ndev, "Stuff error\n"); stats->rx_errors++; if (cf) cf->data[2] |= CAN_ERR_PROT_STUFF; } - if (bdiag1 & (MCP25XXFD_REG_BDIAG1_DFORMERR | - MCP25XXFD_REG_BDIAG1_NFORMERR)) { + if (bdiag1 & (MCP251XFD_REG_BDIAG1_DFORMERR | + MCP251XFD_REG_BDIAG1_NFORMERR)) { netdev_dbg(priv->ndev, "Format error\n"); stats->rx_errors++; @@ -1652,7 +1652,7 @@ static int mcp25xxfd_handle_ivmif(struct mcp25xxfd_priv *priv) } /* TX errors */ - if (bdiag1 & MCP25XXFD_REG_BDIAG1_NACKERR) { + if (bdiag1 & MCP251XFD_REG_BDIAG1_NACKERR) { netdev_dbg(priv->ndev, "NACK error\n"); stats->tx_errors++; @@ -1661,16 +1661,16 @@ static int mcp25xxfd_handle_ivmif(struct mcp25xxfd_priv *priv) cf->data[2] |= CAN_ERR_PROT_TX; } } - if (bdiag1 & (MCP25XXFD_REG_BDIAG1_DBIT1ERR | - MCP25XXFD_REG_BDIAG1_NBIT1ERR)) { + if (bdiag1 & (MCP251XFD_REG_BDIAG1_DBIT1ERR | + MCP251XFD_REG_BDIAG1_NBIT1ERR)) { netdev_dbg(priv->ndev, "Bit1 error\n"); stats->tx_errors++; if (cf) cf->data[2] |= CAN_ERR_PROT_TX | CAN_ERR_PROT_BIT1; } - if (bdiag1 & (MCP25XXFD_REG_BDIAG1_DBIT0ERR | - MCP25XXFD_REG_BDIAG1_NBIT0ERR)) { + if (bdiag1 & (MCP251XFD_REG_BDIAG1_DBIT0ERR | + MCP251XFD_REG_BDIAG1_NBIT0ERR)) { netdev_dbg(priv->ndev, "Bit0 error\n"); stats->tx_errors++; @@ -1688,7 +1688,7 @@ static int mcp25xxfd_handle_ivmif(struct mcp25xxfd_priv *priv) return 0; } -static int mcp25xxfd_handle_cerrif(struct mcp25xxfd_priv *priv) +static int mcp251xfd_handle_cerrif(struct mcp251xfd_priv *priv) { struct net_device_stats *stats = &priv->ndev->stats; struct sk_buff *skb; @@ -1697,22 +1697,22 @@ static int mcp25xxfd_handle_cerrif(struct mcp25xxfd_priv *priv) u32 trec, timestamp; int err; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_TREC, &trec); + err = regmap_read(priv->map_reg, MCP251XFD_REG_TREC, &trec); if (err) return err; - if (trec & MCP25XXFD_REG_TREC_TXBO) + if (trec & MCP251XFD_REG_TREC_TXBO) tx_state = CAN_STATE_BUS_OFF; - else if (trec & MCP25XXFD_REG_TREC_TXBP) + else if (trec & MCP251XFD_REG_TREC_TXBP) tx_state = CAN_STATE_ERROR_PASSIVE; - else if (trec & MCP25XXFD_REG_TREC_TXWARN) + else if (trec & MCP251XFD_REG_TREC_TXWARN) tx_state = CAN_STATE_ERROR_WARNING; else tx_state = CAN_STATE_ERROR_ACTIVE; - if (trec & MCP25XXFD_REG_TREC_RXBP) + if (trec & MCP251XFD_REG_TREC_RXBP) rx_state = CAN_STATE_ERROR_PASSIVE; - else if (trec & MCP25XXFD_REG_TREC_RXWARN) + else if (trec & MCP251XFD_REG_TREC_RXWARN) rx_state = CAN_STATE_ERROR_WARNING; else rx_state = CAN_STATE_ERROR_ACTIVE; @@ -1724,7 +1724,7 @@ static int mcp25xxfd_handle_cerrif(struct mcp25xxfd_priv *priv) /* The skb allocation might fail, but can_change_state() * handles cf == NULL. */ - skb = mcp25xxfd_alloc_can_err_skb(priv, &cf, ×tamp); + skb = mcp251xfd_alloc_can_err_skb(priv, &cf, ×tamp); can_change_state(priv->ndev, cf, tx_state, rx_state); if (new_state == CAN_STATE_BUS_OFF) { @@ -1733,11 +1733,11 @@ static int mcp25xxfd_handle_cerrif(struct mcp25xxfd_priv *priv) * userspace, if do_get_berr_counter() is called while * the chip is in Bus Off. */ - err = __mcp25xxfd_get_berr_counter(priv->ndev, &priv->bec); + err = __mcp251xfd_get_berr_counter(priv->ndev, &priv->bec); if (err) return err; - mcp25xxfd_chip_stop(priv, CAN_STATE_BUS_OFF); + mcp251xfd_chip_stop(priv, CAN_STATE_BUS_OFF); can_bus_off(priv->ndev); } @@ -1747,7 +1747,7 @@ static int mcp25xxfd_handle_cerrif(struct mcp25xxfd_priv *priv) if (new_state != CAN_STATE_BUS_OFF) { struct can_berr_counter bec; - err = mcp25xxfd_get_berr_counter(priv->ndev, &bec); + err = mcp251xfd_get_berr_counter(priv->ndev, &bec); if (err) return err; cf->data[6] = bec.txerr; @@ -1762,20 +1762,20 @@ static int mcp25xxfd_handle_cerrif(struct mcp25xxfd_priv *priv) } static int -mcp25xxfd_handle_modif(const struct mcp25xxfd_priv *priv, bool *set_normal_mode) +mcp251xfd_handle_modif(const struct mcp251xfd_priv *priv, bool *set_normal_mode) { - const u8 mode_reference = mcp25xxfd_get_normal_mode(priv); + const u8 mode_reference = mcp251xfd_get_normal_mode(priv); u8 mode; int err; - err = mcp25xxfd_chip_get_mode(priv, &mode); + err = mcp251xfd_chip_get_mode(priv, &mode); if (err) return err; if (mode == mode_reference) { netdev_dbg(priv->ndev, "Controller changed into %s Mode (%u).\n", - mcp25xxfd_get_mode_str(mode), mode); + mcp251xfd_get_mode_str(mode), mode); return 0; } @@ -1789,16 +1789,16 @@ mcp25xxfd_handle_modif(const struct mcp25xxfd_priv *priv, bool *set_normal_mode) * first. When polling this bit we see that it will transition * to Restricted Operation Mode shortly after. */ - if ((priv->devtype_data.quirks & MCP25XXFD_QUIRK_MAB_NO_WARN) && - (mode == MCP25XXFD_REG_CON_MODE_RESTRICTED || - mode == MCP25XXFD_REG_CON_MODE_LISTENONLY)) + if ((priv->devtype_data.quirks & MCP251XFD_QUIRK_MAB_NO_WARN) && + (mode == MCP251XFD_REG_CON_MODE_RESTRICTED || + mode == MCP251XFD_REG_CON_MODE_LISTENONLY)) netdev_dbg(priv->ndev, "Controller changed into %s Mode (%u).\n", - mcp25xxfd_get_mode_str(mode), mode); + mcp251xfd_get_mode_str(mode), mode); else netdev_err(priv->ndev, "Controller changed into %s Mode (%u).\n", - mcp25xxfd_get_mode_str(mode), mode); + mcp251xfd_get_mode_str(mode), mode); /* After the application requests Normal mode, the Controller * will automatically attempt to retransmit the message that @@ -1806,19 +1806,19 @@ mcp25xxfd_handle_modif(const struct mcp25xxfd_priv *priv, bool *set_normal_mode) * * However, if there is an ECC error in the TX-RAM, we first * have to reload the tx-object before requesting Normal - * mode. This is done later in mcp25xxfd_handle_eccif(). + * mode. This is done later in mcp251xfd_handle_eccif(). */ - if (priv->regs_status.intf & MCP25XXFD_REG_INT_ECCIF) { + if (priv->regs_status.intf & MCP251XFD_REG_INT_ECCIF) { *set_normal_mode = true; return 0; } - return mcp25xxfd_chip_set_normal_mode_nowait(priv); + return mcp251xfd_chip_set_normal_mode_nowait(priv); } -static int mcp25xxfd_handle_serrif(struct mcp25xxfd_priv *priv) +static int mcp251xfd_handle_serrif(struct mcp251xfd_priv *priv) { - struct mcp25xxfd_ecc *ecc = &priv->ecc; + struct mcp251xfd_ecc *ecc = &priv->ecc; struct net_device_stats *stats = &priv->ndev->stats; bool handled = false; @@ -1844,19 +1844,19 @@ static int mcp25xxfd_handle_serrif(struct mcp25xxfd_priv *priv) * * Treat all as a known system errors.. */ - if ((priv->regs_status.intf & MCP25XXFD_REG_INT_MODIF && - priv->regs_status.intf & MCP25XXFD_REG_INT_IVMIF) || - priv->regs_status.intf & MCP25XXFD_REG_INT_ECCIF || + if ((priv->regs_status.intf & MCP251XFD_REG_INT_MODIF && + priv->regs_status.intf & MCP251XFD_REG_INT_IVMIF) || + priv->regs_status.intf & MCP251XFD_REG_INT_ECCIF || ecc->cnt) { const char *msg; - if (priv->regs_status.intf & MCP25XXFD_REG_INT_ECCIF || + if (priv->regs_status.intf & MCP251XFD_REG_INT_ECCIF || ecc->cnt) msg = "TX MAB underflow due to ECC error detected."; else msg = "TX MAB underflow detected."; - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_MAB_NO_WARN) + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_MAB_NO_WARN) netdev_dbg(priv->ndev, "%s\n", msg); else netdev_info(priv->ndev, "%s\n", msg); @@ -1880,8 +1880,8 @@ static int mcp25xxfd_handle_serrif(struct mcp25xxfd_priv *priv) * * Treat all as a known system errors.. */ - if (priv->regs_status.intf & MCP25XXFD_REG_INT_RXOVIF || - priv->regs_status.intf & MCP25XXFD_REG_INT_RXIF) { + if (priv->regs_status.intf & MCP251XFD_REG_INT_RXOVIF || + priv->regs_status.intf & MCP251XFD_REG_INT_RXIF) { stats->rx_dropped++; handled = true; } @@ -1895,22 +1895,22 @@ static int mcp25xxfd_handle_serrif(struct mcp25xxfd_priv *priv) } static int -mcp25xxfd_handle_eccif_recover(struct mcp25xxfd_priv *priv, u8 nr) +mcp251xfd_handle_eccif_recover(struct mcp251xfd_priv *priv, u8 nr) { - struct mcp25xxfd_tx_ring *tx_ring = priv->tx; - struct mcp25xxfd_ecc *ecc = &priv->ecc; - struct mcp25xxfd_tx_obj *tx_obj; + struct mcp251xfd_tx_ring *tx_ring = priv->tx; + struct mcp251xfd_ecc *ecc = &priv->ecc; + struct mcp251xfd_tx_obj *tx_obj; u8 chip_tx_tail, tx_tail, offset; u16 addr; int err; - addr = FIELD_GET(MCP25XXFD_REG_ECCSTAT_ERRADDR_MASK, ecc->ecc_stat); + addr = FIELD_GET(MCP251XFD_REG_ECCSTAT_ERRADDR_MASK, ecc->ecc_stat); - err = mcp25xxfd_tx_tail_get_from_chip(priv, &chip_tx_tail); + err = mcp251xfd_tx_tail_get_from_chip(priv, &chip_tx_tail); if (err) return err; - tx_tail = mcp25xxfd_get_tx_tail(tx_ring); + tx_tail = mcp251xfd_get_tx_tail(tx_ring); offset = (nr - chip_tx_tail) & (tx_ring->obj_num - 1); /* Bail out if one of the following is met: @@ -1919,7 +1919,7 @@ mcp25xxfd_handle_eccif_recover(struct mcp25xxfd_priv *priv, u8 nr) * - for mcp2518fd: offset not 0 or 1 */ if (chip_tx_tail != tx_tail || - !(offset == 0 || (offset == 1 && mcp25xxfd_is_2518(priv)))) { + !(offset == 0 || (offset == 1 && mcp251xfd_is_2518(priv)))) { netdev_err(priv->ndev, "ECC Error information inconsistent (addr=0x%04x, nr=%d, tx_tail=0x%08x(%d), chip_tx_tail=%d, offset=%d).\n", addr, nr, tx_ring->tail, tx_tail, chip_tx_tail, @@ -1929,7 +1929,7 @@ mcp25xxfd_handle_eccif_recover(struct mcp25xxfd_priv *priv, u8 nr) netdev_info(priv->ndev, "Recovering %s ECC Error at address 0x%04x (in TX-RAM, tx_obj=%d, tx_tail=0x%08x(%d), offset=%d).\n", - ecc->ecc_stat & MCP25XXFD_REG_ECCSTAT_SECIF ? + ecc->ecc_stat & MCP251XFD_REG_ECCSTAT_SECIF ? "Single" : "Double", addr, nr, tx_ring->tail, tx_tail, offset); @@ -1940,13 +1940,13 @@ mcp25xxfd_handle_eccif_recover(struct mcp25xxfd_priv *priv, u8 nr) return err; /* ... and trigger retransmit */ - return mcp25xxfd_chip_set_normal_mode(priv); + return mcp251xfd_chip_set_normal_mode(priv); } static int -mcp25xxfd_handle_eccif(struct mcp25xxfd_priv *priv, bool set_normal_mode) +mcp251xfd_handle_eccif(struct mcp251xfd_priv *priv, bool set_normal_mode) { - struct mcp25xxfd_ecc *ecc = &priv->ecc; + struct mcp251xfd_ecc *ecc = &priv->ecc; const char *msg; bool in_tx_ram; u32 ecc_stat; @@ -1954,18 +1954,18 @@ mcp25xxfd_handle_eccif(struct mcp25xxfd_priv *priv, bool set_normal_mode) u8 nr; int err; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_ECCSTAT, &ecc_stat); + err = regmap_read(priv->map_reg, MCP251XFD_REG_ECCSTAT, &ecc_stat); if (err) return err; - err = regmap_update_bits(priv->map_reg, MCP25XXFD_REG_ECCSTAT, - MCP25XXFD_REG_ECCSTAT_IF_MASK, ~ecc_stat); + err = regmap_update_bits(priv->map_reg, MCP251XFD_REG_ECCSTAT, + MCP251XFD_REG_ECCSTAT_IF_MASK, ~ecc_stat); if (err) return err; /* Check if ECC error occurred in TX-RAM */ - addr = FIELD_GET(MCP25XXFD_REG_ECCSTAT_ERRADDR_MASK, ecc_stat); - err = mcp25xxfd_get_tx_nr_by_addr(priv->tx, &nr, addr); + addr = FIELD_GET(MCP251XFD_REG_ECCSTAT_ERRADDR_MASK, ecc_stat); + err = mcp251xfd_get_tx_nr_by_addr(priv->tx, &nr, addr); if (!err) in_tx_ram = true; else if (err == -ENOENT) @@ -1973,9 +1973,21 @@ mcp25xxfd_handle_eccif(struct mcp25xxfd_priv *priv, bool set_normal_mode) else return err; - if (ecc_stat & MCP25XXFD_REG_ECCSTAT_SECIF) - msg = "Single ECC Error corrected at address"; - else if (ecc_stat & MCP25XXFD_REG_ECCSTAT_DEDIF) + /* Errata Reference: + * mcp2517fd: DS80000789B, mcp2518fd: DS80000792C 2. + * + * ECC single error correction does not work in all cases: + * + * Fix/Work Around: + * Enable single error correction and double error detection + * interrupts by setting SECIE and DEDIE. Handle SECIF as a + * detection interrupt and do not rely on the error + * correction. Instead, handle both interrupts as a + * notification that the RAM word at ERRADDR was corrupted. + */ + if (ecc_stat & MCP251XFD_REG_ECCSTAT_SECIF) + msg = "Single ECC Error detected at address"; + else if (ecc_stat & MCP251XFD_REG_ECCSTAT_DEDIF) msg = "Double ECC Error detected at address"; else return -EINVAL; @@ -1983,12 +1995,7 @@ mcp25xxfd_handle_eccif(struct mcp25xxfd_priv *priv, bool set_normal_mode) if (!in_tx_ram) { ecc->ecc_stat = 0; - if (ecc_stat & MCP25XXFD_REG_ECCSTAT_SECIF) - netdev_info(priv->ndev, "%s 0x%04x.\n", - msg, addr); - else - netdev_notice(priv->ndev, "%s 0x%04x.\n", - msg, addr); + netdev_notice(priv->ndev, "%s 0x%04x.\n", msg, addr); } else { /* Re-occurring error? */ if (ecc->ecc_stat == ecc_stat) { @@ -2002,57 +2009,57 @@ mcp25xxfd_handle_eccif(struct mcp25xxfd_priv *priv, bool set_normal_mode) "%s 0x%04x (in TX-RAM, tx_obj=%d), occurred %d time%s.\n", msg, addr, nr, ecc->cnt, ecc->cnt > 1 ? "s" : ""); - if (ecc->cnt >= MCP25XXFD_ECC_CNT_MAX) - return mcp25xxfd_handle_eccif_recover(priv, nr); + if (ecc->cnt >= MCP251XFD_ECC_CNT_MAX) + return mcp251xfd_handle_eccif_recover(priv, nr); } if (set_normal_mode) - return mcp25xxfd_chip_set_normal_mode_nowait(priv); + return mcp251xfd_chip_set_normal_mode_nowait(priv); return 0; } -static int mcp25xxfd_handle_spicrcif(struct mcp25xxfd_priv *priv) +static int mcp251xfd_handle_spicrcif(struct mcp251xfd_priv *priv) { int err; u32 crc; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_CRC, &crc); + err = regmap_read(priv->map_reg, MCP251XFD_REG_CRC, &crc); if (err) return err; - err = regmap_update_bits(priv->map_reg, MCP25XXFD_REG_CRC, - MCP25XXFD_REG_CRC_IF_MASK, + err = regmap_update_bits(priv->map_reg, MCP251XFD_REG_CRC, + MCP251XFD_REG_CRC_IF_MASK, ~crc); if (err) return err; - if (crc & MCP25XXFD_REG_CRC_FERRIF) + if (crc & MCP251XFD_REG_CRC_FERRIF) netdev_notice(priv->ndev, "CRC write command format error.\n"); - else if (crc & MCP25XXFD_REG_CRC_CRCERRIF) + else if (crc & MCP251XFD_REG_CRC_CRCERRIF) netdev_notice(priv->ndev, "CRC write error detected. CRC=0x%04lx.\n", - FIELD_GET(MCP25XXFD_REG_CRC_MASK, crc)); + FIELD_GET(MCP251XFD_REG_CRC_MASK, crc)); return 0; } -#define mcp25xxfd_handle(priv, irq, ...) \ +#define mcp251xfd_handle(priv, irq, ...) \ ({ \ - struct mcp25xxfd_priv *_priv = (priv); \ + struct mcp251xfd_priv *_priv = (priv); \ int err; \ \ - err = mcp25xxfd_handle_##irq(_priv, ## __VA_ARGS__); \ + err = mcp251xfd_handle_##irq(_priv, ## __VA_ARGS__); \ if (err) \ netdev_err(_priv->ndev, \ - "IRQ handler mcp25xxfd_handle_%s() returned %d.\n", \ + "IRQ handler mcp251xfd_handle_%s() returned %d.\n", \ __stringify(irq), err); \ err; \ }) -static irqreturn_t mcp25xxfd_irq(int irq, void *dev_id) +static irqreturn_t mcp251xfd_irq(int irq, void *dev_id) { - struct mcp25xxfd_priv *priv = dev_id; + struct mcp251xfd_priv *priv = dev_id; irqreturn_t handled = IRQ_NONE; int err; @@ -2064,7 +2071,7 @@ static irqreturn_t mcp25xxfd_irq(int irq, void *dev_id) if (!rx_pending) break; - err = mcp25xxfd_handle(priv, rxif); + err = mcp251xfd_handle(priv, rxif); if (err) goto out_fail; @@ -2073,92 +2080,92 @@ static irqreturn_t mcp25xxfd_irq(int irq, void *dev_id) do { u32 intf_pending, intf_pending_clearable; - bool set_normal_mode; + bool set_normal_mode = false; - err = regmap_bulk_read(priv->map_reg, MCP25XXFD_REG_INT, + err = regmap_bulk_read(priv->map_reg, MCP251XFD_REG_INT, &priv->regs_status, sizeof(priv->regs_status) / sizeof(u32)); if (err) goto out_fail; - intf_pending = FIELD_GET(MCP25XXFD_REG_INT_IF_MASK, + intf_pending = FIELD_GET(MCP251XFD_REG_INT_IF_MASK, priv->regs_status.intf) & - FIELD_GET(MCP25XXFD_REG_INT_IE_MASK, + FIELD_GET(MCP251XFD_REG_INT_IE_MASK, priv->regs_status.intf); if (!(intf_pending)) return handled; /* Some interrupts must be ACKed in the - * MCP25XXFD_REG_INT register. + * MCP251XFD_REG_INT register. * - First ACK then handle, to avoid lost-IRQ race * condition on fast re-occurring interrupts. * - Write "0" to clear active IRQs, "1" to all other, * to avoid r/m/w race condition on the - * MCP25XXFD_REG_INT register. + * MCP251XFD_REG_INT register. */ intf_pending_clearable = intf_pending & - MCP25XXFD_REG_INT_IF_CLEARABLE_MASK; + MCP251XFD_REG_INT_IF_CLEARABLE_MASK; if (intf_pending_clearable) { err = regmap_update_bits(priv->map_reg, - MCP25XXFD_REG_INT, - MCP25XXFD_REG_INT_IF_MASK, + MCP251XFD_REG_INT, + MCP251XFD_REG_INT_IF_MASK, ~intf_pending_clearable); if (err) goto out_fail; } - if (intf_pending & MCP25XXFD_REG_INT_MODIF) { - err = mcp25xxfd_handle(priv, modif, &set_normal_mode); + if (intf_pending & MCP251XFD_REG_INT_MODIF) { + err = mcp251xfd_handle(priv, modif, &set_normal_mode); if (err) goto out_fail; } - if (intf_pending & MCP25XXFD_REG_INT_RXIF) { - err = mcp25xxfd_handle(priv, rxif); + if (intf_pending & MCP251XFD_REG_INT_RXIF) { + err = mcp251xfd_handle(priv, rxif); if (err) goto out_fail; } - if (intf_pending & MCP25XXFD_REG_INT_TEFIF) { - err = mcp25xxfd_handle(priv, tefif); + if (intf_pending & MCP251XFD_REG_INT_TEFIF) { + err = mcp251xfd_handle(priv, tefif); if (err) goto out_fail; } - if (intf_pending & MCP25XXFD_REG_INT_RXOVIF) { - err = mcp25xxfd_handle(priv, rxovif); + if (intf_pending & MCP251XFD_REG_INT_RXOVIF) { + err = mcp251xfd_handle(priv, rxovif); if (err) goto out_fail; } - if (intf_pending & MCP25XXFD_REG_INT_TXATIF) { - err = mcp25xxfd_handle(priv, txatif); + if (intf_pending & MCP251XFD_REG_INT_TXATIF) { + err = mcp251xfd_handle(priv, txatif); if (err) goto out_fail; } - if (intf_pending & MCP25XXFD_REG_INT_IVMIF) { - err = mcp25xxfd_handle(priv, ivmif); + if (intf_pending & MCP251XFD_REG_INT_IVMIF) { + err = mcp251xfd_handle(priv, ivmif); if (err) goto out_fail; } - if (intf_pending & MCP25XXFD_REG_INT_SERRIF) { - err = mcp25xxfd_handle(priv, serrif); + if (intf_pending & MCP251XFD_REG_INT_SERRIF) { + err = mcp251xfd_handle(priv, serrif); if (err) goto out_fail; } - if (intf_pending & MCP25XXFD_REG_INT_ECCIF) { - err = mcp25xxfd_handle(priv, eccif, set_normal_mode); + if (intf_pending & MCP251XFD_REG_INT_ECCIF) { + err = mcp251xfd_handle(priv, eccif, set_normal_mode); if (err) goto out_fail; } - if (intf_pending & MCP25XXFD_REG_INT_SPICRCIF) { - err = mcp25xxfd_handle(priv, spicrcif); + if (intf_pending & MCP251XFD_REG_INT_SPICRCIF) { + err = mcp251xfd_handle(priv, spicrcif); if (err) goto out_fail; } @@ -2167,16 +2174,16 @@ static irqreturn_t mcp25xxfd_irq(int irq, void *dev_id) * CERRIF IRQ on the transition TX ERROR_WARNING -> TX * ERROR_ACTIVE. */ - if (intf_pending & MCP25XXFD_REG_INT_CERRIF || + if (intf_pending & MCP251XFD_REG_INT_CERRIF || priv->can.state > CAN_STATE_ERROR_ACTIVE) { - err = mcp25xxfd_handle(priv, cerrif); + err = mcp251xfd_handle(priv, cerrif); if (err) goto out_fail; /* In Bus Off we completely shut down the * controller. Every subsequent register read * will read bogus data, and if - * MCP25XXFD_QUIRK_CRC_REG is enabled the CRC + * MCP251XFD_QUIRK_CRC_REG is enabled the CRC * check will fail, too. So leave IRQ handler * directly. */ @@ -2190,30 +2197,30 @@ static irqreturn_t mcp25xxfd_irq(int irq, void *dev_id) out_fail: netdev_err(priv->ndev, "IRQ handler returned %d (intf=0x%08x).\n", err, priv->regs_status.intf); - mcp25xxfd_chip_interrupts_disable(priv); + mcp251xfd_chip_interrupts_disable(priv); return handled; } static inline struct -mcp25xxfd_tx_obj *mcp25xxfd_get_tx_obj_next(struct mcp25xxfd_tx_ring *tx_ring) +mcp251xfd_tx_obj *mcp251xfd_get_tx_obj_next(struct mcp251xfd_tx_ring *tx_ring) { u8 tx_head; - tx_head = mcp25xxfd_get_tx_head(tx_ring); + tx_head = mcp251xfd_get_tx_head(tx_ring); return &tx_ring->obj[tx_head]; } static void -mcp25xxfd_tx_obj_from_skb(const struct mcp25xxfd_priv *priv, - struct mcp25xxfd_tx_obj *tx_obj, +mcp251xfd_tx_obj_from_skb(const struct mcp251xfd_priv *priv, + struct mcp251xfd_tx_obj *tx_obj, const struct sk_buff *skb, unsigned int seq) { const struct canfd_frame *cfd = (struct canfd_frame *)skb->data; - struct mcp25xxfd_hw_tx_obj_raw *hw_tx_obj; - union mcp25xxfd_tx_obj_load_buf *load_buf; + struct mcp251xfd_hw_tx_obj_raw *hw_tx_obj; + union mcp251xfd_tx_obj_load_buf *load_buf; u8 dlc; u32 id, flags; int offset, len; @@ -2221,15 +2228,15 @@ mcp25xxfd_tx_obj_from_skb(const struct mcp25xxfd_priv *priv, if (cfd->can_id & CAN_EFF_FLAG) { u32 sid, eid; - sid = FIELD_GET(MCP25XXFD_REG_FRAME_EFF_SID_MASK, cfd->can_id); - eid = FIELD_GET(MCP25XXFD_REG_FRAME_EFF_EID_MASK, cfd->can_id); + sid = FIELD_GET(MCP251XFD_REG_FRAME_EFF_SID_MASK, cfd->can_id); + eid = FIELD_GET(MCP251XFD_REG_FRAME_EFF_EID_MASK, cfd->can_id); - id = FIELD_PREP(MCP25XXFD_OBJ_ID_EID_MASK, eid) | - FIELD_PREP(MCP25XXFD_OBJ_ID_SID_MASK, sid); + id = FIELD_PREP(MCP251XFD_OBJ_ID_EID_MASK, eid) | + FIELD_PREP(MCP251XFD_OBJ_ID_SID_MASK, sid); - flags = MCP25XXFD_OBJ_FLAGS_IDE; + flags = MCP251XFD_OBJ_FLAGS_IDE; } else { - id = FIELD_PREP(MCP25XXFD_OBJ_ID_SID_MASK, cfd->can_id); + id = FIELD_PREP(MCP251XFD_OBJ_ID_SID_MASK, cfd->can_id); flags = 0; } @@ -2238,25 +2245,25 @@ mcp25xxfd_tx_obj_from_skb(const struct mcp25xxfd_priv *priv, * TEF object. */ dlc = can_len2dlc(cfd->len); - flags |= FIELD_PREP(MCP25XXFD_OBJ_FLAGS_SEQ_MCP2518FD_MASK, seq) | - FIELD_PREP(MCP25XXFD_OBJ_FLAGS_DLC, dlc); + flags |= FIELD_PREP(MCP251XFD_OBJ_FLAGS_SEQ_MCP2518FD_MASK, seq) | + FIELD_PREP(MCP251XFD_OBJ_FLAGS_DLC, dlc); if (cfd->can_id & CAN_RTR_FLAG) - flags |= MCP25XXFD_OBJ_FLAGS_RTR; + flags |= MCP251XFD_OBJ_FLAGS_RTR; /* CANFD */ if (can_is_canfd_skb(skb)) { if (cfd->flags & CANFD_ESI) - flags |= MCP25XXFD_OBJ_FLAGS_ESI; + flags |= MCP251XFD_OBJ_FLAGS_ESI; - flags |= MCP25XXFD_OBJ_FLAGS_FDF; + flags |= MCP251XFD_OBJ_FLAGS_FDF; if (cfd->flags & CANFD_BRS) - flags |= MCP25XXFD_OBJ_FLAGS_BRS; + flags |= MCP251XFD_OBJ_FLAGS_BRS; } load_buf = &tx_obj->buf; - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_TX) + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_TX) hw_tx_obj = &load_buf->crc.hw_tx_obj; else hw_tx_obj = &load_buf->nocrc.hw_tx_obj; @@ -2267,25 +2274,25 @@ mcp25xxfd_tx_obj_from_skb(const struct mcp25xxfd_priv *priv, /* Clear data at end of CAN frame */ offset = round_down(cfd->len, sizeof(u32)); len = round_up(can_dlc2len(dlc), sizeof(u32)) - offset; - if (MCP25XXFD_SANITIZE_CAN && len) + if (MCP251XFD_SANITIZE_CAN && len) memset(hw_tx_obj->data + offset, 0x0, len); memcpy(hw_tx_obj->data, cfd->data, cfd->len); /* Number of bytes to be written into the RAM of the controller */ len = sizeof(hw_tx_obj->id) + sizeof(hw_tx_obj->flags); - if (MCP25XXFD_SANITIZE_CAN) + if (MCP251XFD_SANITIZE_CAN) len += round_up(can_dlc2len(dlc), sizeof(u32)); else len += round_up(cfd->len, sizeof(u32)); - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_TX) { + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_TX) { u16 crc; - mcp25xxfd_spi_cmd_crc_set_len_in_ram(&load_buf->crc.cmd, + mcp251xfd_spi_cmd_crc_set_len_in_ram(&load_buf->crc.cmd, len); /* CRC */ len += sizeof(load_buf->crc.cmd); - crc = mcp25xxfd_crc16_compute(&load_buf->crc, len); + crc = mcp251xfd_crc16_compute(&load_buf->crc, len); put_unaligned_be16(crc, (void *)load_buf + len); /* Total length */ @@ -2297,16 +2304,16 @@ mcp25xxfd_tx_obj_from_skb(const struct mcp25xxfd_priv *priv, tx_obj->xfer[0].len = len; } -static int mcp25xxfd_tx_obj_write(const struct mcp25xxfd_priv *priv, - struct mcp25xxfd_tx_obj *tx_obj) +static int mcp251xfd_tx_obj_write(const struct mcp251xfd_priv *priv, + struct mcp251xfd_tx_obj *tx_obj) { return spi_async(priv->spi, &tx_obj->msg); } -static bool mcp25xxfd_tx_busy(const struct mcp25xxfd_priv *priv, - struct mcp25xxfd_tx_ring *tx_ring) +static bool mcp251xfd_tx_busy(const struct mcp251xfd_priv *priv, + struct mcp251xfd_tx_ring *tx_ring) { - if (mcp25xxfd_get_tx_free(tx_ring) > 0) + if (mcp251xfd_get_tx_free(tx_ring) > 0) return false; netif_stop_queue(priv->ndev); @@ -2314,7 +2321,7 @@ static bool mcp25xxfd_tx_busy(const struct mcp25xxfd_priv *priv, /* Memory barrier before checking tx_free (head and tail) */ smp_mb(); - if (mcp25xxfd_get_tx_free(tx_ring) == 0) { + if (mcp251xfd_get_tx_free(tx_ring) == 0) { netdev_dbg(priv->ndev, "Stopping tx-queue (tx_head=0x%08x, tx_tail=0x%08x, len=%d).\n", tx_ring->head, tx_ring->tail, @@ -2328,33 +2335,33 @@ static bool mcp25xxfd_tx_busy(const struct mcp25xxfd_priv *priv, return false; } -static netdev_tx_t mcp25xxfd_start_xmit(struct sk_buff *skb, +static netdev_tx_t mcp251xfd_start_xmit(struct sk_buff *skb, struct net_device *ndev) { - struct mcp25xxfd_priv *priv = netdev_priv(ndev); - struct mcp25xxfd_tx_ring *tx_ring = priv->tx; - struct mcp25xxfd_tx_obj *tx_obj; + struct mcp251xfd_priv *priv = netdev_priv(ndev); + struct mcp251xfd_tx_ring *tx_ring = priv->tx; + struct mcp251xfd_tx_obj *tx_obj; u8 tx_head; int err; if (can_dropped_invalid_skb(ndev, skb)) return NETDEV_TX_OK; - if (mcp25xxfd_tx_busy(priv, tx_ring)) + if (mcp251xfd_tx_busy(priv, tx_ring)) return NETDEV_TX_BUSY; - tx_obj = mcp25xxfd_get_tx_obj_next(tx_ring); - mcp25xxfd_tx_obj_from_skb(priv, tx_obj, skb, tx_ring->head); + tx_obj = mcp251xfd_get_tx_obj_next(tx_ring); + mcp251xfd_tx_obj_from_skb(priv, tx_obj, skb, tx_ring->head); /* Stop queue if we occupy the complete TX FIFO */ - tx_head = mcp25xxfd_get_tx_head(tx_ring); + tx_head = mcp251xfd_get_tx_head(tx_ring); tx_ring->head++; if (tx_ring->head - tx_ring->tail >= tx_ring->obj_num) netif_stop_queue(ndev); can_put_echo_skb(skb, ndev, tx_head); - err = mcp25xxfd_tx_obj_write(priv, tx_obj); + err = mcp251xfd_tx_obj_write(priv, tx_obj); if (err) goto out_err; @@ -2366,9 +2373,9 @@ static netdev_tx_t mcp25xxfd_start_xmit(struct sk_buff *skb, return NETDEV_TX_OK; } -static int mcp25xxfd_open(struct net_device *ndev) +static int mcp251xfd_open(struct net_device *ndev) { - struct mcp25xxfd_priv *priv = netdev_priv(ndev); + struct mcp251xfd_priv *priv = netdev_priv(ndev); const struct spi_device *spi = priv->spi; int err; @@ -2382,27 +2389,27 @@ static int mcp25xxfd_open(struct net_device *ndev) if (err) goto out_pm_runtime_put; - err = mcp25xxfd_ring_alloc(priv); + err = mcp251xfd_ring_alloc(priv); if (err) goto out_close_candev; - err = mcp25xxfd_transceiver_enable(priv); + err = mcp251xfd_transceiver_enable(priv); if (err) - goto out_mcp25xxfd_ring_free; + goto out_mcp251xfd_ring_free; - err = mcp25xxfd_chip_start(priv); + err = mcp251xfd_chip_start(priv); if (err) goto out_transceiver_disable; can_rx_offload_enable(&priv->offload); - err = request_threaded_irq(spi->irq, NULL, mcp25xxfd_irq, + err = request_threaded_irq(spi->irq, NULL, mcp251xfd_irq, IRQF_ONESHOT, dev_name(&spi->dev), priv); if (err) goto out_can_rx_offload_disable; - err = mcp25xxfd_chip_interrupts_enable(priv); + err = mcp251xfd_chip_interrupts_enable(priv); if (err) goto out_free_irq; @@ -2415,29 +2422,29 @@ static int mcp25xxfd_open(struct net_device *ndev) out_can_rx_offload_disable: can_rx_offload_disable(&priv->offload); out_transceiver_disable: - mcp25xxfd_transceiver_disable(priv); - out_mcp25xxfd_ring_free: - mcp25xxfd_ring_free(priv); + mcp251xfd_transceiver_disable(priv); + out_mcp251xfd_ring_free: + mcp251xfd_ring_free(priv); out_close_candev: close_candev(ndev); out_pm_runtime_put: - mcp25xxfd_chip_stop(priv, CAN_STATE_STOPPED); + mcp251xfd_chip_stop(priv, CAN_STATE_STOPPED); pm_runtime_put(ndev->dev.parent); return err; } -static int mcp25xxfd_stop(struct net_device *ndev) +static int mcp251xfd_stop(struct net_device *ndev) { - struct mcp25xxfd_priv *priv = netdev_priv(ndev); + struct mcp251xfd_priv *priv = netdev_priv(ndev); netif_stop_queue(ndev); - mcp25xxfd_chip_interrupts_disable(priv); + mcp251xfd_chip_interrupts_disable(priv); free_irq(ndev->irq, priv); can_rx_offload_disable(&priv->offload); - mcp25xxfd_chip_stop(priv, CAN_STATE_STOPPED); - mcp25xxfd_transceiver_disable(priv); - mcp25xxfd_ring_free(priv); + mcp251xfd_chip_stop(priv, CAN_STATE_STOPPED); + mcp251xfd_transceiver_disable(priv); + mcp251xfd_ring_free(priv); close_candev(ndev); pm_runtime_put(ndev->dev.parent); @@ -2445,72 +2452,72 @@ static int mcp25xxfd_stop(struct net_device *ndev) return 0; } -static const struct net_device_ops mcp25xxfd_netdev_ops = { - .ndo_open = mcp25xxfd_open, - .ndo_stop = mcp25xxfd_stop, - .ndo_start_xmit = mcp25xxfd_start_xmit, +static const struct net_device_ops mcp251xfd_netdev_ops = { + .ndo_open = mcp251xfd_open, + .ndo_stop = mcp251xfd_stop, + .ndo_start_xmit = mcp251xfd_start_xmit, .ndo_change_mtu = can_change_mtu, }; static void -mcp25xxfd_register_quirks(struct mcp25xxfd_priv *priv) +mcp251xfd_register_quirks(struct mcp251xfd_priv *priv) { const struct spi_device *spi = priv->spi; const struct spi_controller *ctlr = spi->controller; if (ctlr->flags & SPI_CONTROLLER_HALF_DUPLEX) - priv->devtype_data.quirks |= MCP25XXFD_QUIRK_HALF_DUPLEX; + priv->devtype_data.quirks |= MCP251XFD_QUIRK_HALF_DUPLEX; } -static int mcp25xxfd_register_chip_detect(struct mcp25xxfd_priv *priv) +static int mcp251xfd_register_chip_detect(struct mcp251xfd_priv *priv) { const struct net_device *ndev = priv->ndev; - const struct mcp25xxfd_devtype_data *devtype_data; + const struct mcp251xfd_devtype_data *devtype_data; u32 osc; int err; /* The OSC_LPMEN is only supported on MCP2518FD, so use it to * autodetect the model. */ - err = regmap_update_bits(priv->map_reg, MCP25XXFD_REG_OSC, - MCP25XXFD_REG_OSC_LPMEN, - MCP25XXFD_REG_OSC_LPMEN); + err = regmap_update_bits(priv->map_reg, MCP251XFD_REG_OSC, + MCP251XFD_REG_OSC_LPMEN, + MCP251XFD_REG_OSC_LPMEN); if (err) return err; - err = regmap_read(priv->map_reg, MCP25XXFD_REG_OSC, &osc); + err = regmap_read(priv->map_reg, MCP251XFD_REG_OSC, &osc); if (err) return err; - if (osc & MCP25XXFD_REG_OSC_LPMEN) - devtype_data = &mcp25xxfd_devtype_data_mcp2518fd; + if (osc & MCP251XFD_REG_OSC_LPMEN) + devtype_data = &mcp251xfd_devtype_data_mcp2518fd; else - devtype_data = &mcp25xxfd_devtype_data_mcp2517fd; + devtype_data = &mcp251xfd_devtype_data_mcp2517fd; - if (!mcp25xxfd_is_25XX(priv) && + if (!mcp251xfd_is_251X(priv) && priv->devtype_data.model != devtype_data->model) { netdev_info(ndev, "Detected %s, but firmware specifies a %s. Fixing up.", - __mcp25xxfd_get_model_str(devtype_data->model), - mcp25xxfd_get_model_str(priv)); + __mcp251xfd_get_model_str(devtype_data->model), + mcp251xfd_get_model_str(priv)); } priv->devtype_data = *devtype_data; /* We need to preserve the Half Duplex Quirk. */ - mcp25xxfd_register_quirks(priv); + mcp251xfd_register_quirks(priv); /* Re-init regmap with quirks of detected model. */ - return mcp25xxfd_regmap_init(priv); + return mcp251xfd_regmap_init(priv); } -static int mcp25xxfd_register_check_rx_int(struct mcp25xxfd_priv *priv) +static int mcp251xfd_register_check_rx_int(struct mcp251xfd_priv *priv) { int err, rx_pending; if (!priv->rx_int) return 0; - err = mcp25xxfd_chip_rx_int_enable(priv); + err = mcp251xfd_chip_rx_int_enable(priv); if (err) return err; @@ -2519,7 +2526,7 @@ static int mcp25xxfd_register_check_rx_int(struct mcp25xxfd_priv *priv) */ rx_pending = gpiod_get_value_cansleep(priv->rx_int); - err = mcp25xxfd_chip_rx_int_disable(priv); + err = mcp251xfd_chip_rx_int_disable(priv); if (err) return err; @@ -2535,11 +2542,11 @@ static int mcp25xxfd_register_check_rx_int(struct mcp25xxfd_priv *priv) } static int -mcp25xxfd_register_get_dev_id(const struct mcp25xxfd_priv *priv, +mcp251xfd_register_get_dev_id(const struct mcp251xfd_priv *priv, u32 *dev_id, u32 *effective_speed_hz) { - struct mcp25xxfd_map_buf_nocrc *buf_rx; - struct mcp25xxfd_map_buf_nocrc *buf_tx; + struct mcp251xfd_map_buf_nocrc *buf_rx; + struct mcp251xfd_map_buf_nocrc *buf_tx; struct spi_transfer xfer[2] = { }; int err; @@ -2558,7 +2565,7 @@ mcp25xxfd_register_get_dev_id(const struct mcp25xxfd_priv *priv, xfer[1].rx_buf = buf_rx->data; xfer[1].len = sizeof(dev_id); - mcp25xxfd_spi_cmd_read_nocrc(&buf_tx->cmd, MCP25XXFD_REG_DEVID); + mcp251xfd_spi_cmd_read_nocrc(&buf_tx->cmd, MCP251XFD_REG_DEVID); err = spi_sync_transfer(priv->spi, xfer, ARRAY_SIZE(xfer)); if (err) goto out_kfree_buf_tx; @@ -2574,32 +2581,32 @@ mcp25xxfd_register_get_dev_id(const struct mcp25xxfd_priv *priv, return 0; } -#define MCP25XXFD_QUIRK_ACTIVE(quirk) \ - (priv->devtype_data.quirks & MCP25XXFD_QUIRK_##quirk ? '+' : '-') +#define MCP251XFD_QUIRK_ACTIVE(quirk) \ + (priv->devtype_data.quirks & MCP251XFD_QUIRK_##quirk ? '+' : '-') static int -mcp25xxfd_register_done(const struct mcp25xxfd_priv *priv) +mcp251xfd_register_done(const struct mcp251xfd_priv *priv) { u32 dev_id, effective_speed_hz; int err; - err = mcp25xxfd_register_get_dev_id(priv, &dev_id, + err = mcp251xfd_register_get_dev_id(priv, &dev_id, &effective_speed_hz); if (err) return err; netdev_info(priv->ndev, "%s rev%lu.%lu (%cRX_INT %cMAB_NO_WARN %cCRC_REG %cCRC_RX %cCRC_TX %cECC %cHD c:%u.%02uMHz m:%u.%02uMHz r:%u.%02uMHz e:%u.%02uMHz) successfully initialized.\n", - mcp25xxfd_get_model_str(priv), - FIELD_GET(MCP25XXFD_REG_DEVID_ID_MASK, dev_id), - FIELD_GET(MCP25XXFD_REG_DEVID_REV_MASK, dev_id), + mcp251xfd_get_model_str(priv), + FIELD_GET(MCP251XFD_REG_DEVID_ID_MASK, dev_id), + FIELD_GET(MCP251XFD_REG_DEVID_REV_MASK, dev_id), priv->rx_int ? '+' : '-', - MCP25XXFD_QUIRK_ACTIVE(MAB_NO_WARN), - MCP25XXFD_QUIRK_ACTIVE(CRC_REG), - MCP25XXFD_QUIRK_ACTIVE(CRC_RX), - MCP25XXFD_QUIRK_ACTIVE(CRC_TX), - MCP25XXFD_QUIRK_ACTIVE(ECC), - MCP25XXFD_QUIRK_ACTIVE(HALF_DUPLEX), + MCP251XFD_QUIRK_ACTIVE(MAB_NO_WARN), + MCP251XFD_QUIRK_ACTIVE(CRC_REG), + MCP251XFD_QUIRK_ACTIVE(CRC_RX), + MCP251XFD_QUIRK_ACTIVE(CRC_TX), + MCP251XFD_QUIRK_ACTIVE(ECC), + MCP251XFD_QUIRK_ACTIVE(HALF_DUPLEX), priv->can.clock.freq / 1000000, priv->can.clock.freq % 1000000 / 1000 / 10, priv->spi_max_speed_hz_orig / 1000000, @@ -2612,12 +2619,12 @@ mcp25xxfd_register_done(const struct mcp25xxfd_priv *priv) return 0; } -static int mcp25xxfd_register(struct mcp25xxfd_priv *priv) +static int mcp251xfd_register(struct mcp251xfd_priv *priv) { struct net_device *ndev = priv->ndev; int err; - err = mcp25xxfd_clks_and_vdd_enable(priv); + err = mcp251xfd_clks_and_vdd_enable(priv); if (err) return err; @@ -2627,19 +2634,19 @@ static int mcp25xxfd_register(struct mcp25xxfd_priv *priv) goto out_runtime_put_noidle; pm_runtime_enable(ndev->dev.parent); - mcp25xxfd_register_quirks(priv); + mcp251xfd_register_quirks(priv); - err = mcp25xxfd_chip_softreset(priv); + err = mcp251xfd_chip_softreset(priv); if (err == -ENODEV) goto out_runtime_disable; if (err) goto out_chip_set_mode_sleep; - err = mcp25xxfd_register_chip_detect(priv); + err = mcp251xfd_register_chip_detect(priv); if (err) goto out_chip_set_mode_sleep; - err = mcp25xxfd_register_check_rx_int(priv); + err = mcp251xfd_register_check_rx_int(priv); if (err) goto out_chip_set_mode_sleep; @@ -2647,7 +2654,7 @@ static int mcp25xxfd_register(struct mcp25xxfd_priv *priv) if (err) goto out_chip_set_mode_sleep; - err = mcp25xxfd_register_done(priv); + err = mcp251xfd_register_done(priv); if (err) goto out_unregister_candev; @@ -2655,7 +2662,7 @@ static int mcp25xxfd_register(struct mcp25xxfd_priv *priv) * disable the clocks and vdd. If CONFIG_PM is not enabled, * the clocks and vdd will stay powered. */ - err = mcp25xxfd_chip_set_mode(priv, MCP25XXFD_REG_CON_MODE_SLEEP); + err = mcp251xfd_chip_set_mode(priv, MCP251XFD_REG_CON_MODE_SLEEP); if (err) goto out_unregister_candev; @@ -2666,17 +2673,17 @@ static int mcp25xxfd_register(struct mcp25xxfd_priv *priv) out_unregister_candev: unregister_candev(ndev); out_chip_set_mode_sleep: - mcp25xxfd_chip_set_mode(priv, MCP25XXFD_REG_CON_MODE_SLEEP); + mcp251xfd_chip_set_mode(priv, MCP251XFD_REG_CON_MODE_SLEEP); out_runtime_disable: pm_runtime_disable(ndev->dev.parent); out_runtime_put_noidle: pm_runtime_put_noidle(ndev->dev.parent); - mcp25xxfd_clks_and_vdd_disable(priv); + mcp251xfd_clks_and_vdd_disable(priv); return err; } -static inline void mcp25xxfd_unregister(struct mcp25xxfd_priv *priv) +static inline void mcp251xfd_unregister(struct mcp251xfd_priv *priv) { struct net_device *ndev = priv->ndev; @@ -2684,47 +2691,47 @@ static inline void mcp25xxfd_unregister(struct mcp25xxfd_priv *priv) pm_runtime_get_sync(ndev->dev.parent); pm_runtime_put_noidle(ndev->dev.parent); - mcp25xxfd_clks_and_vdd_disable(priv); + mcp251xfd_clks_and_vdd_disable(priv); pm_runtime_disable(ndev->dev.parent); } -static const struct of_device_id mcp25xxfd_of_match[] = { +static const struct of_device_id mcp251xfd_of_match[] = { { .compatible = "microchip,mcp2517fd", - .data = &mcp25xxfd_devtype_data_mcp2517fd, + .data = &mcp251xfd_devtype_data_mcp2517fd, }, { .compatible = "microchip,mcp2518fd", - .data = &mcp25xxfd_devtype_data_mcp2518fd, + .data = &mcp251xfd_devtype_data_mcp2518fd, }, { - .compatible = "microchip,mcp25xxfd", - .data = &mcp25xxfd_devtype_data_mcp25xxfd, + .compatible = "microchip,mcp251xfd", + .data = &mcp251xfd_devtype_data_mcp251xfd, }, { /* sentinel */ }, }; -MODULE_DEVICE_TABLE(of, mcp25xxfd_of_match); +MODULE_DEVICE_TABLE(of, mcp251xfd_of_match); -static const struct spi_device_id mcp25xxfd_id_table[] = { +static const struct spi_device_id mcp251xfd_id_table[] = { { .name = "mcp2517fd", - .driver_data = (kernel_ulong_t)&mcp25xxfd_devtype_data_mcp2517fd, + .driver_data = (kernel_ulong_t)&mcp251xfd_devtype_data_mcp2517fd, }, { .name = "mcp2518fd", - .driver_data = (kernel_ulong_t)&mcp25xxfd_devtype_data_mcp2518fd, + .driver_data = (kernel_ulong_t)&mcp251xfd_devtype_data_mcp2518fd, }, { - .name = "mcp25xxfd", - .driver_data = (kernel_ulong_t)&mcp25xxfd_devtype_data_mcp25xxfd, + .name = "mcp251xfd", + .driver_data = (kernel_ulong_t)&mcp251xfd_devtype_data_mcp251xfd, }, { /* sentinel */ }, }; -MODULE_DEVICE_TABLE(spi, mcp25xxfd_id_table); +MODULE_DEVICE_TABLE(spi, mcp251xfd_id_table); -static int mcp25xxfd_probe(struct spi_device *spi) +static int mcp251xfd_probe(struct spi_device *spi) { const void *match; struct net_device *ndev; - struct mcp25xxfd_priv *priv; + struct mcp251xfd_priv *priv; struct gpio_desc *rx_int; struct regulator *reg_vdd, *reg_xceiver; struct clk *clk; @@ -2762,39 +2769,39 @@ static int mcp25xxfd_probe(struct spi_device *spi) freq = clk_get_rate(clk); /* Sanity check */ - if (freq < MCP25XXFD_SYSCLOCK_HZ_MIN || - freq > MCP25XXFD_SYSCLOCK_HZ_MAX) { + if (freq < MCP251XFD_SYSCLOCK_HZ_MIN || + freq > MCP251XFD_SYSCLOCK_HZ_MAX) { dev_err(&spi->dev, "Oscillator frequency (%u Hz) is too low or high.\n", freq); return -ERANGE; } - if (freq <= MCP25XXFD_SYSCLOCK_HZ_MAX / MCP25XXFD_OSC_PLL_MULTIPLIER) { + if (freq <= MCP251XFD_SYSCLOCK_HZ_MAX / MCP251XFD_OSC_PLL_MULTIPLIER) { dev_err(&spi->dev, "Oscillator frequency (%u Hz) is too low and PLL is not supported.\n", freq); return -ERANGE; } - ndev = alloc_candev(sizeof(struct mcp25xxfd_priv), - MCP25XXFD_TX_OBJ_NUM_MAX); + ndev = alloc_candev(sizeof(struct mcp251xfd_priv), + MCP251XFD_TX_OBJ_NUM_MAX); if (!ndev) return -ENOMEM; SET_NETDEV_DEV(ndev, &spi->dev); - ndev->netdev_ops = &mcp25xxfd_netdev_ops; + ndev->netdev_ops = &mcp251xfd_netdev_ops; ndev->irq = spi->irq; ndev->flags |= IFF_ECHO; priv = netdev_priv(ndev); spi_set_drvdata(spi, priv); priv->can.clock.freq = freq; - priv->can.do_set_mode = mcp25xxfd_set_mode; - priv->can.do_get_berr_counter = mcp25xxfd_get_berr_counter; - priv->can.bittiming_const = &mcp25xxfd_bittiming_const; - priv->can.data_bittiming_const = &mcp25xxfd_data_bittiming_const; + priv->can.do_set_mode = mcp251xfd_set_mode; + priv->can.do_get_berr_counter = mcp251xfd_get_berr_counter; + priv->can.bittiming_const = &mcp251xfd_bittiming_const; + priv->can.data_bittiming_const = &mcp251xfd_data_bittiming_const; priv->can.ctrlmode_supported = CAN_CTRLMODE_LISTENONLY | CAN_CTRLMODE_BERR_REPORTING | CAN_CTRLMODE_FD | CAN_CTRLMODE_FD_NON_ISO; @@ -2807,16 +2814,26 @@ static int mcp25xxfd_probe(struct spi_device *spi) match = device_get_match_data(&spi->dev); if (match) - priv->devtype_data = *(struct mcp25xxfd_devtype_data *)match; + priv->devtype_data = *(struct mcp251xfd_devtype_data *)match; else - priv->devtype_data = *(struct mcp25xxfd_devtype_data *) + priv->devtype_data = *(struct mcp251xfd_devtype_data *) spi_get_device_id(spi)->driver_data; - /* According to the datasheet the SPI clock must be less or - * equal SYSCLOCK / 2. + /* Errata Reference: + * mcp2517fd: DS80000789B, mcp2518fd: DS80000792C 4. + * + * The SPI can write corrupted data to the RAM at fast SPI + * speeds: + * + * Simultaneous activity on the CAN bus while writing data to + * RAM via the SPI interface, with high SCK frequency, can + * lead to corrupted data being written to RAM. + * + * Fix/Work Around: + * Ensure that FSCK is less than or equal to 0.85 * + * (FSYSCLK/2). * - * It turns out, that the Controller is not stable at this - * rate. Known good and bad combinations are: + * Known good and bad combinations are: * * MCP ext-clk SoC SPI SPI-clk max-clk parent-clk Status config * @@ -2829,7 +2846,6 @@ static int mcp25xxfd_probe(struct spi_device *spi) * 2517 40 MHz atmel,sama5d27 atmel,at91rm9200-spi 16400000 Hz 82.00% 82000000 Hz good default * 2518 40 MHz atmel,sama5d27 atmel,at91rm9200-spi 16400000 Hz 82.00% 82000000 Hz good default * - * Limit SPI clock to 85% of SYSCLOCK / 2 for now. */ priv->spi_max_speed_hz_orig = spi->max_speed_hz; spi->max_speed_hz = min(spi->max_speed_hz, freq / 2 / 1000 * 850); @@ -2839,16 +2855,16 @@ static int mcp25xxfd_probe(struct spi_device *spi) if (err) goto out_free_candev; - err = mcp25xxfd_regmap_init(priv); + err = mcp251xfd_regmap_init(priv); if (err) goto out_free_candev; err = can_rx_offload_add_manual(ndev, &priv->offload, - MCP25XXFD_NAPI_WEIGHT); + MCP251XFD_NAPI_WEIGHT); if (err) goto out_free_candev; - err = mcp25xxfd_register(priv); + err = mcp251xfd_register(priv); if (err) goto out_free_candev; @@ -2862,50 +2878,50 @@ static int mcp25xxfd_probe(struct spi_device *spi) return err; } -static int mcp25xxfd_remove(struct spi_device *spi) +static int mcp251xfd_remove(struct spi_device *spi) { - struct mcp25xxfd_priv *priv = spi_get_drvdata(spi); + struct mcp251xfd_priv *priv = spi_get_drvdata(spi); struct net_device *ndev = priv->ndev; can_rx_offload_del(&priv->offload); - mcp25xxfd_unregister(priv); + mcp251xfd_unregister(priv); spi->max_speed_hz = priv->spi_max_speed_hz_orig; free_candev(ndev); return 0; } -static int __maybe_unused mcp25xxfd_runtime_suspend(struct device *device) +static int __maybe_unused mcp251xfd_runtime_suspend(struct device *device) { - const struct mcp25xxfd_priv *priv = dev_get_drvdata(device); + const struct mcp251xfd_priv *priv = dev_get_drvdata(device); - return mcp25xxfd_clks_and_vdd_disable(priv); + return mcp251xfd_clks_and_vdd_disable(priv); } -static int __maybe_unused mcp25xxfd_runtime_resume(struct device *device) +static int __maybe_unused mcp251xfd_runtime_resume(struct device *device) { - const struct mcp25xxfd_priv *priv = dev_get_drvdata(device); + const struct mcp251xfd_priv *priv = dev_get_drvdata(device); - return mcp25xxfd_clks_and_vdd_enable(priv); + return mcp251xfd_clks_and_vdd_enable(priv); } -static const struct dev_pm_ops mcp25xxfd_pm_ops = { - SET_RUNTIME_PM_OPS(mcp25xxfd_runtime_suspend, - mcp25xxfd_runtime_resume, NULL) +static const struct dev_pm_ops mcp251xfd_pm_ops = { + SET_RUNTIME_PM_OPS(mcp251xfd_runtime_suspend, + mcp251xfd_runtime_resume, NULL) }; -static struct spi_driver mcp25xxfd_driver = { +static struct spi_driver mcp251xfd_driver = { .driver = { .name = DEVICE_NAME, - .pm = &mcp25xxfd_pm_ops, - .of_match_table = mcp25xxfd_of_match, + .pm = &mcp251xfd_pm_ops, + .of_match_table = mcp251xfd_of_match, }, - .probe = mcp25xxfd_probe, - .remove = mcp25xxfd_remove, - .id_table = mcp25xxfd_id_table, + .probe = mcp251xfd_probe, + .remove = mcp251xfd_remove, + .id_table = mcp251xfd_id_table, }; -module_spi_driver(mcp25xxfd_driver); +module_spi_driver(mcp251xfd_driver); MODULE_AUTHOR("Marc Kleine-Budde <mkl@pengutronix.de>"); -MODULE_DESCRIPTION("Microchip MCP25xxFD Family CAN controller driver"); +MODULE_DESCRIPTION("Microchip MCP251xFD Family CAN controller driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/can/spi/mcp25xxfd/mcp25xxfd-crc16.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-crc16.c index 79d09aaebf33..a02ca76ac239 100644 --- a/drivers/net/can/spi/mcp25xxfd/mcp25xxfd-crc16.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-crc16.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 // -// mcp25xxfd - Microchip MCP25xxFD Family CAN controller driver +// mcp251xfd - Microchip MCP251xFD Family CAN controller driver // // Copyright (c) 2020 Pengutronix, // Marc Kleine-Budde <kernel@pengutronix.de> @@ -12,7 +12,7 @@ // Copyright (c) 2019 Martin Sperl <kernel@martin.sperl.org> // -#include "mcp25xxfd.h" +#include "mcp251xfd.h" /* The standard crc16 in linux/crc16.h is unfortunately not computing * the correct results (left shift vs. right shift). So here an @@ -20,7 +20,7 @@ * * http://lkml.iu.edu/hypermail/linux/kernel/0508.1/1085.html */ -static const u16 mcp25xxfd_crc16_table[] = { +static const u16 mcp251xfd_crc16_table[] = { 0x0000, 0x8005, 0x800f, 0x000a, 0x801b, 0x001e, 0x0014, 0x8011, 0x8033, 0x0036, 0x003c, 0x8039, 0x0028, 0x802d, 0x8027, 0x0022, 0x8063, 0x0066, 0x006c, 0x8069, 0x0078, 0x807d, 0x8077, 0x0072, @@ -55,35 +55,35 @@ static const u16 mcp25xxfd_crc16_table[] = { 0x8213, 0x0216, 0x021c, 0x8219, 0x0208, 0x820d, 0x8207, 0x0202 }; -static inline u16 mcp25xxfd_crc16_byte(u16 crc, const u8 data) +static inline u16 mcp251xfd_crc16_byte(u16 crc, const u8 data) { u8 index = (crc >> 8) ^ data; - return (crc << 8) ^ mcp25xxfd_crc16_table[index]; + return (crc << 8) ^ mcp251xfd_crc16_table[index]; } -static u16 mcp25xxfd_crc16(u16 crc, u8 const *buffer, size_t len) +static u16 mcp251xfd_crc16(u16 crc, u8 const *buffer, size_t len) { while (len--) - crc = mcp25xxfd_crc16_byte(crc, *buffer++); + crc = mcp251xfd_crc16_byte(crc, *buffer++); return crc; } -u16 mcp25xxfd_crc16_compute(const void *data, size_t data_size) +u16 mcp251xfd_crc16_compute(const void *data, size_t data_size) { u16 crc = 0xffff; - return mcp25xxfd_crc16(crc, data, data_size); + return mcp251xfd_crc16(crc, data, data_size); } -u16 mcp25xxfd_crc16_compute2(const void *cmd, size_t cmd_size, +u16 mcp251xfd_crc16_compute2(const void *cmd, size_t cmd_size, const void *data, size_t data_size) { u16 crc; - crc = mcp25xxfd_crc16_compute(cmd, cmd_size); - crc = mcp25xxfd_crc16(crc, data, data_size); + crc = mcp251xfd_crc16_compute(cmd, cmd_size); + crc = mcp251xfd_crc16(crc, data, data_size); return crc; } diff --git a/drivers/net/can/spi/mcp25xxfd/mcp25xxfd-regmap.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-regmap.c index 376649c7e443..ba25902dd78c 100644 --- a/drivers/net/can/spi/mcp25xxfd/mcp25xxfd-regmap.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-regmap.c @@ -1,19 +1,19 @@ // SPDX-License-Identifier: GPL-2.0 // -// mcp25xxfd - Microchip MCP25xxFD Family CAN controller driver +// mcp251xfd - Microchip MCP251xFD Family CAN controller driver // // Copyright (c) 2019, 2020 Pengutronix, // Marc Kleine-Budde <kernel@pengutronix.de> // -#include "mcp25xxfd.h" +#include "mcp251xfd.h" #include <asm/unaligned.h> -static const struct regmap_config mcp25xxfd_regmap_crc; +static const struct regmap_config mcp251xfd_regmap_crc; static int -mcp25xxfd_regmap_nocrc_write(void *context, const void *data, size_t count) +mcp251xfd_regmap_nocrc_write(void *context, const void *data, size_t count) { struct spi_device *spi = context; @@ -21,13 +21,13 @@ mcp25xxfd_regmap_nocrc_write(void *context, const void *data, size_t count) } static int -mcp25xxfd_regmap_nocrc_gather_write(void *context, +mcp251xfd_regmap_nocrc_gather_write(void *context, const void *reg, size_t reg_len, const void *val, size_t val_len) { struct spi_device *spi = context; - struct mcp25xxfd_priv *priv = spi_get_drvdata(spi); - struct mcp25xxfd_map_buf_nocrc *buf_tx = priv->map_buf_nocrc_tx; + struct mcp251xfd_priv *priv = spi_get_drvdata(spi); + struct mcp251xfd_map_buf_nocrc *buf_tx = priv->map_buf_nocrc_tx; struct spi_transfer xfer[] = { { .tx_buf = buf_tx, @@ -37,7 +37,7 @@ mcp25xxfd_regmap_nocrc_gather_write(void *context, BUILD_BUG_ON(sizeof(buf_tx->cmd) != sizeof(__be16)); - if (IS_ENABLED(CONFIG_CAN_MCP25XXFD_SANITY) && + if (IS_ENABLED(CONFIG_CAN_MCP251XFD_SANITY) && reg_len != sizeof(buf_tx->cmd.cmd)) return -EINVAL; @@ -47,20 +47,20 @@ mcp25xxfd_regmap_nocrc_gather_write(void *context, return spi_sync_transfer(spi, xfer, ARRAY_SIZE(xfer)); } -static inline bool mcp25xxfd_update_bits_read_reg(unsigned int reg) +static inline bool mcp251xfd_update_bits_read_reg(unsigned int reg) { switch (reg) { - case MCP25XXFD_REG_INT: - case MCP25XXFD_REG_TEFCON: - case MCP25XXFD_REG_FIFOCON(MCP25XXFD_RX_FIFO(0)): - case MCP25XXFD_REG_FLTCON(0): - case MCP25XXFD_REG_ECCSTAT: - case MCP25XXFD_REG_CRC: + case MCP251XFD_REG_INT: + case MCP251XFD_REG_TEFCON: + case MCP251XFD_REG_FIFOCON(MCP251XFD_RX_FIFO(0)): + case MCP251XFD_REG_FLTCON(0): + case MCP251XFD_REG_ECCSTAT: + case MCP251XFD_REG_CRC: return false; - case MCP25XXFD_REG_CON: - case MCP25XXFD_REG_FIFOSTA(MCP25XXFD_RX_FIFO(0)): - case MCP25XXFD_REG_OSC: - case MCP25XXFD_REG_ECCCON: + case MCP251XFD_REG_CON: + case MCP251XFD_REG_FIFOSTA(MCP251XFD_RX_FIFO(0)): + case MCP251XFD_REG_OSC: + case MCP251XFD_REG_ECCCON: return true; default: WARN(1, "Status of reg 0x%04x unknown.\n", reg); @@ -70,13 +70,13 @@ static inline bool mcp25xxfd_update_bits_read_reg(unsigned int reg) } static int -mcp25xxfd_regmap_nocrc_update_bits(void *context, unsigned int reg, +mcp251xfd_regmap_nocrc_update_bits(void *context, unsigned int reg, unsigned int mask, unsigned int val) { struct spi_device *spi = context; - struct mcp25xxfd_priv *priv = spi_get_drvdata(spi); - struct mcp25xxfd_map_buf_nocrc *buf_rx = priv->map_buf_nocrc_rx; - struct mcp25xxfd_map_buf_nocrc *buf_tx = priv->map_buf_nocrc_tx; + struct mcp251xfd_priv *priv = spi_get_drvdata(spi); + struct mcp251xfd_map_buf_nocrc *buf_rx = priv->map_buf_nocrc_rx; + struct mcp251xfd_map_buf_nocrc *buf_tx = priv->map_buf_nocrc_tx; __le32 orig_le32 = 0, mask_le32, val_le32, tmp_le32; u8 first_byte, last_byte, len; int err; @@ -84,22 +84,22 @@ mcp25xxfd_regmap_nocrc_update_bits(void *context, unsigned int reg, BUILD_BUG_ON(sizeof(buf_rx->cmd) != sizeof(__be16)); BUILD_BUG_ON(sizeof(buf_tx->cmd) != sizeof(__be16)); - if (IS_ENABLED(CONFIG_CAN_MCP25XXFD_SANITY) && + if (IS_ENABLED(CONFIG_CAN_MCP251XFD_SANITY) && mask == 0) return -EINVAL; - first_byte = mcp25xxfd_first_byte_set(mask); - last_byte = mcp25xxfd_last_byte_set(mask); + first_byte = mcp251xfd_first_byte_set(mask); + last_byte = mcp251xfd_last_byte_set(mask); len = last_byte - first_byte + 1; - if (mcp25xxfd_update_bits_read_reg(reg)) { + if (mcp251xfd_update_bits_read_reg(reg)) { struct spi_transfer xfer[2] = { }; struct spi_message msg; spi_message_init(&msg); spi_message_add_tail(&xfer[0], &msg); - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_HALF_DUPLEX) { + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_HALF_DUPLEX) { xfer[0].tx_buf = buf_tx; xfer[0].len = sizeof(buf_tx->cmd); @@ -111,11 +111,11 @@ mcp25xxfd_regmap_nocrc_update_bits(void *context, unsigned int reg, xfer[0].rx_buf = buf_rx; xfer[0].len = sizeof(buf_tx->cmd) + len; - if (MCP25XXFD_SANITIZE_SPI) + if (MCP251XFD_SANITIZE_SPI) memset(buf_tx->data, 0x0, len); } - mcp25xxfd_spi_cmd_read_nocrc(&buf_tx->cmd, reg + first_byte); + mcp251xfd_spi_cmd_read_nocrc(&buf_tx->cmd, reg + first_byte); err = spi_sync(spi, &msg); if (err) return err; @@ -129,21 +129,21 @@ mcp25xxfd_regmap_nocrc_update_bits(void *context, unsigned int reg, tmp_le32 = orig_le32 & ~mask_le32; tmp_le32 |= val_le32 & mask_le32; - mcp25xxfd_spi_cmd_write_nocrc(&buf_tx->cmd, reg + first_byte); + mcp251xfd_spi_cmd_write_nocrc(&buf_tx->cmd, reg + first_byte); memcpy(buf_tx->data, &tmp_le32, len); return spi_write(spi, buf_tx, sizeof(buf_tx->cmd) + len); } static int -mcp25xxfd_regmap_nocrc_read(void *context, +mcp251xfd_regmap_nocrc_read(void *context, const void *reg, size_t reg_len, void *val_buf, size_t val_len) { struct spi_device *spi = context; - struct mcp25xxfd_priv *priv = spi_get_drvdata(spi); - struct mcp25xxfd_map_buf_nocrc *buf_rx = priv->map_buf_nocrc_rx; - struct mcp25xxfd_map_buf_nocrc *buf_tx = priv->map_buf_nocrc_tx; + struct mcp251xfd_priv *priv = spi_get_drvdata(spi); + struct mcp251xfd_map_buf_nocrc *buf_rx = priv->map_buf_nocrc_rx; + struct mcp251xfd_map_buf_nocrc *buf_tx = priv->map_buf_nocrc_tx; struct spi_transfer xfer[2] = { }; struct spi_message msg; int err; @@ -151,14 +151,14 @@ mcp25xxfd_regmap_nocrc_read(void *context, BUILD_BUG_ON(sizeof(buf_rx->cmd) != sizeof(__be16)); BUILD_BUG_ON(sizeof(buf_tx->cmd) != sizeof(__be16)); - if (IS_ENABLED(CONFIG_CAN_MCP25XXFD_SANITY) && + if (IS_ENABLED(CONFIG_CAN_MCP251XFD_SANITY) && reg_len != sizeof(buf_tx->cmd.cmd)) return -EINVAL; spi_message_init(&msg); spi_message_add_tail(&xfer[0], &msg); - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_HALF_DUPLEX) { + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_HALF_DUPLEX) { xfer[0].tx_buf = reg; xfer[0].len = sizeof(buf_tx->cmd); @@ -171,7 +171,7 @@ mcp25xxfd_regmap_nocrc_read(void *context, xfer[0].len = sizeof(buf_tx->cmd) + val_len; memcpy(&buf_tx->cmd, reg, sizeof(buf_tx->cmd)); - if (MCP25XXFD_SANITIZE_SPI) + if (MCP251XFD_SANITIZE_SPI) memset(buf_tx->data, 0x0, val_len); }; @@ -179,20 +179,20 @@ mcp25xxfd_regmap_nocrc_read(void *context, if (err) return err; - if (!(priv->devtype_data.quirks & MCP25XXFD_QUIRK_HALF_DUPLEX)) + if (!(priv->devtype_data.quirks & MCP251XFD_QUIRK_HALF_DUPLEX)) memcpy(val_buf, buf_rx->data, val_len); return 0; } static int -mcp25xxfd_regmap_crc_gather_write(void *context, +mcp251xfd_regmap_crc_gather_write(void *context, const void *reg_p, size_t reg_len, const void *val, size_t val_len) { struct spi_device *spi = context; - struct mcp25xxfd_priv *priv = spi_get_drvdata(spi); - struct mcp25xxfd_map_buf_crc *buf_tx = priv->map_buf_crc_tx; + struct mcp251xfd_priv *priv = spi_get_drvdata(spi); + struct mcp251xfd_map_buf_crc *buf_tx = priv->map_buf_crc_tx; struct spi_transfer xfer[] = { { .tx_buf = buf_tx, @@ -205,39 +205,39 @@ mcp25xxfd_regmap_crc_gather_write(void *context, BUILD_BUG_ON(sizeof(buf_tx->cmd) != sizeof(__be16) + sizeof(u8)); - if (IS_ENABLED(CONFIG_CAN_MCP25XXFD_SANITY) && + if (IS_ENABLED(CONFIG_CAN_MCP251XFD_SANITY) && reg_len != sizeof(buf_tx->cmd.cmd) + - mcp25xxfd_regmap_crc.pad_bits / BITS_PER_BYTE) + mcp251xfd_regmap_crc.pad_bits / BITS_PER_BYTE) return -EINVAL; - mcp25xxfd_spi_cmd_write_crc(&buf_tx->cmd, reg, val_len); + mcp251xfd_spi_cmd_write_crc(&buf_tx->cmd, reg, val_len); memcpy(buf_tx->data, val, val_len); - crc = mcp25xxfd_crc16_compute(buf_tx, sizeof(buf_tx->cmd) + val_len); + crc = mcp251xfd_crc16_compute(buf_tx, sizeof(buf_tx->cmd) + val_len); put_unaligned_be16(crc, buf_tx->data + val_len); return spi_sync_transfer(spi, xfer, ARRAY_SIZE(xfer)); } static int -mcp25xxfd_regmap_crc_write(void *context, +mcp251xfd_regmap_crc_write(void *context, const void *data, size_t count) { const size_t data_offset = sizeof(__be16) + - mcp25xxfd_regmap_crc.pad_bits / BITS_PER_BYTE; + mcp251xfd_regmap_crc.pad_bits / BITS_PER_BYTE; - return mcp25xxfd_regmap_crc_gather_write(context, + return mcp251xfd_regmap_crc_gather_write(context, data, data_offset, data + data_offset, count - data_offset); } static int -mcp25xxfd_regmap_crc_read_one(struct mcp25xxfd_priv *priv, +mcp251xfd_regmap_crc_read_one(struct mcp251xfd_priv *priv, struct spi_message *msg, unsigned int data_len) { - const struct mcp25xxfd_map_buf_crc *buf_rx = priv->map_buf_crc_rx; - const struct mcp25xxfd_map_buf_crc *buf_tx = priv->map_buf_crc_tx; + const struct mcp251xfd_map_buf_crc *buf_rx = priv->map_buf_crc_rx; + const struct mcp251xfd_map_buf_crc *buf_tx = priv->map_buf_crc_tx; u16 crc_received, crc_calculated; int err; @@ -249,7 +249,7 @@ mcp25xxfd_regmap_crc_read_one(struct mcp25xxfd_priv *priv, return err; crc_received = get_unaligned_be16(buf_rx->data + data_len); - crc_calculated = mcp25xxfd_crc16_compute2(&buf_tx->cmd, + crc_calculated = mcp251xfd_crc16_compute2(&buf_tx->cmd, sizeof(buf_tx->cmd), buf_rx->data, data_len); @@ -260,14 +260,14 @@ mcp25xxfd_regmap_crc_read_one(struct mcp25xxfd_priv *priv, } static int -mcp25xxfd_regmap_crc_read(void *context, +mcp251xfd_regmap_crc_read(void *context, const void *reg_p, size_t reg_len, void *val_buf, size_t val_len) { struct spi_device *spi = context; - struct mcp25xxfd_priv *priv = spi_get_drvdata(spi); - struct mcp25xxfd_map_buf_crc *buf_rx = priv->map_buf_crc_rx; - struct mcp25xxfd_map_buf_crc *buf_tx = priv->map_buf_crc_tx; + struct mcp251xfd_priv *priv = spi_get_drvdata(spi); + struct mcp251xfd_map_buf_crc *buf_rx = priv->map_buf_crc_rx; + struct mcp251xfd_map_buf_crc *buf_tx = priv->map_buf_crc_tx; struct spi_transfer xfer[2] = { }; struct spi_message msg; u16 reg = *(u16 *)reg_p; @@ -276,15 +276,15 @@ mcp25xxfd_regmap_crc_read(void *context, BUILD_BUG_ON(sizeof(buf_rx->cmd) != sizeof(__be16) + sizeof(u8)); BUILD_BUG_ON(sizeof(buf_tx->cmd) != sizeof(__be16) + sizeof(u8)); - if (IS_ENABLED(CONFIG_CAN_MCP25XXFD_SANITY) && + if (IS_ENABLED(CONFIG_CAN_MCP251XFD_SANITY) && reg_len != sizeof(buf_tx->cmd.cmd) + - mcp25xxfd_regmap_crc.pad_bits / BITS_PER_BYTE) + mcp251xfd_regmap_crc.pad_bits / BITS_PER_BYTE) return -EINVAL; spi_message_init(&msg); spi_message_add_tail(&xfer[0], &msg); - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_HALF_DUPLEX) { + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_HALF_DUPLEX) { xfer[0].tx_buf = buf_tx; xfer[0].len = sizeof(buf_tx->cmd); @@ -297,21 +297,21 @@ mcp25xxfd_regmap_crc_read(void *context, xfer[0].len = sizeof(buf_tx->cmd) + val_len + sizeof(buf_tx->crc); - if (MCP25XXFD_SANITIZE_SPI) + if (MCP251XFD_SANITIZE_SPI) memset(buf_tx->data, 0x0, val_len + sizeof(buf_tx->crc)); } - mcp25xxfd_spi_cmd_read_crc(&buf_tx->cmd, reg, val_len); + mcp251xfd_spi_cmd_read_crc(&buf_tx->cmd, reg, val_len); - for (i = 0; i < MCP25XXFD_READ_CRC_RETRIES_MAX; i++) { - err = mcp25xxfd_regmap_crc_read_one(priv, &msg, val_len); + for (i = 0; i < MCP251XFD_READ_CRC_RETRIES_MAX; i++) { + err = mcp251xfd_regmap_crc_read_one(priv, &msg, val_len); if (!err) goto out; if (err != -EBADMSG) return err; - /* MCP25XXFD_REG_OSC is the first ever reg we read from. + /* MCP251XFD_REG_OSC is the first ever reg we read from. * * The chip may be in deep sleep and this SPI transfer * (i.e. the assertion of the CS) will wake the chip @@ -325,7 +325,7 @@ mcp25xxfd_regmap_crc_read(void *context, * to the caller. It will take care of both cases. * */ - if (reg == MCP25XXFD_REG_OSC) { + if (reg == MCP251XFD_REG_OSC) { err = 0; goto out; } @@ -350,88 +350,88 @@ mcp25xxfd_regmap_crc_read(void *context, return 0; } -static const struct regmap_range mcp25xxfd_reg_table_yes_range[] = { +static const struct regmap_range mcp251xfd_reg_table_yes_range[] = { regmap_reg_range(0x000, 0x2ec), /* CAN FD Controller Module SFR */ regmap_reg_range(0x400, 0xbfc), /* RAM */ regmap_reg_range(0xe00, 0xe14), /* MCP2517/18FD SFR */ }; -static const struct regmap_access_table mcp25xxfd_reg_table = { - .yes_ranges = mcp25xxfd_reg_table_yes_range, - .n_yes_ranges = ARRAY_SIZE(mcp25xxfd_reg_table_yes_range), +static const struct regmap_access_table mcp251xfd_reg_table = { + .yes_ranges = mcp251xfd_reg_table_yes_range, + .n_yes_ranges = ARRAY_SIZE(mcp251xfd_reg_table_yes_range), }; -static const struct regmap_config mcp25xxfd_regmap_nocrc = { +static const struct regmap_config mcp251xfd_regmap_nocrc = { .name = "nocrc", .reg_bits = 16, .reg_stride = 4, .pad_bits = 0, .val_bits = 32, .max_register = 0xffc, - .wr_table = &mcp25xxfd_reg_table, - .rd_table = &mcp25xxfd_reg_table, + .wr_table = &mcp251xfd_reg_table, + .rd_table = &mcp251xfd_reg_table, .cache_type = REGCACHE_NONE, .read_flag_mask = (__force unsigned long) - cpu_to_be16(MCP25XXFD_SPI_INSTRUCTION_READ), + cpu_to_be16(MCP251XFD_SPI_INSTRUCTION_READ), .write_flag_mask = (__force unsigned long) - cpu_to_be16(MCP25XXFD_SPI_INSTRUCTION_WRITE), + cpu_to_be16(MCP251XFD_SPI_INSTRUCTION_WRITE), }; -static const struct regmap_bus mcp25xxfd_bus_nocrc = { - .write = mcp25xxfd_regmap_nocrc_write, - .gather_write = mcp25xxfd_regmap_nocrc_gather_write, - .reg_update_bits = mcp25xxfd_regmap_nocrc_update_bits, - .read = mcp25xxfd_regmap_nocrc_read, +static const struct regmap_bus mcp251xfd_bus_nocrc = { + .write = mcp251xfd_regmap_nocrc_write, + .gather_write = mcp251xfd_regmap_nocrc_gather_write, + .reg_update_bits = mcp251xfd_regmap_nocrc_update_bits, + .read = mcp251xfd_regmap_nocrc_read, .reg_format_endian_default = REGMAP_ENDIAN_BIG, .val_format_endian_default = REGMAP_ENDIAN_LITTLE, - .max_raw_read = sizeof_field(struct mcp25xxfd_map_buf_nocrc, data), - .max_raw_write = sizeof_field(struct mcp25xxfd_map_buf_nocrc, data), + .max_raw_read = sizeof_field(struct mcp251xfd_map_buf_nocrc, data), + .max_raw_write = sizeof_field(struct mcp251xfd_map_buf_nocrc, data), }; -static const struct regmap_config mcp25xxfd_regmap_crc = { +static const struct regmap_config mcp251xfd_regmap_crc = { .name = "crc", .reg_bits = 16, .reg_stride = 4, .pad_bits = 16, /* keep data bits aligned */ .val_bits = 32, .max_register = 0xffc, - .wr_table = &mcp25xxfd_reg_table, - .rd_table = &mcp25xxfd_reg_table, + .wr_table = &mcp251xfd_reg_table, + .rd_table = &mcp251xfd_reg_table, .cache_type = REGCACHE_NONE, }; -static const struct regmap_bus mcp25xxfd_bus_crc = { - .write = mcp25xxfd_regmap_crc_write, - .gather_write = mcp25xxfd_regmap_crc_gather_write, - .read = mcp25xxfd_regmap_crc_read, +static const struct regmap_bus mcp251xfd_bus_crc = { + .write = mcp251xfd_regmap_crc_write, + .gather_write = mcp251xfd_regmap_crc_gather_write, + .read = mcp251xfd_regmap_crc_read, .reg_format_endian_default = REGMAP_ENDIAN_NATIVE, .val_format_endian_default = REGMAP_ENDIAN_LITTLE, - .max_raw_read = sizeof_field(struct mcp25xxfd_map_buf_crc, data), - .max_raw_write = sizeof_field(struct mcp25xxfd_map_buf_crc, data), + .max_raw_read = sizeof_field(struct mcp251xfd_map_buf_crc, data), + .max_raw_write = sizeof_field(struct mcp251xfd_map_buf_crc, data), }; static inline bool -mcp25xxfd_regmap_use_nocrc(struct mcp25xxfd_priv *priv) +mcp251xfd_regmap_use_nocrc(struct mcp251xfd_priv *priv) { - return (!(priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_REG)) || - (!(priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_RX)); + return (!(priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_REG)) || + (!(priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_RX)); } static inline bool -mcp25xxfd_regmap_use_crc(struct mcp25xxfd_priv *priv) +mcp251xfd_regmap_use_crc(struct mcp251xfd_priv *priv) { - return (priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_REG) || - (priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_RX); + return (priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_REG) || + (priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_RX); } static int -mcp25xxfd_regmap_init_nocrc(struct mcp25xxfd_priv *priv) +mcp251xfd_regmap_init_nocrc(struct mcp251xfd_priv *priv) { if (!priv->map_nocrc) { struct regmap *map; - map = devm_regmap_init(&priv->spi->dev, &mcp25xxfd_bus_nocrc, - priv->spi, &mcp25xxfd_regmap_nocrc); + map = devm_regmap_init(&priv->spi->dev, &mcp251xfd_bus_nocrc, + priv->spi, &mcp251xfd_regmap_nocrc); if (IS_ERR(map)) return PTR_ERR(map); @@ -456,16 +456,16 @@ mcp25xxfd_regmap_init_nocrc(struct mcp25xxfd_priv *priv) return -ENOMEM; } - if (!(priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_REG)) + if (!(priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_REG)) priv->map_reg = priv->map_nocrc; - if (!(priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_RX)) + if (!(priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_RX)) priv->map_rx = priv->map_nocrc; return 0; } -static void mcp25xxfd_regmap_destroy_nocrc(struct mcp25xxfd_priv *priv) +static void mcp251xfd_regmap_destroy_nocrc(struct mcp251xfd_priv *priv) { if (priv->map_buf_nocrc_rx) { devm_kfree(&priv->spi->dev, priv->map_buf_nocrc_rx); @@ -478,13 +478,13 @@ static void mcp25xxfd_regmap_destroy_nocrc(struct mcp25xxfd_priv *priv) } static int -mcp25xxfd_regmap_init_crc(struct mcp25xxfd_priv *priv) +mcp251xfd_regmap_init_crc(struct mcp251xfd_priv *priv) { if (!priv->map_crc) { struct regmap *map; - map = devm_regmap_init(&priv->spi->dev, &mcp25xxfd_bus_crc, - priv->spi, &mcp25xxfd_regmap_crc); + map = devm_regmap_init(&priv->spi->dev, &mcp251xfd_bus_crc, + priv->spi, &mcp251xfd_regmap_crc); if (IS_ERR(map)) return PTR_ERR(map); @@ -509,16 +509,16 @@ mcp25xxfd_regmap_init_crc(struct mcp25xxfd_priv *priv) return -ENOMEM; } - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_REG) + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_REG) priv->map_reg = priv->map_crc; - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_RX) + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_RX) priv->map_rx = priv->map_crc; return 0; } -static void mcp25xxfd_regmap_destroy_crc(struct mcp25xxfd_priv *priv) +static void mcp251xfd_regmap_destroy_crc(struct mcp251xfd_priv *priv) { if (priv->map_buf_crc_rx) { devm_kfree(&priv->spi->dev, priv->map_buf_crc_rx); @@ -530,26 +530,26 @@ static void mcp25xxfd_regmap_destroy_crc(struct mcp25xxfd_priv *priv) } } -int mcp25xxfd_regmap_init(struct mcp25xxfd_priv *priv) +int mcp251xfd_regmap_init(struct mcp251xfd_priv *priv) { int err; - if (mcp25xxfd_regmap_use_nocrc(priv)) { - err = mcp25xxfd_regmap_init_nocrc(priv); + if (mcp251xfd_regmap_use_nocrc(priv)) { + err = mcp251xfd_regmap_init_nocrc(priv); if (err) return err; } else { - mcp25xxfd_regmap_destroy_nocrc(priv); + mcp251xfd_regmap_destroy_nocrc(priv); } - if (mcp25xxfd_regmap_use_crc(priv)) { - err = mcp25xxfd_regmap_init_crc(priv); + if (mcp251xfd_regmap_use_crc(priv)) { + err = mcp251xfd_regmap_init_crc(priv); if (err) return err; } else { - mcp25xxfd_regmap_destroy_crc(priv); + mcp251xfd_regmap_destroy_crc(priv); } return 0; diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd.h b/drivers/net/can/spi/mcp251xfd/mcp251xfd.h new file mode 100644 index 000000000000..fa1246e39980 --- /dev/null +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd.h @@ -0,0 +1,835 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * mcp251xfd - Microchip MCP251xFD Family CAN controller driver + * + * Copyright (c) 2019 Pengutronix, + * Marc Kleine-Budde <kernel@pengutronix.de> + * Copyright (c) 2019 Martin Sperl <kernel@martin.sperl.org> + */ + +#ifndef _MCP251XFD_H +#define _MCP251XFD_H + +#include <linux/can/core.h> +#include <linux/can/dev.h> +#include <linux/can/rx-offload.h> +#include <linux/gpio/consumer.h> +#include <linux/kernel.h> +#include <linux/regmap.h> +#include <linux/regulator/consumer.h> +#include <linux/spi/spi.h> + +/* MPC251x registers */ + +/* CAN FD Controller Module SFR */ +#define MCP251XFD_REG_CON 0x00 +#define MCP251XFD_REG_CON_TXBWS_MASK GENMASK(31, 28) +#define MCP251XFD_REG_CON_ABAT BIT(27) +#define MCP251XFD_REG_CON_REQOP_MASK GENMASK(26, 24) +#define MCP251XFD_REG_CON_MODE_MIXED 0 +#define MCP251XFD_REG_CON_MODE_SLEEP 1 +#define MCP251XFD_REG_CON_MODE_INT_LOOPBACK 2 +#define MCP251XFD_REG_CON_MODE_LISTENONLY 3 +#define MCP251XFD_REG_CON_MODE_CONFIG 4 +#define MCP251XFD_REG_CON_MODE_EXT_LOOPBACK 5 +#define MCP251XFD_REG_CON_MODE_CAN2_0 6 +#define MCP251XFD_REG_CON_MODE_RESTRICTED 7 +#define MCP251XFD_REG_CON_OPMOD_MASK GENMASK(23, 21) +#define MCP251XFD_REG_CON_TXQEN BIT(20) +#define MCP251XFD_REG_CON_STEF BIT(19) +#define MCP251XFD_REG_CON_SERR2LOM BIT(18) +#define MCP251XFD_REG_CON_ESIGM BIT(17) +#define MCP251XFD_REG_CON_RTXAT BIT(16) +#define MCP251XFD_REG_CON_BRSDIS BIT(12) +#define MCP251XFD_REG_CON_BUSY BIT(11) +#define MCP251XFD_REG_CON_WFT_MASK GENMASK(10, 9) +#define MCP251XFD_REG_CON_WFT_T00FILTER 0x0 +#define MCP251XFD_REG_CON_WFT_T01FILTER 0x1 +#define MCP251XFD_REG_CON_WFT_T10FILTER 0x2 +#define MCP251XFD_REG_CON_WFT_T11FILTER 0x3 +#define MCP251XFD_REG_CON_WAKFIL BIT(8) +#define MCP251XFD_REG_CON_PXEDIS BIT(6) +#define MCP251XFD_REG_CON_ISOCRCEN BIT(5) +#define MCP251XFD_REG_CON_DNCNT_MASK GENMASK(4, 0) + +#define MCP251XFD_REG_NBTCFG 0x04 +#define MCP251XFD_REG_NBTCFG_BRP_MASK GENMASK(31, 24) +#define MCP251XFD_REG_NBTCFG_TSEG1_MASK GENMASK(23, 16) +#define MCP251XFD_REG_NBTCFG_TSEG2_MASK GENMASK(14, 8) +#define MCP251XFD_REG_NBTCFG_SJW_MASK GENMASK(6, 0) + +#define MCP251XFD_REG_DBTCFG 0x08 +#define MCP251XFD_REG_DBTCFG_BRP_MASK GENMASK(31, 24) +#define MCP251XFD_REG_DBTCFG_TSEG1_MASK GENMASK(20, 16) +#define MCP251XFD_REG_DBTCFG_TSEG2_MASK GENMASK(11, 8) +#define MCP251XFD_REG_DBTCFG_SJW_MASK GENMASK(3, 0) + +#define MCP251XFD_REG_TDC 0x0c +#define MCP251XFD_REG_TDC_EDGFLTEN BIT(25) +#define MCP251XFD_REG_TDC_SID11EN BIT(24) +#define MCP251XFD_REG_TDC_TDCMOD_MASK GENMASK(17, 16) +#define MCP251XFD_REG_TDC_TDCMOD_AUTO 2 +#define MCP251XFD_REG_TDC_TDCMOD_MANUAL 1 +#define MCP251XFD_REG_TDC_TDCMOD_DISABLED 0 +#define MCP251XFD_REG_TDC_TDCO_MASK GENMASK(14, 8) +#define MCP251XFD_REG_TDC_TDCV_MASK GENMASK(5, 0) + +#define MCP251XFD_REG_TBC 0x10 + +#define MCP251XFD_REG_TSCON 0x14 +#define MCP251XFD_REG_TSCON_TSRES BIT(18) +#define MCP251XFD_REG_TSCON_TSEOF BIT(17) +#define MCP251XFD_REG_TSCON_TBCEN BIT(16) +#define MCP251XFD_REG_TSCON_TBCPRE_MASK GENMASK(9, 0) + +#define MCP251XFD_REG_VEC 0x18 +#define MCP251XFD_REG_VEC_RXCODE_MASK GENMASK(30, 24) +#define MCP251XFD_REG_VEC_TXCODE_MASK GENMASK(22, 16) +#define MCP251XFD_REG_VEC_FILHIT_MASK GENMASK(12, 8) +#define MCP251XFD_REG_VEC_ICODE_MASK GENMASK(6, 0) + +#define MCP251XFD_REG_INT 0x1c +#define MCP251XFD_REG_INT_IF_MASK GENMASK(15, 0) +#define MCP251XFD_REG_INT_IE_MASK GENMASK(31, 16) +#define MCP251XFD_REG_INT_IVMIE BIT(31) +#define MCP251XFD_REG_INT_WAKIE BIT(30) +#define MCP251XFD_REG_INT_CERRIE BIT(29) +#define MCP251XFD_REG_INT_SERRIE BIT(28) +#define MCP251XFD_REG_INT_RXOVIE BIT(27) +#define MCP251XFD_REG_INT_TXATIE BIT(26) +#define MCP251XFD_REG_INT_SPICRCIE BIT(25) +#define MCP251XFD_REG_INT_ECCIE BIT(24) +#define MCP251XFD_REG_INT_TEFIE BIT(20) +#define MCP251XFD_REG_INT_MODIE BIT(19) +#define MCP251XFD_REG_INT_TBCIE BIT(18) +#define MCP251XFD_REG_INT_RXIE BIT(17) +#define MCP251XFD_REG_INT_TXIE BIT(16) +#define MCP251XFD_REG_INT_IVMIF BIT(15) +#define MCP251XFD_REG_INT_WAKIF BIT(14) +#define MCP251XFD_REG_INT_CERRIF BIT(13) +#define MCP251XFD_REG_INT_SERRIF BIT(12) +#define MCP251XFD_REG_INT_RXOVIF BIT(11) +#define MCP251XFD_REG_INT_TXATIF BIT(10) +#define MCP251XFD_REG_INT_SPICRCIF BIT(9) +#define MCP251XFD_REG_INT_ECCIF BIT(8) +#define MCP251XFD_REG_INT_TEFIF BIT(4) +#define MCP251XFD_REG_INT_MODIF BIT(3) +#define MCP251XFD_REG_INT_TBCIF BIT(2) +#define MCP251XFD_REG_INT_RXIF BIT(1) +#define MCP251XFD_REG_INT_TXIF BIT(0) +/* These IRQ flags must be cleared by SW in the CAN_INT register */ +#define MCP251XFD_REG_INT_IF_CLEARABLE_MASK \ + (MCP251XFD_REG_INT_IVMIF | MCP251XFD_REG_INT_WAKIF | \ + MCP251XFD_REG_INT_CERRIF | MCP251XFD_REG_INT_SERRIF | \ + MCP251XFD_REG_INT_MODIF) + +#define MCP251XFD_REG_RXIF 0x20 +#define MCP251XFD_REG_TXIF 0x24 +#define MCP251XFD_REG_RXOVIF 0x28 +#define MCP251XFD_REG_TXATIF 0x2c +#define MCP251XFD_REG_TXREQ 0x30 + +#define MCP251XFD_REG_TREC 0x34 +#define MCP251XFD_REG_TREC_TXBO BIT(21) +#define MCP251XFD_REG_TREC_TXBP BIT(20) +#define MCP251XFD_REG_TREC_RXBP BIT(19) +#define MCP251XFD_REG_TREC_TXWARN BIT(18) +#define MCP251XFD_REG_TREC_RXWARN BIT(17) +#define MCP251XFD_REG_TREC_EWARN BIT(16) +#define MCP251XFD_REG_TREC_TEC_MASK GENMASK(15, 8) +#define MCP251XFD_REG_TREC_REC_MASK GENMASK(7, 0) + +#define MCP251XFD_REG_BDIAG0 0x38 +#define MCP251XFD_REG_BDIAG0_DTERRCNT_MASK GENMASK(31, 24) +#define MCP251XFD_REG_BDIAG0_DRERRCNT_MASK GENMASK(23, 16) +#define MCP251XFD_REG_BDIAG0_NTERRCNT_MASK GENMASK(15, 8) +#define MCP251XFD_REG_BDIAG0_NRERRCNT_MASK GENMASK(7, 0) + +#define MCP251XFD_REG_BDIAG1 0x3c +#define MCP251XFD_REG_BDIAG1_DLCMM BIT(31) +#define MCP251XFD_REG_BDIAG1_ESI BIT(30) +#define MCP251XFD_REG_BDIAG1_DCRCERR BIT(29) +#define MCP251XFD_REG_BDIAG1_DSTUFERR BIT(28) +#define MCP251XFD_REG_BDIAG1_DFORMERR BIT(27) +#define MCP251XFD_REG_BDIAG1_DBIT1ERR BIT(25) +#define MCP251XFD_REG_BDIAG1_DBIT0ERR BIT(24) +#define MCP251XFD_REG_BDIAG1_TXBOERR BIT(23) +#define MCP251XFD_REG_BDIAG1_NCRCERR BIT(21) +#define MCP251XFD_REG_BDIAG1_NSTUFERR BIT(20) +#define MCP251XFD_REG_BDIAG1_NFORMERR BIT(19) +#define MCP251XFD_REG_BDIAG1_NACKERR BIT(18) +#define MCP251XFD_REG_BDIAG1_NBIT1ERR BIT(17) +#define MCP251XFD_REG_BDIAG1_NBIT0ERR BIT(16) +#define MCP251XFD_REG_BDIAG1_BERR_MASK \ + (MCP251XFD_REG_BDIAG1_DLCMM | MCP251XFD_REG_BDIAG1_ESI | \ + MCP251XFD_REG_BDIAG1_DCRCERR | MCP251XFD_REG_BDIAG1_DSTUFERR | \ + MCP251XFD_REG_BDIAG1_DFORMERR | MCP251XFD_REG_BDIAG1_DBIT1ERR | \ + MCP251XFD_REG_BDIAG1_DBIT0ERR | MCP251XFD_REG_BDIAG1_TXBOERR | \ + MCP251XFD_REG_BDIAG1_NCRCERR | MCP251XFD_REG_BDIAG1_NSTUFERR | \ + MCP251XFD_REG_BDIAG1_NFORMERR | MCP251XFD_REG_BDIAG1_NACKERR | \ + MCP251XFD_REG_BDIAG1_NBIT1ERR | MCP251XFD_REG_BDIAG1_NBIT0ERR) +#define MCP251XFD_REG_BDIAG1_EFMSGCNT_MASK GENMASK(15, 0) + +#define MCP251XFD_REG_TEFCON 0x40 +#define MCP251XFD_REG_TEFCON_FSIZE_MASK GENMASK(28, 24) +#define MCP251XFD_REG_TEFCON_FRESET BIT(10) +#define MCP251XFD_REG_TEFCON_UINC BIT(8) +#define MCP251XFD_REG_TEFCON_TEFTSEN BIT(5) +#define MCP251XFD_REG_TEFCON_TEFOVIE BIT(3) +#define MCP251XFD_REG_TEFCON_TEFFIE BIT(2) +#define MCP251XFD_REG_TEFCON_TEFHIE BIT(1) +#define MCP251XFD_REG_TEFCON_TEFNEIE BIT(0) + +#define MCP251XFD_REG_TEFSTA 0x44 +#define MCP251XFD_REG_TEFSTA_TEFOVIF BIT(3) +#define MCP251XFD_REG_TEFSTA_TEFFIF BIT(2) +#define MCP251XFD_REG_TEFSTA_TEFHIF BIT(1) +#define MCP251XFD_REG_TEFSTA_TEFNEIF BIT(0) + +#define MCP251XFD_REG_TEFUA 0x48 + +#define MCP251XFD_REG_TXQCON 0x50 +#define MCP251XFD_REG_TXQCON_PLSIZE_MASK GENMASK(31, 29) +#define MCP251XFD_REG_TXQCON_PLSIZE_8 0 +#define MCP251XFD_REG_TXQCON_PLSIZE_12 1 +#define MCP251XFD_REG_TXQCON_PLSIZE_16 2 +#define MCP251XFD_REG_TXQCON_PLSIZE_20 3 +#define MCP251XFD_REG_TXQCON_PLSIZE_24 4 +#define MCP251XFD_REG_TXQCON_PLSIZE_32 5 +#define MCP251XFD_REG_TXQCON_PLSIZE_48 6 +#define MCP251XFD_REG_TXQCON_PLSIZE_64 7 +#define MCP251XFD_REG_TXQCON_FSIZE_MASK GENMASK(28, 24) +#define MCP251XFD_REG_TXQCON_TXAT_UNLIMITED 3 +#define MCP251XFD_REG_TXQCON_TXAT_THREE_SHOT 1 +#define MCP251XFD_REG_TXQCON_TXAT_ONE_SHOT 0 +#define MCP251XFD_REG_TXQCON_TXAT_MASK GENMASK(22, 21) +#define MCP251XFD_REG_TXQCON_TXPRI_MASK GENMASK(20, 16) +#define MCP251XFD_REG_TXQCON_FRESET BIT(10) +#define MCP251XFD_REG_TXQCON_TXREQ BIT(9) +#define MCP251XFD_REG_TXQCON_UINC BIT(8) +#define MCP251XFD_REG_TXQCON_TXEN BIT(7) +#define MCP251XFD_REG_TXQCON_TXATIE BIT(4) +#define MCP251XFD_REG_TXQCON_TXQEIE BIT(2) +#define MCP251XFD_REG_TXQCON_TXQNIE BIT(0) + +#define MCP251XFD_REG_TXQSTA 0x54 +#define MCP251XFD_REG_TXQSTA_TXQCI_MASK GENMASK(12, 8) +#define MCP251XFD_REG_TXQSTA_TXABT BIT(7) +#define MCP251XFD_REG_TXQSTA_TXLARB BIT(6) +#define MCP251XFD_REG_TXQSTA_TXERR BIT(5) +#define MCP251XFD_REG_TXQSTA_TXATIF BIT(4) +#define MCP251XFD_REG_TXQSTA_TXQEIF BIT(2) +#define MCP251XFD_REG_TXQSTA_TXQNIF BIT(0) + +#define MCP251XFD_REG_TXQUA 0x58 + +#define MCP251XFD_REG_FIFOCON(x) (0x50 + 0xc * (x)) +#define MCP251XFD_REG_FIFOCON_PLSIZE_MASK GENMASK(31, 29) +#define MCP251XFD_REG_FIFOCON_PLSIZE_8 0 +#define MCP251XFD_REG_FIFOCON_PLSIZE_12 1 +#define MCP251XFD_REG_FIFOCON_PLSIZE_16 2 +#define MCP251XFD_REG_FIFOCON_PLSIZE_20 3 +#define MCP251XFD_REG_FIFOCON_PLSIZE_24 4 +#define MCP251XFD_REG_FIFOCON_PLSIZE_32 5 +#define MCP251XFD_REG_FIFOCON_PLSIZE_48 6 +#define MCP251XFD_REG_FIFOCON_PLSIZE_64 7 +#define MCP251XFD_REG_FIFOCON_FSIZE_MASK GENMASK(28, 24) +#define MCP251XFD_REG_FIFOCON_TXAT_MASK GENMASK(22, 21) +#define MCP251XFD_REG_FIFOCON_TXAT_ONE_SHOT 0 +#define MCP251XFD_REG_FIFOCON_TXAT_THREE_SHOT 1 +#define MCP251XFD_REG_FIFOCON_TXAT_UNLIMITED 3 +#define MCP251XFD_REG_FIFOCON_TXPRI_MASK GENMASK(20, 16) +#define MCP251XFD_REG_FIFOCON_FRESET BIT(10) +#define MCP251XFD_REG_FIFOCON_TXREQ BIT(9) +#define MCP251XFD_REG_FIFOCON_UINC BIT(8) +#define MCP251XFD_REG_FIFOCON_TXEN BIT(7) +#define MCP251XFD_REG_FIFOCON_RTREN BIT(6) +#define MCP251XFD_REG_FIFOCON_RXTSEN BIT(5) +#define MCP251XFD_REG_FIFOCON_TXATIE BIT(4) +#define MCP251XFD_REG_FIFOCON_RXOVIE BIT(3) +#define MCP251XFD_REG_FIFOCON_TFERFFIE BIT(2) +#define MCP251XFD_REG_FIFOCON_TFHRFHIE BIT(1) +#define MCP251XFD_REG_FIFOCON_TFNRFNIE BIT(0) + +#define MCP251XFD_REG_FIFOSTA(x) (0x54 + 0xc * (x)) +#define MCP251XFD_REG_FIFOSTA_FIFOCI_MASK GENMASK(12, 8) +#define MCP251XFD_REG_FIFOSTA_TXABT BIT(7) +#define MCP251XFD_REG_FIFOSTA_TXLARB BIT(6) +#define MCP251XFD_REG_FIFOSTA_TXERR BIT(5) +#define MCP251XFD_REG_FIFOSTA_TXATIF BIT(4) +#define MCP251XFD_REG_FIFOSTA_RXOVIF BIT(3) +#define MCP251XFD_REG_FIFOSTA_TFERFFIF BIT(2) +#define MCP251XFD_REG_FIFOSTA_TFHRFHIF BIT(1) +#define MCP251XFD_REG_FIFOSTA_TFNRFNIF BIT(0) + +#define MCP251XFD_REG_FIFOUA(x) (0x58 + 0xc * (x)) + +#define MCP251XFD_REG_FLTCON(x) (0x1d0 + 0x4 * (x)) +#define MCP251XFD_REG_FLTCON_FLTEN3 BIT(31) +#define MCP251XFD_REG_FLTCON_F3BP_MASK GENMASK(28, 24) +#define MCP251XFD_REG_FLTCON_FLTEN2 BIT(23) +#define MCP251XFD_REG_FLTCON_F2BP_MASK GENMASK(20, 16) +#define MCP251XFD_REG_FLTCON_FLTEN1 BIT(15) +#define MCP251XFD_REG_FLTCON_F1BP_MASK GENMASK(12, 8) +#define MCP251XFD_REG_FLTCON_FLTEN0 BIT(7) +#define MCP251XFD_REG_FLTCON_F0BP_MASK GENMASK(4, 0) +#define MCP251XFD_REG_FLTCON_FLTEN(x) (BIT(7) << 8 * ((x) & 0x3)) +#define MCP251XFD_REG_FLTCON_FLT_MASK(x) (GENMASK(7, 0) << (8 * ((x) & 0x3))) +#define MCP251XFD_REG_FLTCON_FBP(x, fifo) ((fifo) << 8 * ((x) & 0x3)) + +#define MCP251XFD_REG_FLTOBJ(x) (0x1f0 + 0x8 * (x)) +#define MCP251XFD_REG_FLTOBJ_EXIDE BIT(30) +#define MCP251XFD_REG_FLTOBJ_SID11 BIT(29) +#define MCP251XFD_REG_FLTOBJ_EID_MASK GENMASK(28, 11) +#define MCP251XFD_REG_FLTOBJ_SID_MASK GENMASK(10, 0) + +#define MCP251XFD_REG_FLTMASK(x) (0x1f4 + 0x8 * (x)) +#define MCP251XFD_REG_MASK_MIDE BIT(30) +#define MCP251XFD_REG_MASK_MSID11 BIT(29) +#define MCP251XFD_REG_MASK_MEID_MASK GENMASK(28, 11) +#define MCP251XFD_REG_MASK_MSID_MASK GENMASK(10, 0) + +/* RAM */ +#define MCP251XFD_RAM_START 0x400 +#define MCP251XFD_RAM_SIZE SZ_2K + +/* Message Object */ +#define MCP251XFD_OBJ_ID_SID11 BIT(29) +#define MCP251XFD_OBJ_ID_EID_MASK GENMASK(28, 11) +#define MCP251XFD_OBJ_ID_SID_MASK GENMASK(10, 0) +#define MCP251XFD_OBJ_FLAGS_SEQ_MCP2518FD_MASK GENMASK(31, 9) +#define MCP251XFD_OBJ_FLAGS_SEQ_MCP2517FD_MASK GENMASK(15, 9) +#define MCP251XFD_OBJ_FLAGS_SEQ_MASK MCP251XFD_OBJ_FLAGS_SEQ_MCP2518FD_MASK +#define MCP251XFD_OBJ_FLAGS_ESI BIT(8) +#define MCP251XFD_OBJ_FLAGS_FDF BIT(7) +#define MCP251XFD_OBJ_FLAGS_BRS BIT(6) +#define MCP251XFD_OBJ_FLAGS_RTR BIT(5) +#define MCP251XFD_OBJ_FLAGS_IDE BIT(4) +#define MCP251XFD_OBJ_FLAGS_DLC GENMASK(3, 0) + +#define MCP251XFD_REG_FRAME_EFF_SID_MASK GENMASK(28, 18) +#define MCP251XFD_REG_FRAME_EFF_EID_MASK GENMASK(17, 0) + +/* MCP2517/18FD SFR */ +#define MCP251XFD_REG_OSC 0xe00 +#define MCP251XFD_REG_OSC_SCLKRDY BIT(12) +#define MCP251XFD_REG_OSC_OSCRDY BIT(10) +#define MCP251XFD_REG_OSC_PLLRDY BIT(8) +#define MCP251XFD_REG_OSC_CLKODIV_10 3 +#define MCP251XFD_REG_OSC_CLKODIV_4 2 +#define MCP251XFD_REG_OSC_CLKODIV_2 1 +#define MCP251XFD_REG_OSC_CLKODIV_1 0 +#define MCP251XFD_REG_OSC_CLKODIV_MASK GENMASK(6, 5) +#define MCP251XFD_REG_OSC_SCLKDIV BIT(4) +#define MCP251XFD_REG_OSC_LPMEN BIT(3) /* MCP2518FD only */ +#define MCP251XFD_REG_OSC_OSCDIS BIT(2) +#define MCP251XFD_REG_OSC_PLLEN BIT(0) + +#define MCP251XFD_REG_IOCON 0xe04 +#define MCP251XFD_REG_IOCON_INTOD BIT(30) +#define MCP251XFD_REG_IOCON_SOF BIT(29) +#define MCP251XFD_REG_IOCON_TXCANOD BIT(28) +#define MCP251XFD_REG_IOCON_PM1 BIT(25) +#define MCP251XFD_REG_IOCON_PM0 BIT(24) +#define MCP251XFD_REG_IOCON_GPIO1 BIT(17) +#define MCP251XFD_REG_IOCON_GPIO0 BIT(16) +#define MCP251XFD_REG_IOCON_LAT1 BIT(9) +#define MCP251XFD_REG_IOCON_LAT0 BIT(8) +#define MCP251XFD_REG_IOCON_XSTBYEN BIT(6) +#define MCP251XFD_REG_IOCON_TRIS1 BIT(1) +#define MCP251XFD_REG_IOCON_TRIS0 BIT(0) + +#define MCP251XFD_REG_CRC 0xe08 +#define MCP251XFD_REG_CRC_FERRIE BIT(25) +#define MCP251XFD_REG_CRC_CRCERRIE BIT(24) +#define MCP251XFD_REG_CRC_FERRIF BIT(17) +#define MCP251XFD_REG_CRC_CRCERRIF BIT(16) +#define MCP251XFD_REG_CRC_IF_MASK GENMASK(17, 16) +#define MCP251XFD_REG_CRC_MASK GENMASK(15, 0) + +#define MCP251XFD_REG_ECCCON 0xe0c +#define MCP251XFD_REG_ECCCON_PARITY_MASK GENMASK(14, 8) +#define MCP251XFD_REG_ECCCON_DEDIE BIT(2) +#define MCP251XFD_REG_ECCCON_SECIE BIT(1) +#define MCP251XFD_REG_ECCCON_ECCEN BIT(0) + +#define MCP251XFD_REG_ECCSTAT 0xe10 +#define MCP251XFD_REG_ECCSTAT_ERRADDR_MASK GENMASK(27, 16) +#define MCP251XFD_REG_ECCSTAT_IF_MASK GENMASK(2, 1) +#define MCP251XFD_REG_ECCSTAT_DEDIF BIT(2) +#define MCP251XFD_REG_ECCSTAT_SECIF BIT(1) + +#define MCP251XFD_REG_DEVID 0xe14 /* MCP2518FD only */ +#define MCP251XFD_REG_DEVID_ID_MASK GENMASK(7, 4) +#define MCP251XFD_REG_DEVID_REV_MASK GENMASK(3, 0) + +/* number of TX FIFO objects, depending on CAN mode + * + * FIFO setup: tef: 8*12 bytes = 96 bytes, tx: 8*16 bytes = 128 bytes + * FIFO setup: tef: 4*12 bytes = 48 bytes, tx: 4*72 bytes = 288 bytes + */ +#define MCP251XFD_TX_OBJ_NUM_CAN 8 +#define MCP251XFD_TX_OBJ_NUM_CANFD 4 + +#if MCP251XFD_TX_OBJ_NUM_CAN > MCP251XFD_TX_OBJ_NUM_CANFD +#define MCP251XFD_TX_OBJ_NUM_MAX MCP251XFD_TX_OBJ_NUM_CAN +#else +#define MCP251XFD_TX_OBJ_NUM_MAX MCP251XFD_TX_OBJ_NUM_CANFD +#endif + +#define MCP251XFD_NAPI_WEIGHT 32 +#define MCP251XFD_TX_FIFO 1 +#define MCP251XFD_RX_FIFO(x) (MCP251XFD_TX_FIFO + 1 + (x)) + +/* SPI commands */ +#define MCP251XFD_SPI_INSTRUCTION_RESET 0x0000 +#define MCP251XFD_SPI_INSTRUCTION_WRITE 0x2000 +#define MCP251XFD_SPI_INSTRUCTION_READ 0x3000 +#define MCP251XFD_SPI_INSTRUCTION_WRITE_CRC 0xa000 +#define MCP251XFD_SPI_INSTRUCTION_READ_CRC 0xb000 +#define MCP251XFD_SPI_INSTRUCTION_WRITE_CRC_SAFE 0xc000 +#define MCP251XFD_SPI_ADDRESS_MASK GENMASK(11, 0) + +#define MCP251XFD_SYSCLOCK_HZ_MAX 40000000 +#define MCP251XFD_SYSCLOCK_HZ_MIN 1000000 +#define MCP251XFD_SPICLOCK_HZ_MAX 20000000 +#define MCP251XFD_OSC_PLL_MULTIPLIER 10 +#define MCP251XFD_OSC_STAB_SLEEP_US (3 * USEC_PER_MSEC) +#define MCP251XFD_OSC_STAB_TIMEOUT_US (10 * MCP251XFD_OSC_STAB_SLEEP_US) +#define MCP251XFD_POLL_SLEEP_US (10) +#define MCP251XFD_POLL_TIMEOUT_US (USEC_PER_MSEC) +#define MCP251XFD_SOFTRESET_RETRIES_MAX 3 +#define MCP251XFD_READ_CRC_RETRIES_MAX 3 +#define MCP251XFD_ECC_CNT_MAX 2 +#define MCP251XFD_SANITIZE_SPI 1 +#define MCP251XFD_SANITIZE_CAN 1 + +/* Silence TX MAB overflow warnings */ +#define MCP251XFD_QUIRK_MAB_NO_WARN BIT(0) +/* Use CRC to access registers */ +#define MCP251XFD_QUIRK_CRC_REG BIT(1) +/* Use CRC to access RX/TEF-RAM */ +#define MCP251XFD_QUIRK_CRC_RX BIT(2) +/* Use CRC to access TX-RAM */ +#define MCP251XFD_QUIRK_CRC_TX BIT(3) +/* Enable ECC for RAM */ +#define MCP251XFD_QUIRK_ECC BIT(4) +/* Use Half Duplex SPI transfers */ +#define MCP251XFD_QUIRK_HALF_DUPLEX BIT(5) + +struct mcp251xfd_hw_tef_obj { + u32 id; + u32 flags; + u32 ts; +}; + +/* The tx_obj_raw version is used in spi async, i.e. without + * regmap. We have to take care of endianness ourselves. + */ +struct mcp251xfd_hw_tx_obj_raw { + __le32 id; + __le32 flags; + u8 data[sizeof_field(struct canfd_frame, data)]; +}; + +struct mcp251xfd_hw_tx_obj_can { + u32 id; + u32 flags; + u8 data[sizeof_field(struct can_frame, data)]; +}; + +struct mcp251xfd_hw_tx_obj_canfd { + u32 id; + u32 flags; + u8 data[sizeof_field(struct canfd_frame, data)]; +}; + +struct mcp251xfd_hw_rx_obj_can { + u32 id; + u32 flags; + u32 ts; + u8 data[sizeof_field(struct can_frame, data)]; +}; + +struct mcp251xfd_hw_rx_obj_canfd { + u32 id; + u32 flags; + u32 ts; + u8 data[sizeof_field(struct canfd_frame, data)]; +}; + +struct mcp251xfd_tef_ring { + unsigned int head; + unsigned int tail; + + /* u8 obj_num equals tx_ring->obj_num */ + /* u8 obj_size equals sizeof(struct mcp251xfd_hw_tef_obj) */ +}; + +struct __packed mcp251xfd_buf_cmd { + __be16 cmd; +}; + +struct __packed mcp251xfd_buf_cmd_crc { + __be16 cmd; + u8 len; +}; + +union mcp251xfd_tx_obj_load_buf { + struct __packed { + struct mcp251xfd_buf_cmd cmd; + struct mcp251xfd_hw_tx_obj_raw hw_tx_obj; + } nocrc; + struct __packed { + struct mcp251xfd_buf_cmd_crc cmd; + struct mcp251xfd_hw_tx_obj_raw hw_tx_obj; + __be16 crc; + } crc; +} ____cacheline_aligned; + +union mcp251xfd_write_reg_buf { + struct __packed { + struct mcp251xfd_buf_cmd cmd; + u8 data[4]; + } nocrc; + struct __packed { + struct mcp251xfd_buf_cmd_crc cmd; + u8 data[4]; + __be16 crc; + } crc; +} ____cacheline_aligned; + +struct mcp251xfd_tx_obj { + struct spi_message msg; + struct spi_transfer xfer[2]; + union mcp251xfd_tx_obj_load_buf buf; +}; + +struct mcp251xfd_tx_ring { + unsigned int head; + unsigned int tail; + + u16 base; + u8 obj_num; + u8 obj_size; + + struct mcp251xfd_tx_obj obj[MCP251XFD_TX_OBJ_NUM_MAX]; + union mcp251xfd_write_reg_buf rts_buf; +}; + +struct mcp251xfd_rx_ring { + unsigned int head; + unsigned int tail; + + u16 base; + u8 nr; + u8 fifo_nr; + u8 obj_num; + u8 obj_size; + + struct mcp251xfd_hw_rx_obj_canfd obj[]; +}; + +struct __packed mcp251xfd_map_buf_nocrc { + struct mcp251xfd_buf_cmd cmd; + u8 data[256]; +} ____cacheline_aligned; + +struct __packed mcp251xfd_map_buf_crc { + struct mcp251xfd_buf_cmd_crc cmd; + u8 data[256 - 4]; + __be16 crc; +} ____cacheline_aligned; + +struct mcp251xfd_ecc { + u32 ecc_stat; + int cnt; +}; + +struct mcp251xfd_regs_status { + u32 intf; +}; + +enum mcp251xfd_model { + MCP251XFD_MODEL_MCP2517FD = 0x2517, + MCP251XFD_MODEL_MCP2518FD = 0x2518, + MCP251XFD_MODEL_MCP251XFD = 0xffff, /* autodetect model */ +}; + +struct mcp251xfd_devtype_data { + enum mcp251xfd_model model; + u32 quirks; +}; + +struct mcp251xfd_priv { + struct can_priv can; + struct can_rx_offload offload; + struct net_device *ndev; + + struct regmap *map_reg; /* register access */ + struct regmap *map_rx; /* RX/TEF RAM access */ + + struct regmap *map_nocrc; + struct mcp251xfd_map_buf_nocrc *map_buf_nocrc_rx; + struct mcp251xfd_map_buf_nocrc *map_buf_nocrc_tx; + + struct regmap *map_crc; + struct mcp251xfd_map_buf_crc *map_buf_crc_rx; + struct mcp251xfd_map_buf_crc *map_buf_crc_tx; + + struct spi_device *spi; + u32 spi_max_speed_hz_orig; + + struct mcp251xfd_tef_ring tef; + struct mcp251xfd_tx_ring tx[1]; + struct mcp251xfd_rx_ring *rx[1]; + + u8 rx_ring_num; + + struct mcp251xfd_ecc ecc; + struct mcp251xfd_regs_status regs_status; + + struct gpio_desc *rx_int; + struct clk *clk; + struct regulator *reg_vdd; + struct regulator *reg_xceiver; + + struct mcp251xfd_devtype_data devtype_data; + struct can_berr_counter bec; +}; + +#define MCP251XFD_IS(_model) \ +static inline bool \ +mcp251xfd_is_##_model(const struct mcp251xfd_priv *priv) \ +{ \ + return priv->devtype_data.model == MCP251XFD_MODEL_MCP##_model##FD; \ +} + +MCP251XFD_IS(2517); +MCP251XFD_IS(2518); +MCP251XFD_IS(251X); + +static inline u8 mcp251xfd_first_byte_set(u32 mask) +{ + return (mask & 0x0000ffff) ? + ((mask & 0x000000ff) ? 0 : 1) : + ((mask & 0x00ff0000) ? 2 : 3); +} + +static inline u8 mcp251xfd_last_byte_set(u32 mask) +{ + return (mask & 0xffff0000) ? + ((mask & 0xff000000) ? 3 : 2) : + ((mask & 0x0000ff00) ? 1 : 0); +} + +static inline __be16 mcp251xfd_cmd_reset(void) +{ + return cpu_to_be16(MCP251XFD_SPI_INSTRUCTION_RESET); +} + +static inline void +mcp251xfd_spi_cmd_read_nocrc(struct mcp251xfd_buf_cmd *cmd, u16 addr) +{ + cmd->cmd = cpu_to_be16(MCP251XFD_SPI_INSTRUCTION_READ | addr); +} + +static inline void +mcp251xfd_spi_cmd_write_nocrc(struct mcp251xfd_buf_cmd *cmd, u16 addr) +{ + cmd->cmd = cpu_to_be16(MCP251XFD_SPI_INSTRUCTION_WRITE | addr); +} + +static inline bool mcp251xfd_reg_in_ram(unsigned int reg) +{ + static const struct regmap_range range = + regmap_reg_range(MCP251XFD_RAM_START, + MCP251XFD_RAM_START + MCP251XFD_RAM_SIZE - 4); + + return regmap_reg_in_range(reg, &range); +} + +static inline void +__mcp251xfd_spi_cmd_crc_set_len(struct mcp251xfd_buf_cmd_crc *cmd, + u16 len, bool in_ram) +{ + /* Number of u32 for RAM access, number of u8 otherwise. */ + if (in_ram) + cmd->len = len >> 2; + else + cmd->len = len; +} + +static inline void +mcp251xfd_spi_cmd_crc_set_len_in_ram(struct mcp251xfd_buf_cmd_crc *cmd, u16 len) +{ + __mcp251xfd_spi_cmd_crc_set_len(cmd, len, true); +} + +static inline void +mcp251xfd_spi_cmd_crc_set_len_in_reg(struct mcp251xfd_buf_cmd_crc *cmd, u16 len) +{ + __mcp251xfd_spi_cmd_crc_set_len(cmd, len, false); +} + +static inline void +mcp251xfd_spi_cmd_read_crc_set_addr(struct mcp251xfd_buf_cmd_crc *cmd, u16 addr) +{ + cmd->cmd = cpu_to_be16(MCP251XFD_SPI_INSTRUCTION_READ_CRC | addr); +} + +static inline void +mcp251xfd_spi_cmd_read_crc(struct mcp251xfd_buf_cmd_crc *cmd, + u16 addr, u16 len) +{ + mcp251xfd_spi_cmd_read_crc_set_addr(cmd, addr); + __mcp251xfd_spi_cmd_crc_set_len(cmd, len, mcp251xfd_reg_in_ram(addr)); +} + +static inline void +mcp251xfd_spi_cmd_write_crc_set_addr(struct mcp251xfd_buf_cmd_crc *cmd, + u16 addr) +{ + cmd->cmd = cpu_to_be16(MCP251XFD_SPI_INSTRUCTION_WRITE_CRC | addr); +} + +static inline void +mcp251xfd_spi_cmd_write_crc(struct mcp251xfd_buf_cmd_crc *cmd, + u16 addr, u16 len) +{ + mcp251xfd_spi_cmd_write_crc_set_addr(cmd, addr); + __mcp251xfd_spi_cmd_crc_set_len(cmd, len, mcp251xfd_reg_in_ram(addr)); +} + +static inline u8 * +mcp251xfd_spi_cmd_write(const struct mcp251xfd_priv *priv, + union mcp251xfd_write_reg_buf *write_reg_buf, + u16 addr) +{ + u8 *data; + + if (priv->devtype_data.quirks & MCP251XFD_QUIRK_CRC_REG) { + mcp251xfd_spi_cmd_write_crc_set_addr(&write_reg_buf->crc.cmd, + addr); + data = write_reg_buf->crc.data; + } else { + mcp251xfd_spi_cmd_write_nocrc(&write_reg_buf->nocrc.cmd, + addr); + data = write_reg_buf->nocrc.data; + } + + return data; +} + +static inline u16 mcp251xfd_get_tef_obj_addr(u8 n) +{ + return MCP251XFD_RAM_START + + sizeof(struct mcp251xfd_hw_tef_obj) * n; +} + +static inline u16 +mcp251xfd_get_tx_obj_addr(const struct mcp251xfd_tx_ring *ring, u8 n) +{ + return ring->base + ring->obj_size * n; +} + +static inline u16 +mcp251xfd_get_rx_obj_addr(const struct mcp251xfd_rx_ring *ring, u8 n) +{ + return ring->base + ring->obj_size * n; +} + +static inline u8 mcp251xfd_get_tef_head(const struct mcp251xfd_priv *priv) +{ + return priv->tef.head & (priv->tx->obj_num - 1); +} + +static inline u8 mcp251xfd_get_tef_tail(const struct mcp251xfd_priv *priv) +{ + return priv->tef.tail & (priv->tx->obj_num - 1); +} + +static inline u8 mcp251xfd_get_tef_len(const struct mcp251xfd_priv *priv) +{ + return priv->tef.head - priv->tef.tail; +} + +static inline u8 mcp251xfd_get_tef_linear_len(const struct mcp251xfd_priv *priv) +{ + u8 len; + + len = mcp251xfd_get_tef_len(priv); + + return min_t(u8, len, priv->tx->obj_num - mcp251xfd_get_tef_tail(priv)); +} + +static inline u8 mcp251xfd_get_tx_head(const struct mcp251xfd_tx_ring *ring) +{ + return ring->head & (ring->obj_num - 1); +} + +static inline u8 mcp251xfd_get_tx_tail(const struct mcp251xfd_tx_ring *ring) +{ + return ring->tail & (ring->obj_num - 1); +} + +static inline u8 mcp251xfd_get_tx_free(const struct mcp251xfd_tx_ring *ring) +{ + return ring->obj_num - (ring->head - ring->tail); +} + +static inline int +mcp251xfd_get_tx_nr_by_addr(const struct mcp251xfd_tx_ring *tx_ring, u8 *nr, + u16 addr) +{ + if (addr < mcp251xfd_get_tx_obj_addr(tx_ring, 0) || + addr >= mcp251xfd_get_tx_obj_addr(tx_ring, tx_ring->obj_num)) + return -ENOENT; + + *nr = (addr - mcp251xfd_get_tx_obj_addr(tx_ring, 0)) / + tx_ring->obj_size; + + return 0; +} + +static inline u8 mcp251xfd_get_rx_head(const struct mcp251xfd_rx_ring *ring) +{ + return ring->head & (ring->obj_num - 1); +} + +static inline u8 mcp251xfd_get_rx_tail(const struct mcp251xfd_rx_ring *ring) +{ + return ring->tail & (ring->obj_num - 1); +} + +static inline u8 mcp251xfd_get_rx_len(const struct mcp251xfd_rx_ring *ring) +{ + return ring->head - ring->tail; +} + +static inline u8 +mcp251xfd_get_rx_linear_len(const struct mcp251xfd_rx_ring *ring) +{ + u8 len; + + len = mcp251xfd_get_rx_len(ring); + + return min_t(u8, len, ring->obj_num - mcp251xfd_get_rx_tail(ring)); +} + +#define mcp251xfd_for_each_tx_obj(ring, _obj, n) \ + for ((n) = 0, (_obj) = &(ring)->obj[(n)]; \ + (n) < (ring)->obj_num; \ + (n)++, (_obj) = &(ring)->obj[(n)]) + +#define mcp251xfd_for_each_rx_ring(priv, ring, n) \ + for ((n) = 0, (ring) = *((priv)->rx + (n)); \ + (n) < (priv)->rx_ring_num; \ + (n)++, (ring) = *((priv)->rx + (n))) + +int mcp251xfd_regmap_init(struct mcp251xfd_priv *priv); +u16 mcp251xfd_crc16_compute2(const void *cmd, size_t cmd_size, + const void *data, size_t data_size); +u16 mcp251xfd_crc16_compute(const void *data, size_t data_size); + +#endif diff --git a/drivers/net/can/spi/mcp25xxfd/Makefile b/drivers/net/can/spi/mcp25xxfd/Makefile deleted file mode 100644 index 4e17f592e22e..000000000000 --- a/drivers/net/can/spi/mcp25xxfd/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - -obj-$(CONFIG_CAN_MCP25XXFD) += mcp25xxfd.o - -mcp25xxfd-objs := -mcp25xxfd-objs += mcp25xxfd-core.o -mcp25xxfd-objs += mcp25xxfd-crc16.o -mcp25xxfd-objs += mcp25xxfd-regmap.o diff --git a/drivers/net/can/spi/mcp25xxfd/mcp25xxfd.h b/drivers/net/can/spi/mcp25xxfd/mcp25xxfd.h deleted file mode 100644 index 3bc799204cb0..000000000000 --- a/drivers/net/can/spi/mcp25xxfd/mcp25xxfd.h +++ /dev/null @@ -1,835 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 - * - * mcp25xxfd - Microchip MCP25xxFD Family CAN controller driver - * - * Copyright (c) 2019 Pengutronix, - * Marc Kleine-Budde <kernel@pengutronix.de> - * Copyright (c) 2019 Martin Sperl <kernel@martin.sperl.org> - */ - -#ifndef _MCP25XXFD_H -#define _MCP25XXFD_H - -#include <linux/can/core.h> -#include <linux/can/dev.h> -#include <linux/can/rx-offload.h> -#include <linux/gpio/consumer.h> -#include <linux/kernel.h> -#include <linux/regmap.h> -#include <linux/regulator/consumer.h> -#include <linux/spi/spi.h> - -/* MPC25xx registers */ - -/* CAN FD Controller Module SFR */ -#define MCP25XXFD_REG_CON 0x00 -#define MCP25XXFD_REG_CON_TXBWS_MASK GENMASK(31, 28) -#define MCP25XXFD_REG_CON_ABAT BIT(27) -#define MCP25XXFD_REG_CON_REQOP_MASK GENMASK(26, 24) -#define MCP25XXFD_REG_CON_MODE_MIXED 0 -#define MCP25XXFD_REG_CON_MODE_SLEEP 1 -#define MCP25XXFD_REG_CON_MODE_INT_LOOPBACK 2 -#define MCP25XXFD_REG_CON_MODE_LISTENONLY 3 -#define MCP25XXFD_REG_CON_MODE_CONFIG 4 -#define MCP25XXFD_REG_CON_MODE_EXT_LOOPBACK 5 -#define MCP25XXFD_REG_CON_MODE_CAN2_0 6 -#define MCP25XXFD_REG_CON_MODE_RESTRICTED 7 -#define MCP25XXFD_REG_CON_OPMOD_MASK GENMASK(23, 21) -#define MCP25XXFD_REG_CON_TXQEN BIT(20) -#define MCP25XXFD_REG_CON_STEF BIT(19) -#define MCP25XXFD_REG_CON_SERR2LOM BIT(18) -#define MCP25XXFD_REG_CON_ESIGM BIT(17) -#define MCP25XXFD_REG_CON_RTXAT BIT(16) -#define MCP25XXFD_REG_CON_BRSDIS BIT(12) -#define MCP25XXFD_REG_CON_BUSY BIT(11) -#define MCP25XXFD_REG_CON_WFT_MASK GENMASK(10, 9) -#define MCP25XXFD_REG_CON_WFT_T00FILTER 0x0 -#define MCP25XXFD_REG_CON_WFT_T01FILTER 0x1 -#define MCP25XXFD_REG_CON_WFT_T10FILTER 0x2 -#define MCP25XXFD_REG_CON_WFT_T11FILTER 0x3 -#define MCP25XXFD_REG_CON_WAKFIL BIT(8) -#define MCP25XXFD_REG_CON_PXEDIS BIT(6) -#define MCP25XXFD_REG_CON_ISOCRCEN BIT(5) -#define MCP25XXFD_REG_CON_DNCNT_MASK GENMASK(4, 0) - -#define MCP25XXFD_REG_NBTCFG 0x04 -#define MCP25XXFD_REG_NBTCFG_BRP_MASK GENMASK(31, 24) -#define MCP25XXFD_REG_NBTCFG_TSEG1_MASK GENMASK(23, 16) -#define MCP25XXFD_REG_NBTCFG_TSEG2_MASK GENMASK(14, 8) -#define MCP25XXFD_REG_NBTCFG_SJW_MASK GENMASK(6, 0) - -#define MCP25XXFD_REG_DBTCFG 0x08 -#define MCP25XXFD_REG_DBTCFG_BRP_MASK GENMASK(31, 24) -#define MCP25XXFD_REG_DBTCFG_TSEG1_MASK GENMASK(20, 16) -#define MCP25XXFD_REG_DBTCFG_TSEG2_MASK GENMASK(11, 8) -#define MCP25XXFD_REG_DBTCFG_SJW_MASK GENMASK(3, 0) - -#define MCP25XXFD_REG_TDC 0x0c -#define MCP25XXFD_REG_TDC_EDGFLTEN BIT(25) -#define MCP25XXFD_REG_TDC_SID11EN BIT(24) -#define MCP25XXFD_REG_TDC_TDCMOD_MASK GENMASK(17, 16) -#define MCP25XXFD_REG_TDC_TDCMOD_AUTO 2 -#define MCP25XXFD_REG_TDC_TDCMOD_MANUAL 1 -#define MCP25XXFD_REG_TDC_TDCMOD_DISABLED 0 -#define MCP25XXFD_REG_TDC_TDCO_MASK GENMASK(14, 8) -#define MCP25XXFD_REG_TDC_TDCV_MASK GENMASK(5, 0) - -#define MCP25XXFD_REG_TBC 0x10 - -#define MCP25XXFD_REG_TSCON 0x14 -#define MCP25XXFD_REG_TSCON_TSRES BIT(18) -#define MCP25XXFD_REG_TSCON_TSEOF BIT(17) -#define MCP25XXFD_REG_TSCON_TBCEN BIT(16) -#define MCP25XXFD_REG_TSCON_TBCPRE_MASK GENMASK(9, 0) - -#define MCP25XXFD_REG_VEC 0x18 -#define MCP25XXFD_REG_VEC_RXCODE_MASK GENMASK(30, 24) -#define MCP25XXFD_REG_VEC_TXCODE_MASK GENMASK(22, 16) -#define MCP25XXFD_REG_VEC_FILHIT_MASK GENMASK(12, 8) -#define MCP25XXFD_REG_VEC_ICODE_MASK GENMASK(6, 0) - -#define MCP25XXFD_REG_INT 0x1c -#define MCP25XXFD_REG_INT_IF_MASK GENMASK(15, 0) -#define MCP25XXFD_REG_INT_IE_MASK GENMASK(31, 16) -#define MCP25XXFD_REG_INT_IVMIE BIT(31) -#define MCP25XXFD_REG_INT_WAKIE BIT(30) -#define MCP25XXFD_REG_INT_CERRIE BIT(29) -#define MCP25XXFD_REG_INT_SERRIE BIT(28) -#define MCP25XXFD_REG_INT_RXOVIE BIT(27) -#define MCP25XXFD_REG_INT_TXATIE BIT(26) -#define MCP25XXFD_REG_INT_SPICRCIE BIT(25) -#define MCP25XXFD_REG_INT_ECCIE BIT(24) -#define MCP25XXFD_REG_INT_TEFIE BIT(20) -#define MCP25XXFD_REG_INT_MODIE BIT(19) -#define MCP25XXFD_REG_INT_TBCIE BIT(18) -#define MCP25XXFD_REG_INT_RXIE BIT(17) -#define MCP25XXFD_REG_INT_TXIE BIT(16) -#define MCP25XXFD_REG_INT_IVMIF BIT(15) -#define MCP25XXFD_REG_INT_WAKIF BIT(14) -#define MCP25XXFD_REG_INT_CERRIF BIT(13) -#define MCP25XXFD_REG_INT_SERRIF BIT(12) -#define MCP25XXFD_REG_INT_RXOVIF BIT(11) -#define MCP25XXFD_REG_INT_TXATIF BIT(10) -#define MCP25XXFD_REG_INT_SPICRCIF BIT(9) -#define MCP25XXFD_REG_INT_ECCIF BIT(8) -#define MCP25XXFD_REG_INT_TEFIF BIT(4) -#define MCP25XXFD_REG_INT_MODIF BIT(3) -#define MCP25XXFD_REG_INT_TBCIF BIT(2) -#define MCP25XXFD_REG_INT_RXIF BIT(1) -#define MCP25XXFD_REG_INT_TXIF BIT(0) -/* These IRQ flags must be cleared by SW in the CAN_INT register */ -#define MCP25XXFD_REG_INT_IF_CLEARABLE_MASK \ - (MCP25XXFD_REG_INT_IVMIF | MCP25XXFD_REG_INT_WAKIF | \ - MCP25XXFD_REG_INT_CERRIF | MCP25XXFD_REG_INT_SERRIF | \ - MCP25XXFD_REG_INT_MODIF) - -#define MCP25XXFD_REG_RXIF 0x20 -#define MCP25XXFD_REG_TXIF 0x24 -#define MCP25XXFD_REG_RXOVIF 0x28 -#define MCP25XXFD_REG_TXATIF 0x2c -#define MCP25XXFD_REG_TXREQ 0x30 - -#define MCP25XXFD_REG_TREC 0x34 -#define MCP25XXFD_REG_TREC_TXBO BIT(21) -#define MCP25XXFD_REG_TREC_TXBP BIT(20) -#define MCP25XXFD_REG_TREC_RXBP BIT(19) -#define MCP25XXFD_REG_TREC_TXWARN BIT(18) -#define MCP25XXFD_REG_TREC_RXWARN BIT(17) -#define MCP25XXFD_REG_TREC_EWARN BIT(16) -#define MCP25XXFD_REG_TREC_TEC_MASK GENMASK(15, 8) -#define MCP25XXFD_REG_TREC_REC_MASK GENMASK(7, 0) - -#define MCP25XXFD_REG_BDIAG0 0x38 -#define MCP25XXFD_REG_BDIAG0_DTERRCNT_MASK GENMASK(31, 24) -#define MCP25XXFD_REG_BDIAG0_DRERRCNT_MASK GENMASK(23, 16) -#define MCP25XXFD_REG_BDIAG0_NTERRCNT_MASK GENMASK(15, 8) -#define MCP25XXFD_REG_BDIAG0_NRERRCNT_MASK GENMASK(7, 0) - -#define MCP25XXFD_REG_BDIAG1 0x3c -#define MCP25XXFD_REG_BDIAG1_DLCMM BIT(31) -#define MCP25XXFD_REG_BDIAG1_ESI BIT(30) -#define MCP25XXFD_REG_BDIAG1_DCRCERR BIT(29) -#define MCP25XXFD_REG_BDIAG1_DSTUFERR BIT(28) -#define MCP25XXFD_REG_BDIAG1_DFORMERR BIT(27) -#define MCP25XXFD_REG_BDIAG1_DBIT1ERR BIT(25) -#define MCP25XXFD_REG_BDIAG1_DBIT0ERR BIT(24) -#define MCP25XXFD_REG_BDIAG1_TXBOERR BIT(23) -#define MCP25XXFD_REG_BDIAG1_NCRCERR BIT(21) -#define MCP25XXFD_REG_BDIAG1_NSTUFERR BIT(20) -#define MCP25XXFD_REG_BDIAG1_NFORMERR BIT(19) -#define MCP25XXFD_REG_BDIAG1_NACKERR BIT(18) -#define MCP25XXFD_REG_BDIAG1_NBIT1ERR BIT(17) -#define MCP25XXFD_REG_BDIAG1_NBIT0ERR BIT(16) -#define MCP25XXFD_REG_BDIAG1_BERR_MASK \ - (MCP25XXFD_REG_BDIAG1_DLCMM | MCP25XXFD_REG_BDIAG1_ESI | \ - MCP25XXFD_REG_BDIAG1_DCRCERR | MCP25XXFD_REG_BDIAG1_DSTUFERR | \ - MCP25XXFD_REG_BDIAG1_DFORMERR | MCP25XXFD_REG_BDIAG1_DBIT1ERR | \ - MCP25XXFD_REG_BDIAG1_DBIT0ERR | MCP25XXFD_REG_BDIAG1_TXBOERR | \ - MCP25XXFD_REG_BDIAG1_NCRCERR | MCP25XXFD_REG_BDIAG1_NSTUFERR | \ - MCP25XXFD_REG_BDIAG1_NFORMERR | MCP25XXFD_REG_BDIAG1_NACKERR | \ - MCP25XXFD_REG_BDIAG1_NBIT1ERR | MCP25XXFD_REG_BDIAG1_NBIT0ERR) -#define MCP25XXFD_REG_BDIAG1_EFMSGCNT_MASK GENMASK(15, 0) - -#define MCP25XXFD_REG_TEFCON 0x40 -#define MCP25XXFD_REG_TEFCON_FSIZE_MASK GENMASK(28, 24) -#define MCP25XXFD_REG_TEFCON_FRESET BIT(10) -#define MCP25XXFD_REG_TEFCON_UINC BIT(8) -#define MCP25XXFD_REG_TEFCON_TEFTSEN BIT(5) -#define MCP25XXFD_REG_TEFCON_TEFOVIE BIT(3) -#define MCP25XXFD_REG_TEFCON_TEFFIE BIT(2) -#define MCP25XXFD_REG_TEFCON_TEFHIE BIT(1) -#define MCP25XXFD_REG_TEFCON_TEFNEIE BIT(0) - -#define MCP25XXFD_REG_TEFSTA 0x44 -#define MCP25XXFD_REG_TEFSTA_TEFOVIF BIT(3) -#define MCP25XXFD_REG_TEFSTA_TEFFIF BIT(2) -#define MCP25XXFD_REG_TEFSTA_TEFHIF BIT(1) -#define MCP25XXFD_REG_TEFSTA_TEFNEIF BIT(0) - -#define MCP25XXFD_REG_TEFUA 0x48 - -#define MCP25XXFD_REG_TXQCON 0x50 -#define MCP25XXFD_REG_TXQCON_PLSIZE_MASK GENMASK(31, 29) -#define MCP25XXFD_REG_TXQCON_PLSIZE_8 0 -#define MCP25XXFD_REG_TXQCON_PLSIZE_12 1 -#define MCP25XXFD_REG_TXQCON_PLSIZE_16 2 -#define MCP25XXFD_REG_TXQCON_PLSIZE_20 3 -#define MCP25XXFD_REG_TXQCON_PLSIZE_24 4 -#define MCP25XXFD_REG_TXQCON_PLSIZE_32 5 -#define MCP25XXFD_REG_TXQCON_PLSIZE_48 6 -#define MCP25XXFD_REG_TXQCON_PLSIZE_64 7 -#define MCP25XXFD_REG_TXQCON_FSIZE_MASK GENMASK(28, 24) -#define MCP25XXFD_REG_TXQCON_TXAT_UNLIMITED 3 -#define MCP25XXFD_REG_TXQCON_TXAT_THREE_SHOT 1 -#define MCP25XXFD_REG_TXQCON_TXAT_ONE_SHOT 0 -#define MCP25XXFD_REG_TXQCON_TXAT_MASK GENMASK(22, 21) -#define MCP25XXFD_REG_TXQCON_TXPRI_MASK GENMASK(20, 16) -#define MCP25XXFD_REG_TXQCON_FRESET BIT(10) -#define MCP25XXFD_REG_TXQCON_TXREQ BIT(9) -#define MCP25XXFD_REG_TXQCON_UINC BIT(8) -#define MCP25XXFD_REG_TXQCON_TXEN BIT(7) -#define MCP25XXFD_REG_TXQCON_TXATIE BIT(4) -#define MCP25XXFD_REG_TXQCON_TXQEIE BIT(2) -#define MCP25XXFD_REG_TXQCON_TXQNIE BIT(0) - -#define MCP25XXFD_REG_TXQSTA 0x54 -#define MCP25XXFD_REG_TXQSTA_TXQCI_MASK GENMASK(12, 8) -#define MCP25XXFD_REG_TXQSTA_TXABT BIT(7) -#define MCP25XXFD_REG_TXQSTA_TXLARB BIT(6) -#define MCP25XXFD_REG_TXQSTA_TXERR BIT(5) -#define MCP25XXFD_REG_TXQSTA_TXATIF BIT(4) -#define MCP25XXFD_REG_TXQSTA_TXQEIF BIT(2) -#define MCP25XXFD_REG_TXQSTA_TXQNIF BIT(0) - -#define MCP25XXFD_REG_TXQUA 0x58 - -#define MCP25XXFD_REG_FIFOCON(x) (0x50 + 0xc * (x)) -#define MCP25XXFD_REG_FIFOCON_PLSIZE_MASK GENMASK(31, 29) -#define MCP25XXFD_REG_FIFOCON_PLSIZE_8 0 -#define MCP25XXFD_REG_FIFOCON_PLSIZE_12 1 -#define MCP25XXFD_REG_FIFOCON_PLSIZE_16 2 -#define MCP25XXFD_REG_FIFOCON_PLSIZE_20 3 -#define MCP25XXFD_REG_FIFOCON_PLSIZE_24 4 -#define MCP25XXFD_REG_FIFOCON_PLSIZE_32 5 -#define MCP25XXFD_REG_FIFOCON_PLSIZE_48 6 -#define MCP25XXFD_REG_FIFOCON_PLSIZE_64 7 -#define MCP25XXFD_REG_FIFOCON_FSIZE_MASK GENMASK(28, 24) -#define MCP25XXFD_REG_FIFOCON_TXAT_MASK GENMASK(22, 21) -#define MCP25XXFD_REG_FIFOCON_TXAT_ONE_SHOT 0 -#define MCP25XXFD_REG_FIFOCON_TXAT_THREE_SHOT 1 -#define MCP25XXFD_REG_FIFOCON_TXAT_UNLIMITED 3 -#define MCP25XXFD_REG_FIFOCON_TXPRI_MASK GENMASK(20, 16) -#define MCP25XXFD_REG_FIFOCON_FRESET BIT(10) -#define MCP25XXFD_REG_FIFOCON_TXREQ BIT(9) -#define MCP25XXFD_REG_FIFOCON_UINC BIT(8) -#define MCP25XXFD_REG_FIFOCON_TXEN BIT(7) -#define MCP25XXFD_REG_FIFOCON_RTREN BIT(6) -#define MCP25XXFD_REG_FIFOCON_RXTSEN BIT(5) -#define MCP25XXFD_REG_FIFOCON_TXATIE BIT(4) -#define MCP25XXFD_REG_FIFOCON_RXOVIE BIT(3) -#define MCP25XXFD_REG_FIFOCON_TFERFFIE BIT(2) -#define MCP25XXFD_REG_FIFOCON_TFHRFHIE BIT(1) -#define MCP25XXFD_REG_FIFOCON_TFNRFNIE BIT(0) - -#define MCP25XXFD_REG_FIFOSTA(x) (0x54 + 0xc * (x)) -#define MCP25XXFD_REG_FIFOSTA_FIFOCI_MASK GENMASK(12, 8) -#define MCP25XXFD_REG_FIFOSTA_TXABT BIT(7) -#define MCP25XXFD_REG_FIFOSTA_TXLARB BIT(6) -#define MCP25XXFD_REG_FIFOSTA_TXERR BIT(5) -#define MCP25XXFD_REG_FIFOSTA_TXATIF BIT(4) -#define MCP25XXFD_REG_FIFOSTA_RXOVIF BIT(3) -#define MCP25XXFD_REG_FIFOSTA_TFERFFIF BIT(2) -#define MCP25XXFD_REG_FIFOSTA_TFHRFHIF BIT(1) -#define MCP25XXFD_REG_FIFOSTA_TFNRFNIF BIT(0) - -#define MCP25XXFD_REG_FIFOUA(x) (0x58 + 0xc * (x)) - -#define MCP25XXFD_REG_FLTCON(x) (0x1d0 + 0x4 * (x)) -#define MCP25XXFD_REG_FLTCON_FLTEN3 BIT(31) -#define MCP25XXFD_REG_FLTCON_F3BP_MASK GENMASK(28, 24) -#define MCP25XXFD_REG_FLTCON_FLTEN2 BIT(23) -#define MCP25XXFD_REG_FLTCON_F2BP_MASK GENMASK(20, 16) -#define MCP25XXFD_REG_FLTCON_FLTEN1 BIT(15) -#define MCP25XXFD_REG_FLTCON_F1BP_MASK GENMASK(12, 8) -#define MCP25XXFD_REG_FLTCON_FLTEN0 BIT(7) -#define MCP25XXFD_REG_FLTCON_F0BP_MASK GENMASK(4, 0) -#define MCP25XXFD_REG_FLTCON_FLTEN(x) (BIT(7) << 8 * ((x) & 0x3)) -#define MCP25XXFD_REG_FLTCON_FLT_MASK(x) (GENMASK(7, 0) << (8 * ((x) & 0x3))) -#define MCP25XXFD_REG_FLTCON_FBP(x, fifo) ((fifo) << 8 * ((x) & 0x3)) - -#define MCP25XXFD_REG_FLTOBJ(x) (0x1f0 + 0x8 * (x)) -#define MCP25XXFD_REG_FLTOBJ_EXIDE BIT(30) -#define MCP25XXFD_REG_FLTOBJ_SID11 BIT(29) -#define MCP25XXFD_REG_FLTOBJ_EID_MASK GENMASK(28, 11) -#define MCP25XXFD_REG_FLTOBJ_SID_MASK GENMASK(10, 0) - -#define MCP25XXFD_REG_FLTMASK(x) (0x1f4 + 0x8 * (x)) -#define MCP25XXFD_REG_MASK_MIDE BIT(30) -#define MCP25XXFD_REG_MASK_MSID11 BIT(29) -#define MCP25XXFD_REG_MASK_MEID_MASK GENMASK(28, 11) -#define MCP25XXFD_REG_MASK_MSID_MASK GENMASK(10, 0) - -/* RAM */ -#define MCP25XXFD_RAM_START 0x400 -#define MCP25XXFD_RAM_SIZE SZ_2K - -/* Message Object */ -#define MCP25XXFD_OBJ_ID_SID11 BIT(29) -#define MCP25XXFD_OBJ_ID_EID_MASK GENMASK(28, 11) -#define MCP25XXFD_OBJ_ID_SID_MASK GENMASK(10, 0) -#define MCP25XXFD_OBJ_FLAGS_SEQ_MCP2518FD_MASK GENMASK(31, 9) -#define MCP25XXFD_OBJ_FLAGS_SEQ_MCP2517FD_MASK GENMASK(15, 9) -#define MCP25XXFD_OBJ_FLAGS_SEQ_MASK MCP25XXFD_OBJ_FLAGS_SEQ_MCP2518FD_MASK -#define MCP25XXFD_OBJ_FLAGS_ESI BIT(8) -#define MCP25XXFD_OBJ_FLAGS_FDF BIT(7) -#define MCP25XXFD_OBJ_FLAGS_BRS BIT(6) -#define MCP25XXFD_OBJ_FLAGS_RTR BIT(5) -#define MCP25XXFD_OBJ_FLAGS_IDE BIT(4) -#define MCP25XXFD_OBJ_FLAGS_DLC GENMASK(3, 0) - -#define MCP25XXFD_REG_FRAME_EFF_SID_MASK GENMASK(28, 18) -#define MCP25XXFD_REG_FRAME_EFF_EID_MASK GENMASK(17, 0) - -/* MCP2517/18FD SFR */ -#define MCP25XXFD_REG_OSC 0xe00 -#define MCP25XXFD_REG_OSC_SCLKRDY BIT(12) -#define MCP25XXFD_REG_OSC_OSCRDY BIT(10) -#define MCP25XXFD_REG_OSC_PLLRDY BIT(8) -#define MCP25XXFD_REG_OSC_CLKODIV_10 3 -#define MCP25XXFD_REG_OSC_CLKODIV_4 2 -#define MCP25XXFD_REG_OSC_CLKODIV_2 1 -#define MCP25XXFD_REG_OSC_CLKODIV_1 0 -#define MCP25XXFD_REG_OSC_CLKODIV_MASK GENMASK(6, 5) -#define MCP25XXFD_REG_OSC_SCLKDIV BIT(4) -#define MCP25XXFD_REG_OSC_LPMEN BIT(3) /* MCP2518FD only */ -#define MCP25XXFD_REG_OSC_OSCDIS BIT(2) -#define MCP25XXFD_REG_OSC_PLLEN BIT(0) - -#define MCP25XXFD_REG_IOCON 0xe04 -#define MCP25XXFD_REG_IOCON_INTOD BIT(30) -#define MCP25XXFD_REG_IOCON_SOF BIT(29) -#define MCP25XXFD_REG_IOCON_TXCANOD BIT(28) -#define MCP25XXFD_REG_IOCON_PM1 BIT(25) -#define MCP25XXFD_REG_IOCON_PM0 BIT(24) -#define MCP25XXFD_REG_IOCON_GPIO1 BIT(17) -#define MCP25XXFD_REG_IOCON_GPIO0 BIT(16) -#define MCP25XXFD_REG_IOCON_LAT1 BIT(9) -#define MCP25XXFD_REG_IOCON_LAT0 BIT(8) -#define MCP25XXFD_REG_IOCON_XSTBYEN BIT(6) -#define MCP25XXFD_REG_IOCON_TRIS1 BIT(1) -#define MCP25XXFD_REG_IOCON_TRIS0 BIT(0) - -#define MCP25XXFD_REG_CRC 0xe08 -#define MCP25XXFD_REG_CRC_FERRIE BIT(25) -#define MCP25XXFD_REG_CRC_CRCERRIE BIT(24) -#define MCP25XXFD_REG_CRC_FERRIF BIT(17) -#define MCP25XXFD_REG_CRC_CRCERRIF BIT(16) -#define MCP25XXFD_REG_CRC_IF_MASK GENMASK(17, 16) -#define MCP25XXFD_REG_CRC_MASK GENMASK(15, 0) - -#define MCP25XXFD_REG_ECCCON 0xe0c -#define MCP25XXFD_REG_ECCCON_PARITY_MASK GENMASK(14, 8) -#define MCP25XXFD_REG_ECCCON_DEDIE BIT(2) -#define MCP25XXFD_REG_ECCCON_SECIE BIT(1) -#define MCP25XXFD_REG_ECCCON_ECCEN BIT(0) - -#define MCP25XXFD_REG_ECCSTAT 0xe10 -#define MCP25XXFD_REG_ECCSTAT_ERRADDR_MASK GENMASK(27, 16) -#define MCP25XXFD_REG_ECCSTAT_IF_MASK GENMASK(2, 1) -#define MCP25XXFD_REG_ECCSTAT_DEDIF BIT(2) -#define MCP25XXFD_REG_ECCSTAT_SECIF BIT(1) - -#define MCP25XXFD_REG_DEVID 0xe14 /* MCP2518FD only */ -#define MCP25XXFD_REG_DEVID_ID_MASK GENMASK(7, 4) -#define MCP25XXFD_REG_DEVID_REV_MASK GENMASK(3, 0) - -/* number of TX FIFO objects, depending on CAN mode - * - * FIFO setup: tef: 8*12 bytes = 96 bytes, tx: 8*16 bytes = 128 bytes - * FIFO setup: tef: 4*12 bytes = 48 bytes, tx: 4*72 bytes = 288 bytes - */ -#define MCP25XXFD_TX_OBJ_NUM_CAN 8 -#define MCP25XXFD_TX_OBJ_NUM_CANFD 4 - -#if MCP25XXFD_TX_OBJ_NUM_CAN > MCP25XXFD_TX_OBJ_NUM_CANFD -#define MCP25XXFD_TX_OBJ_NUM_MAX MCP25XXFD_TX_OBJ_NUM_CAN -#else -#define MCP25XXFD_TX_OBJ_NUM_MAX MCP25XXFD_TX_OBJ_NUM_CANFD -#endif - -#define MCP25XXFD_NAPI_WEIGHT 32 -#define MCP25XXFD_TX_FIFO 1 -#define MCP25XXFD_RX_FIFO(x) (MCP25XXFD_TX_FIFO + 1 + (x)) - -/* SPI commands */ -#define MCP25XXFD_SPI_INSTRUCTION_RESET 0x0000 -#define MCP25XXFD_SPI_INSTRUCTION_WRITE 0x2000 -#define MCP25XXFD_SPI_INSTRUCTION_READ 0x3000 -#define MCP25XXFD_SPI_INSTRUCTION_WRITE_CRC 0xa000 -#define MCP25XXFD_SPI_INSTRUCTION_READ_CRC 0xb000 -#define MCP25XXFD_SPI_INSTRUCTION_WRITE_CRC_SAFE 0xc000 -#define MCP25XXFD_SPI_ADDRESS_MASK GENMASK(11, 0) - -#define MCP25XXFD_SYSCLOCK_HZ_MAX 40000000 -#define MCP25XXFD_SYSCLOCK_HZ_MIN 1000000 -#define MCP25XXFD_SPICLOCK_HZ_MAX 20000000 -#define MCP25XXFD_OSC_PLL_MULTIPLIER 10 -#define MCP25XXFD_OSC_STAB_SLEEP_US (3 * USEC_PER_MSEC) -#define MCP25XXFD_OSC_STAB_TIMEOUT_US (10 * MCP25XXFD_OSC_STAB_SLEEP_US) -#define MCP25XXFD_POLL_SLEEP_US (10) -#define MCP25XXFD_POLL_TIMEOUT_US (USEC_PER_MSEC) -#define MCP25XXFD_SOFTRESET_RETRIES_MAX 3 -#define MCP25XXFD_READ_CRC_RETRIES_MAX 3 -#define MCP25XXFD_ECC_CNT_MAX 2 -#define MCP25XXFD_SANITIZE_SPI 1 -#define MCP25XXFD_SANITIZE_CAN 1 - -/* Silence TX MAB overflow warnings */ -#define MCP25XXFD_QUIRK_MAB_NO_WARN BIT(0) -/* Use CRC to access registers */ -#define MCP25XXFD_QUIRK_CRC_REG BIT(1) -/* Use CRC to access RX/TEF-RAM */ -#define MCP25XXFD_QUIRK_CRC_RX BIT(2) -/* Use CRC to access TX-RAM */ -#define MCP25XXFD_QUIRK_CRC_TX BIT(3) -/* Enable ECC for RAM */ -#define MCP25XXFD_QUIRK_ECC BIT(4) -/* Use Half Duplex SPI transfers */ -#define MCP25XXFD_QUIRK_HALF_DUPLEX BIT(5) - -struct mcp25xxfd_hw_tef_obj { - u32 id; - u32 flags; - u32 ts; -}; - -/* The tx_obj_raw version is used in spi async, i.e. without - * regmap. We have to take care of endianness ourselves. - */ -struct mcp25xxfd_hw_tx_obj_raw { - __le32 id; - __le32 flags; - u8 data[sizeof_field(struct canfd_frame, data)]; -}; - -struct mcp25xxfd_hw_tx_obj_can { - u32 id; - u32 flags; - u8 data[sizeof_field(struct can_frame, data)]; -}; - -struct mcp25xxfd_hw_tx_obj_canfd { - u32 id; - u32 flags; - u8 data[sizeof_field(struct canfd_frame, data)]; -}; - -struct mcp25xxfd_hw_rx_obj_can { - u32 id; - u32 flags; - u32 ts; - u8 data[sizeof_field(struct can_frame, data)]; -}; - -struct mcp25xxfd_hw_rx_obj_canfd { - u32 id; - u32 flags; - u32 ts; - u8 data[sizeof_field(struct canfd_frame, data)]; -}; - -struct mcp25xxfd_tef_ring { - unsigned int head; - unsigned int tail; - - /* u8 obj_num equals tx_ring->obj_num */ - /* u8 obj_size equals sizeof(struct mcp25xxfd_hw_tef_obj) */ -}; - -struct __packed mcp25xxfd_buf_cmd { - __be16 cmd; -}; - -struct __packed mcp25xxfd_buf_cmd_crc { - __be16 cmd; - u8 len; -}; - -union mcp25xxfd_tx_obj_load_buf { - struct __packed { - struct mcp25xxfd_buf_cmd cmd; - struct mcp25xxfd_hw_tx_obj_raw hw_tx_obj; - } nocrc; - struct __packed { - struct mcp25xxfd_buf_cmd_crc cmd; - struct mcp25xxfd_hw_tx_obj_raw hw_tx_obj; - __be16 crc; - } crc; -} ____cacheline_aligned; - -union mcp25xxfd_write_reg_buf { - struct __packed { - struct mcp25xxfd_buf_cmd cmd; - u8 data[4]; - } nocrc; - struct __packed { - struct mcp25xxfd_buf_cmd_crc cmd; - u8 data[4]; - __be16 crc; - } crc; -} ____cacheline_aligned; - -struct mcp25xxfd_tx_obj { - struct spi_message msg; - struct spi_transfer xfer[2]; - union mcp25xxfd_tx_obj_load_buf buf; -}; - -struct mcp25xxfd_tx_ring { - unsigned int head; - unsigned int tail; - - u16 base; - u8 obj_num; - u8 obj_size; - - struct mcp25xxfd_tx_obj obj[MCP25XXFD_TX_OBJ_NUM_MAX]; - union mcp25xxfd_write_reg_buf rts_buf; -}; - -struct mcp25xxfd_rx_ring { - unsigned int head; - unsigned int tail; - - u16 base; - u8 nr; - u8 fifo_nr; - u8 obj_num; - u8 obj_size; - - struct mcp25xxfd_hw_rx_obj_canfd obj[]; -}; - -struct __packed mcp25xxfd_map_buf_nocrc { - struct mcp25xxfd_buf_cmd cmd; - u8 data[256]; -} ____cacheline_aligned; - -struct __packed mcp25xxfd_map_buf_crc { - struct mcp25xxfd_buf_cmd_crc cmd; - u8 data[256 - 4]; - __be16 crc; -} ____cacheline_aligned; - -struct mcp25xxfd_ecc { - u32 ecc_stat; - int cnt; -}; - -struct mcp25xxfd_regs_status { - u32 intf; -}; - -enum mcp25xxfd_model { - MCP25XXFD_MODEL_MCP2517FD = 0x2517, - MCP25XXFD_MODEL_MCP2518FD = 0x2518, - MCP25XXFD_MODEL_MCP25XXFD = 0xffff, /* autodetect model */ -}; - -struct mcp25xxfd_devtype_data { - enum mcp25xxfd_model model; - u32 quirks; -}; - -struct mcp25xxfd_priv { - struct can_priv can; - struct can_rx_offload offload; - struct net_device *ndev; - - struct regmap *map_reg; /* register access */ - struct regmap *map_rx; /* RX/TEF RAM access */ - - struct regmap *map_nocrc; - struct mcp25xxfd_map_buf_nocrc *map_buf_nocrc_rx; - struct mcp25xxfd_map_buf_nocrc *map_buf_nocrc_tx; - - struct regmap *map_crc; - struct mcp25xxfd_map_buf_crc *map_buf_crc_rx; - struct mcp25xxfd_map_buf_crc *map_buf_crc_tx; - - struct spi_device *spi; - u32 spi_max_speed_hz_orig; - - struct mcp25xxfd_tef_ring tef; - struct mcp25xxfd_tx_ring tx[1]; - struct mcp25xxfd_rx_ring *rx[1]; - - u8 rx_ring_num; - - struct mcp25xxfd_ecc ecc; - struct mcp25xxfd_regs_status regs_status; - - struct gpio_desc *rx_int; - struct clk *clk; - struct regulator *reg_vdd; - struct regulator *reg_xceiver; - - struct mcp25xxfd_devtype_data devtype_data; - struct can_berr_counter bec; -}; - -#define MCP25XXFD_IS(_model) \ -static inline bool \ -mcp25xxfd_is_##_model(const struct mcp25xxfd_priv *priv) \ -{ \ - return priv->devtype_data.model == MCP25XXFD_MODEL_MCP##_model##FD; \ -} - -MCP25XXFD_IS(2517); -MCP25XXFD_IS(2518); -MCP25XXFD_IS(25XX); - -static inline u8 mcp25xxfd_first_byte_set(u32 mask) -{ - return (mask & 0x0000ffff) ? - ((mask & 0x000000ff) ? 0 : 1) : - ((mask & 0x00ff0000) ? 2 : 3); -} - -static inline u8 mcp25xxfd_last_byte_set(u32 mask) -{ - return (mask & 0xffff0000) ? - ((mask & 0xff000000) ? 3 : 2) : - ((mask & 0x0000ff00) ? 1 : 0); -} - -static inline __be16 mcp25xxfd_cmd_reset(void) -{ - return cpu_to_be16(MCP25XXFD_SPI_INSTRUCTION_RESET); -} - -static inline void -mcp25xxfd_spi_cmd_read_nocrc(struct mcp25xxfd_buf_cmd *cmd, u16 addr) -{ - cmd->cmd = cpu_to_be16(MCP25XXFD_SPI_INSTRUCTION_READ | addr); -} - -static inline void -mcp25xxfd_spi_cmd_write_nocrc(struct mcp25xxfd_buf_cmd *cmd, u16 addr) -{ - cmd->cmd = cpu_to_be16(MCP25XXFD_SPI_INSTRUCTION_WRITE | addr); -} - -static inline bool mcp25xxfd_reg_in_ram(unsigned int reg) -{ - static const struct regmap_range range = - regmap_reg_range(MCP25XXFD_RAM_START, - MCP25XXFD_RAM_START + MCP25XXFD_RAM_SIZE - 4); - - return regmap_reg_in_range(reg, &range); -} - -static inline void -__mcp25xxfd_spi_cmd_crc_set_len(struct mcp25xxfd_buf_cmd_crc *cmd, - u16 len, bool in_ram) -{ - /* Number of u32 for RAM access, number of u8 otherwise. */ - if (in_ram) - cmd->len = len >> 2; - else - cmd->len = len; -} - -static inline void -mcp25xxfd_spi_cmd_crc_set_len_in_ram(struct mcp25xxfd_buf_cmd_crc *cmd, u16 len) -{ - __mcp25xxfd_spi_cmd_crc_set_len(cmd, len, true); -} - -static inline void -mcp25xxfd_spi_cmd_crc_set_len_in_reg(struct mcp25xxfd_buf_cmd_crc *cmd, u16 len) -{ - __mcp25xxfd_spi_cmd_crc_set_len(cmd, len, false); -} - -static inline void -mcp25xxfd_spi_cmd_read_crc_set_addr(struct mcp25xxfd_buf_cmd_crc *cmd, u16 addr) -{ - cmd->cmd = cpu_to_be16(MCP25XXFD_SPI_INSTRUCTION_READ_CRC | addr); -} - -static inline void -mcp25xxfd_spi_cmd_read_crc(struct mcp25xxfd_buf_cmd_crc *cmd, - u16 addr, u16 len) -{ - mcp25xxfd_spi_cmd_read_crc_set_addr(cmd, addr); - __mcp25xxfd_spi_cmd_crc_set_len(cmd, len, mcp25xxfd_reg_in_ram(addr)); -} - -static inline void -mcp25xxfd_spi_cmd_write_crc_set_addr(struct mcp25xxfd_buf_cmd_crc *cmd, - u16 addr) -{ - cmd->cmd = cpu_to_be16(MCP25XXFD_SPI_INSTRUCTION_WRITE_CRC | addr); -} - -static inline void -mcp25xxfd_spi_cmd_write_crc(struct mcp25xxfd_buf_cmd_crc *cmd, - u16 addr, u16 len) -{ - mcp25xxfd_spi_cmd_write_crc_set_addr(cmd, addr); - __mcp25xxfd_spi_cmd_crc_set_len(cmd, len, mcp25xxfd_reg_in_ram(addr)); -} - -static inline u8 * -mcp25xxfd_spi_cmd_write(const struct mcp25xxfd_priv *priv, - union mcp25xxfd_write_reg_buf *write_reg_buf, - u16 addr) -{ - u8 *data; - - if (priv->devtype_data.quirks & MCP25XXFD_QUIRK_CRC_REG) { - mcp25xxfd_spi_cmd_write_crc_set_addr(&write_reg_buf->crc.cmd, - addr); - data = write_reg_buf->crc.data; - } else { - mcp25xxfd_spi_cmd_write_nocrc(&write_reg_buf->nocrc.cmd, - addr); - data = write_reg_buf->nocrc.data; - } - - return data; -} - -static inline u16 mcp25xxfd_get_tef_obj_addr(u8 n) -{ - return MCP25XXFD_RAM_START + - sizeof(struct mcp25xxfd_hw_tef_obj) * n; -} - -static inline u16 -mcp25xxfd_get_tx_obj_addr(const struct mcp25xxfd_tx_ring *ring, u8 n) -{ - return ring->base + ring->obj_size * n; -} - -static inline u16 -mcp25xxfd_get_rx_obj_addr(const struct mcp25xxfd_rx_ring *ring, u8 n) -{ - return ring->base + ring->obj_size * n; -} - -static inline u8 mcp25xxfd_get_tef_head(const struct mcp25xxfd_priv *priv) -{ - return priv->tef.head & (priv->tx->obj_num - 1); -} - -static inline u8 mcp25xxfd_get_tef_tail(const struct mcp25xxfd_priv *priv) -{ - return priv->tef.tail & (priv->tx->obj_num - 1); -} - -static inline u8 mcp25xxfd_get_tef_len(const struct mcp25xxfd_priv *priv) -{ - return priv->tef.head - priv->tef.tail; -} - -static inline u8 mcp25xxfd_get_tef_linear_len(const struct mcp25xxfd_priv *priv) -{ - u8 len; - - len = mcp25xxfd_get_tef_len(priv); - - return min_t(u8, len, priv->tx->obj_num - mcp25xxfd_get_tef_tail(priv)); -} - -static inline u8 mcp25xxfd_get_tx_head(const struct mcp25xxfd_tx_ring *ring) -{ - return ring->head & (ring->obj_num - 1); -} - -static inline u8 mcp25xxfd_get_tx_tail(const struct mcp25xxfd_tx_ring *ring) -{ - return ring->tail & (ring->obj_num - 1); -} - -static inline u8 mcp25xxfd_get_tx_free(const struct mcp25xxfd_tx_ring *ring) -{ - return ring->obj_num - (ring->head - ring->tail); -} - -static inline int -mcp25xxfd_get_tx_nr_by_addr(const struct mcp25xxfd_tx_ring *tx_ring, u8 *nr, - u16 addr) -{ - if (addr < mcp25xxfd_get_tx_obj_addr(tx_ring, 0) || - addr >= mcp25xxfd_get_tx_obj_addr(tx_ring, tx_ring->obj_num)) - return -ENOENT; - - *nr = (addr - mcp25xxfd_get_tx_obj_addr(tx_ring, 0)) / - tx_ring->obj_size; - - return 0; -} - -static inline u8 mcp25xxfd_get_rx_head(const struct mcp25xxfd_rx_ring *ring) -{ - return ring->head & (ring->obj_num - 1); -} - -static inline u8 mcp25xxfd_get_rx_tail(const struct mcp25xxfd_rx_ring *ring) -{ - return ring->tail & (ring->obj_num - 1); -} - -static inline u8 mcp25xxfd_get_rx_len(const struct mcp25xxfd_rx_ring *ring) -{ - return ring->head - ring->tail; -} - -static inline u8 -mcp25xxfd_get_rx_linear_len(const struct mcp25xxfd_rx_ring *ring) -{ - u8 len; - - len = mcp25xxfd_get_rx_len(ring); - - return min_t(u8, len, ring->obj_num - mcp25xxfd_get_rx_tail(ring)); -} - -#define mcp25xxfd_for_each_tx_obj(ring, _obj, n) \ - for ((n) = 0, (_obj) = &(ring)->obj[(n)]; \ - (n) < (ring)->obj_num; \ - (n)++, (_obj) = &(ring)->obj[(n)]) - -#define mcp25xxfd_for_each_rx_ring(priv, ring, n) \ - for ((n) = 0, (ring) = *((priv)->rx + (n)); \ - (n) < (priv)->rx_ring_num; \ - (n)++, (ring) = *((priv)->rx + (n))) - -int mcp25xxfd_regmap_init(struct mcp25xxfd_priv *priv); -u16 mcp25xxfd_crc16_compute2(const void *cmd, size_t cmd_size, - const void *data, size_t data_size); -u16 mcp25xxfd_crc16_compute(const void *data, size_t data_size); - -#endif diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index 73507cff3bc4..ce18ba0b74eb 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -2603,6 +2603,7 @@ struct b53_device *b53_switch_alloc(struct device *base, dev->ops = ops; ds->ops = &b53_switch_ops; ds->configure_vlan_while_not_filtering = true; + ds->untag_bridge_pvid = true; dev->vlan_enabled = ds->configure_vlan_while_not_filtering; mutex_init(&dev->reg_mutex); mutex_init(&dev->stats_mutex); diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c index ab3ee5c3fd02..552b1f7bde17 100644 --- a/drivers/net/dsa/ocelot/felix.c +++ b/drivers/net/dsa/ocelot/felix.c @@ -435,8 +435,6 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports) ocelot->num_stats = felix->info->num_stats; ocelot->shared_queue_sz = felix->info->shared_queue_sz; ocelot->num_mact_rows = felix->info->num_mact_rows; - ocelot->vcap_is2_keys = felix->info->vcap_is2_keys; - ocelot->vcap_is2_actions= felix->info->vcap_is2_actions; ocelot->vcap = felix->info->vcap; ocelot->ops = felix->info->ops; ocelot->inj_prefix = OCELOT_TAG_PREFIX_SHORT; @@ -812,3 +810,25 @@ const struct dsa_switch_ops felix_switch_ops = { .cls_flower_stats = felix_cls_flower_stats, .port_setup_tc = felix_port_setup_tc, }; + +struct net_device *felix_port_to_netdev(struct ocelot *ocelot, int port) +{ + struct felix *felix = ocelot_to_felix(ocelot); + struct dsa_switch *ds = felix->ds; + + if (!dsa_is_user_port(ds, port)) + return NULL; + + return dsa_to_port(ds, port)->slave; +} + +int felix_netdev_to_port(struct net_device *dev) +{ + struct dsa_port *dp; + + dp = dsa_port_from_netdev(dev); + if (IS_ERR(dp)) + return -EINVAL; + + return dp->index; +} diff --git a/drivers/net/dsa/ocelot/felix.h b/drivers/net/dsa/ocelot/felix.h index cc3ec83a600a..4c717324ac2f 100644 --- a/drivers/net/dsa/ocelot/felix.h +++ b/drivers/net/dsa/ocelot/felix.h @@ -21,9 +21,7 @@ struct felix_info { unsigned int num_stats; int num_ports; int num_tx_queues; - struct vcap_field *vcap_is2_keys; - struct vcap_field *vcap_is2_actions; - const struct vcap_props *vcap; + struct vcap_props *vcap; int switch_pci_bar; int imdio_pci_bar; const struct ptp_clock_info *ptp_caps; @@ -54,4 +52,7 @@ struct felix { resource_size_t imdio_base; }; +struct net_device *felix_port_to_netdev(struct ocelot *ocelot, int port); +int felix_netdev_to_port(struct net_device *dev); + #endif diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index 2e9270c00096..875ea60c0df0 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -16,9 +16,6 @@ #include <linux/pci.h> #include "felix.h" -#define VSC9959_VCAP_IS2_CNT 1024 -#define VSC9959_VCAP_IS2_ENTRY_WIDTH 376 -#define VSC9959_VCAP_PORT_CNT 6 #define VSC9959_TAS_GCL_ENTRY_MAX 63 static const u32 vsc9959_ana_regmap[] = { @@ -138,14 +135,27 @@ static const u32 vsc9959_qs_regmap[] = { REG_RESERVED(QS_INH_DBG), }; -static const u32 vsc9959_s2_regmap[] = { - REG(S2_CORE_UPDATE_CTRL, 0x000000), - REG(S2_CORE_MV_CFG, 0x000004), - REG(S2_CACHE_ENTRY_DAT, 0x000008), - REG(S2_CACHE_MASK_DAT, 0x000108), - REG(S2_CACHE_ACTION_DAT, 0x000208), - REG(S2_CACHE_CNT_DAT, 0x000308), - REG(S2_CACHE_TG_DAT, 0x000388), +static const u32 vsc9959_vcap_regmap[] = { + /* VCAP_CORE_CFG */ + REG(VCAP_CORE_UPDATE_CTRL, 0x000000), + REG(VCAP_CORE_MV_CFG, 0x000004), + /* VCAP_CORE_CACHE */ + REG(VCAP_CACHE_ENTRY_DAT, 0x000008), + REG(VCAP_CACHE_MASK_DAT, 0x000108), + REG(VCAP_CACHE_ACTION_DAT, 0x000208), + REG(VCAP_CACHE_CNT_DAT, 0x000308), + REG(VCAP_CACHE_TG_DAT, 0x000388), + /* VCAP_CONST */ + REG(VCAP_CONST_VCAP_VER, 0x000398), + REG(VCAP_CONST_ENTRY_WIDTH, 0x00039c), + REG(VCAP_CONST_ENTRY_CNT, 0x0003a0), + REG(VCAP_CONST_ENTRY_SWCNT, 0x0003a4), + REG(VCAP_CONST_ENTRY_TG_WIDTH, 0x0003a8), + REG(VCAP_CONST_ACTION_DEF_CNT, 0x0003ac), + REG(VCAP_CONST_ACTION_WIDTH, 0x0003b0), + REG(VCAP_CONST_CNT_WIDTH, 0x0003b4), + REG(VCAP_CONST_CORE_CNT, 0x0003b8), + REG(VCAP_CONST_IF_CNT, 0x0003bc), }; static const u32 vsc9959_qsys_regmap[] = { @@ -359,7 +369,9 @@ static const u32 *vsc9959_regmap[TARGET_MAX] = { [QSYS] = vsc9959_qsys_regmap, [REW] = vsc9959_rew_regmap, [SYS] = vsc9959_sys_regmap, - [S2] = vsc9959_s2_regmap, + [S0] = vsc9959_vcap_regmap, + [S1] = vsc9959_vcap_regmap, + [S2] = vsc9959_vcap_regmap, [PTP] = vsc9959_ptp_regmap, [GCB] = vsc9959_gcb_regmap, [DEV_GMII] = vsc9959_dev_gmii_regmap, @@ -392,6 +404,16 @@ static const struct resource vsc9959_target_io_res[TARGET_MAX] = { .end = 0x001ffff, .name = "sys", }, + [S0] = { + .start = 0x0040000, + .end = 0x00403ff, + .name = "s0", + }, + [S1] = { + .start = 0x0050000, + .end = 0x00503ff, + .name = "s1", + }, [S2] = { .start = 0x0060000, .end = 0x00603ff, @@ -596,6 +618,113 @@ static const struct ocelot_stat_layout vsc9959_stats_layout[] = { { .offset = 0x111, .name = "drop_green_prio_7", }, }; +static const struct vcap_field vsc9959_vcap_es0_keys[] = { + [VCAP_ES0_EGR_PORT] = { 0, 3}, + [VCAP_ES0_IGR_PORT] = { 3, 3}, + [VCAP_ES0_RSV] = { 6, 2}, + [VCAP_ES0_L2_MC] = { 8, 1}, + [VCAP_ES0_L2_BC] = { 9, 1}, + [VCAP_ES0_VID] = { 10, 12}, + [VCAP_ES0_DP] = { 22, 1}, + [VCAP_ES0_PCP] = { 23, 3}, +}; + +static const struct vcap_field vsc9959_vcap_es0_actions[] = { + [VCAP_ES0_ACT_PUSH_OUTER_TAG] = { 0, 2}, + [VCAP_ES0_ACT_PUSH_INNER_TAG] = { 2, 1}, + [VCAP_ES0_ACT_TAG_A_TPID_SEL] = { 3, 2}, + [VCAP_ES0_ACT_TAG_A_VID_SEL] = { 5, 1}, + [VCAP_ES0_ACT_TAG_A_PCP_SEL] = { 6, 2}, + [VCAP_ES0_ACT_TAG_A_DEI_SEL] = { 8, 2}, + [VCAP_ES0_ACT_TAG_B_TPID_SEL] = { 10, 2}, + [VCAP_ES0_ACT_TAG_B_VID_SEL] = { 12, 1}, + [VCAP_ES0_ACT_TAG_B_PCP_SEL] = { 13, 2}, + [VCAP_ES0_ACT_TAG_B_DEI_SEL] = { 15, 2}, + [VCAP_ES0_ACT_VID_A_VAL] = { 17, 12}, + [VCAP_ES0_ACT_PCP_A_VAL] = { 29, 3}, + [VCAP_ES0_ACT_DEI_A_VAL] = { 32, 1}, + [VCAP_ES0_ACT_VID_B_VAL] = { 33, 12}, + [VCAP_ES0_ACT_PCP_B_VAL] = { 45, 3}, + [VCAP_ES0_ACT_DEI_B_VAL] = { 48, 1}, + [VCAP_ES0_ACT_RSV] = { 49, 23}, + [VCAP_ES0_ACT_HIT_STICKY] = { 72, 1}, +}; + +static const struct vcap_field vsc9959_vcap_is1_keys[] = { + [VCAP_IS1_HK_TYPE] = { 0, 1}, + [VCAP_IS1_HK_LOOKUP] = { 1, 2}, + [VCAP_IS1_HK_IGR_PORT_MASK] = { 3, 7}, + [VCAP_IS1_HK_RSV] = { 10, 9}, + [VCAP_IS1_HK_OAM_Y1731] = { 19, 1}, + [VCAP_IS1_HK_L2_MC] = { 20, 1}, + [VCAP_IS1_HK_L2_BC] = { 21, 1}, + [VCAP_IS1_HK_IP_MC] = { 22, 1}, + [VCAP_IS1_HK_VLAN_TAGGED] = { 23, 1}, + [VCAP_IS1_HK_VLAN_DBL_TAGGED] = { 24, 1}, + [VCAP_IS1_HK_TPID] = { 25, 1}, + [VCAP_IS1_HK_VID] = { 26, 12}, + [VCAP_IS1_HK_DEI] = { 38, 1}, + [VCAP_IS1_HK_PCP] = { 39, 3}, + /* Specific Fields for IS1 Half Key S1_NORMAL */ + [VCAP_IS1_HK_L2_SMAC] = { 42, 48}, + [VCAP_IS1_HK_ETYPE_LEN] = { 90, 1}, + [VCAP_IS1_HK_ETYPE] = { 91, 16}, + [VCAP_IS1_HK_IP_SNAP] = {107, 1}, + [VCAP_IS1_HK_IP4] = {108, 1}, + /* Layer-3 Information */ + [VCAP_IS1_HK_L3_FRAGMENT] = {109, 1}, + [VCAP_IS1_HK_L3_FRAG_OFS_GT0] = {110, 1}, + [VCAP_IS1_HK_L3_OPTIONS] = {111, 1}, + [VCAP_IS1_HK_L3_DSCP] = {112, 6}, + [VCAP_IS1_HK_L3_IP4_SIP] = {118, 32}, + /* Layer-4 Information */ + [VCAP_IS1_HK_TCP_UDP] = {150, 1}, + [VCAP_IS1_HK_TCP] = {151, 1}, + [VCAP_IS1_HK_L4_SPORT] = {152, 16}, + [VCAP_IS1_HK_L4_RNG] = {168, 8}, + /* Specific Fields for IS1 Half Key S1_5TUPLE_IP4 */ + [VCAP_IS1_HK_IP4_INNER_TPID] = { 42, 1}, + [VCAP_IS1_HK_IP4_INNER_VID] = { 43, 12}, + [VCAP_IS1_HK_IP4_INNER_DEI] = { 55, 1}, + [VCAP_IS1_HK_IP4_INNER_PCP] = { 56, 3}, + [VCAP_IS1_HK_IP4_IP4] = { 59, 1}, + [VCAP_IS1_HK_IP4_L3_FRAGMENT] = { 60, 1}, + [VCAP_IS1_HK_IP4_L3_FRAG_OFS_GT0] = { 61, 1}, + [VCAP_IS1_HK_IP4_L3_OPTIONS] = { 62, 1}, + [VCAP_IS1_HK_IP4_L3_DSCP] = { 63, 6}, + [VCAP_IS1_HK_IP4_L3_IP4_DIP] = { 69, 32}, + [VCAP_IS1_HK_IP4_L3_IP4_SIP] = {101, 32}, + [VCAP_IS1_HK_IP4_L3_PROTO] = {133, 8}, + [VCAP_IS1_HK_IP4_TCP_UDP] = {141, 1}, + [VCAP_IS1_HK_IP4_TCP] = {142, 1}, + [VCAP_IS1_HK_IP4_L4_RNG] = {143, 8}, + [VCAP_IS1_HK_IP4_IP_PAYLOAD_S1_5TUPLE] = {151, 32}, +}; + +static const struct vcap_field vsc9959_vcap_is1_actions[] = { + [VCAP_IS1_ACT_DSCP_ENA] = { 0, 1}, + [VCAP_IS1_ACT_DSCP_VAL] = { 1, 6}, + [VCAP_IS1_ACT_QOS_ENA] = { 7, 1}, + [VCAP_IS1_ACT_QOS_VAL] = { 8, 3}, + [VCAP_IS1_ACT_DP_ENA] = { 11, 1}, + [VCAP_IS1_ACT_DP_VAL] = { 12, 1}, + [VCAP_IS1_ACT_PAG_OVERRIDE_MASK] = { 13, 8}, + [VCAP_IS1_ACT_PAG_VAL] = { 21, 8}, + [VCAP_IS1_ACT_RSV] = { 29, 9}, + /* The fields below are incorrectly shifted by 2 in the manual */ + [VCAP_IS1_ACT_VID_REPLACE_ENA] = { 38, 1}, + [VCAP_IS1_ACT_VID_ADD_VAL] = { 39, 12}, + [VCAP_IS1_ACT_FID_SEL] = { 51, 2}, + [VCAP_IS1_ACT_FID_VAL] = { 53, 13}, + [VCAP_IS1_ACT_PCP_DEI_ENA] = { 66, 1}, + [VCAP_IS1_ACT_PCP_VAL] = { 67, 3}, + [VCAP_IS1_ACT_DEI_VAL] = { 70, 1}, + [VCAP_IS1_ACT_VLAN_POP_CNT_ENA] = { 71, 1}, + [VCAP_IS1_ACT_VLAN_POP_CNT] = { 72, 2}, + [VCAP_IS1_ACT_CUSTOM_ACE_TYPE_ENA] = { 74, 4}, + [VCAP_IS1_ACT_HIT_STICKY] = { 78, 1}, +}; + static struct vcap_field vsc9959_vcap_is2_keys[] = { /* Common: 41 bits */ [VCAP_IS2_TYPE] = { 0, 4}, @@ -694,15 +823,32 @@ static struct vcap_field vsc9959_vcap_is2_actions[] = { [VCAP_IS2_ACT_HIT_CNT] = { 49, 32}, }; -static const struct vcap_props vsc9959_vcap_props[] = { +static struct vcap_props vsc9959_vcap_props[] = { + [VCAP_ES0] = { + .action_type_width = 0, + .action_table = { + [ES0_ACTION_TYPE_NORMAL] = { + .width = 72, /* HIT_STICKY not included */ + .count = 1, + }, + }, + .target = S0, + .keys = vsc9959_vcap_es0_keys, + .actions = vsc9959_vcap_es0_actions, + }, + [VCAP_IS1] = { + .action_type_width = 0, + .action_table = { + [IS1_ACTION_TYPE_NORMAL] = { + .width = 78, /* HIT_STICKY not included */ + .count = 4, + }, + }, + .target = S1, + .keys = vsc9959_vcap_is1_keys, + .actions = vsc9959_vcap_is1_actions, + }, [VCAP_IS2] = { - .tg_width = 2, - .sw_count = 4, - .entry_count = VSC9959_VCAP_IS2_CNT, - .entry_width = VSC9959_VCAP_IS2_ENTRY_WIDTH, - .action_count = VSC9959_VCAP_IS2_CNT + - VSC9959_VCAP_PORT_CNT + 2, - .action_width = 89, .action_type_width = 1, .action_table = { [IS2_ACTION_TYPE_NORMAL] = { @@ -714,8 +860,9 @@ static const struct vcap_props vsc9959_vcap_props[] = { .count = 4 }, }, - .counter_words = 4, - .counter_width = 32, + .target = S2, + .keys = vsc9959_vcap_is2_keys, + .actions = vsc9959_vcap_is2_actions, }, }; @@ -860,6 +1007,8 @@ static u16 vsc9959_wm_enc(u16 value) static const struct ocelot_ops vsc9959_ops = { .reset = vsc9959_reset, .wm_enc = vsc9959_wm_enc, + .port_to_netdev = felix_port_to_netdev, + .netdev_to_port = felix_netdev_to_port, }; static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot) @@ -1184,8 +1333,6 @@ static const struct felix_info felix_info_vsc9959 = { .ops = &vsc9959_ops, .stats_layout = vsc9959_stats_layout, .num_stats = ARRAY_SIZE(vsc9959_stats_layout), - .vcap_is2_keys = vsc9959_vcap_is2_keys, - .vcap_is2_actions = vsc9959_vcap_is2_actions, .vcap = vsc9959_vcap_props, .shared_queue_sz = 128 * 1024, .num_mact_rows = 2048, diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index e28dd600f464..f9b7507a4eb3 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -12,10 +12,6 @@ #include <linux/iopoll.h> #include "felix.h" -#define VSC9953_VCAP_IS2_CNT 1024 -#define VSC9953_VCAP_IS2_ENTRY_WIDTH 376 -#define VSC9953_VCAP_PORT_CNT 10 - #define MSCC_MIIM_CMD_OPR_WRITE BIT(1) #define MSCC_MIIM_CMD_OPR_READ BIT(2) #define MSCC_MIIM_CMD_WRDATA_SHIFT 4 @@ -140,14 +136,27 @@ static const u32 vsc9953_qs_regmap[] = { REG_RESERVED(QS_INH_DBG), }; -static const u32 vsc9953_s2_regmap[] = { - REG(S2_CORE_UPDATE_CTRL, 0x000000), - REG(S2_CORE_MV_CFG, 0x000004), - REG(S2_CACHE_ENTRY_DAT, 0x000008), - REG(S2_CACHE_MASK_DAT, 0x000108), - REG(S2_CACHE_ACTION_DAT, 0x000208), - REG(S2_CACHE_CNT_DAT, 0x000308), - REG(S2_CACHE_TG_DAT, 0x000388), +static const u32 vsc9953_vcap_regmap[] = { + /* VCAP_CORE_CFG */ + REG(VCAP_CORE_UPDATE_CTRL, 0x000000), + REG(VCAP_CORE_MV_CFG, 0x000004), + /* VCAP_CORE_CACHE */ + REG(VCAP_CACHE_ENTRY_DAT, 0x000008), + REG(VCAP_CACHE_MASK_DAT, 0x000108), + REG(VCAP_CACHE_ACTION_DAT, 0x000208), + REG(VCAP_CACHE_CNT_DAT, 0x000308), + REG(VCAP_CACHE_TG_DAT, 0x000388), + /* VCAP_CONST */ + REG(VCAP_CONST_VCAP_VER, 0x000398), + REG(VCAP_CONST_ENTRY_WIDTH, 0x00039c), + REG(VCAP_CONST_ENTRY_CNT, 0x0003a0), + REG(VCAP_CONST_ENTRY_SWCNT, 0x0003a4), + REG(VCAP_CONST_ENTRY_TG_WIDTH, 0x0003a8), + REG(VCAP_CONST_ACTION_DEF_CNT, 0x0003ac), + REG(VCAP_CONST_ACTION_WIDTH, 0x0003b0), + REG(VCAP_CONST_CNT_WIDTH, 0x0003b4), + REG_RESERVED(VCAP_CONST_CORE_CNT), + REG_RESERVED(VCAP_CONST_IF_CNT), }; static const u32 vsc9953_qsys_regmap[] = { @@ -352,7 +361,9 @@ static const u32 *vsc9953_regmap[TARGET_MAX] = { [QSYS] = vsc9953_qsys_regmap, [REW] = vsc9953_rew_regmap, [SYS] = vsc9953_sys_regmap, - [S2] = vsc9953_s2_regmap, + [S0] = vsc9953_vcap_regmap, + [S1] = vsc9953_vcap_regmap, + [S2] = vsc9953_vcap_regmap, [GCB] = vsc9953_gcb_regmap, [DEV_GMII] = vsc9953_dev_gmii_regmap, }; @@ -384,6 +395,16 @@ static const struct resource vsc9953_target_io_res[TARGET_MAX] = { .end = 0x001ffff, .name = "sys", }, + [S0] = { + .start = 0x0040000, + .end = 0x00403ff, + .name = "s0", + }, + [S1] = { + .start = 0x0050000, + .end = 0x00503ff, + .name = "s1", + }, [S2] = { .start = 0x0060000, .end = 0x00603ff, @@ -599,6 +620,112 @@ static const struct ocelot_stat_layout vsc9953_stats_layout[] = { { .offset = 0x91, .name = "drop_green_prio_7", }, }; +static const struct vcap_field vsc9953_vcap_es0_keys[] = { + [VCAP_ES0_EGR_PORT] = { 0, 4}, + [VCAP_ES0_IGR_PORT] = { 4, 4}, + [VCAP_ES0_RSV] = { 8, 2}, + [VCAP_ES0_L2_MC] = { 10, 1}, + [VCAP_ES0_L2_BC] = { 11, 1}, + [VCAP_ES0_VID] = { 12, 12}, + [VCAP_ES0_DP] = { 24, 1}, + [VCAP_ES0_PCP] = { 25, 3}, +}; + +static const struct vcap_field vsc9953_vcap_es0_actions[] = { + [VCAP_ES0_ACT_PUSH_OUTER_TAG] = { 0, 2}, + [VCAP_ES0_ACT_PUSH_INNER_TAG] = { 2, 1}, + [VCAP_ES0_ACT_TAG_A_TPID_SEL] = { 3, 2}, + [VCAP_ES0_ACT_TAG_A_VID_SEL] = { 5, 1}, + [VCAP_ES0_ACT_TAG_A_PCP_SEL] = { 6, 2}, + [VCAP_ES0_ACT_TAG_A_DEI_SEL] = { 8, 2}, + [VCAP_ES0_ACT_TAG_B_TPID_SEL] = { 10, 2}, + [VCAP_ES0_ACT_TAG_B_VID_SEL] = { 12, 1}, + [VCAP_ES0_ACT_TAG_B_PCP_SEL] = { 13, 2}, + [VCAP_ES0_ACT_TAG_B_DEI_SEL] = { 15, 2}, + [VCAP_ES0_ACT_VID_A_VAL] = { 17, 12}, + [VCAP_ES0_ACT_PCP_A_VAL] = { 29, 3}, + [VCAP_ES0_ACT_DEI_A_VAL] = { 32, 1}, + [VCAP_ES0_ACT_VID_B_VAL] = { 33, 12}, + [VCAP_ES0_ACT_PCP_B_VAL] = { 45, 3}, + [VCAP_ES0_ACT_DEI_B_VAL] = { 48, 1}, + [VCAP_ES0_ACT_RSV] = { 49, 24}, + [VCAP_ES0_ACT_HIT_STICKY] = { 73, 1}, +}; + +static const struct vcap_field vsc9953_vcap_is1_keys[] = { + [VCAP_IS1_HK_TYPE] = { 0, 1}, + [VCAP_IS1_HK_LOOKUP] = { 1, 2}, + [VCAP_IS1_HK_IGR_PORT_MASK] = { 3, 11}, + [VCAP_IS1_HK_RSV] = { 14, 10}, + /* VCAP_IS1_HK_OAM_Y1731 not supported */ + [VCAP_IS1_HK_L2_MC] = { 24, 1}, + [VCAP_IS1_HK_L2_BC] = { 25, 1}, + [VCAP_IS1_HK_IP_MC] = { 26, 1}, + [VCAP_IS1_HK_VLAN_TAGGED] = { 27, 1}, + [VCAP_IS1_HK_VLAN_DBL_TAGGED] = { 28, 1}, + [VCAP_IS1_HK_TPID] = { 29, 1}, + [VCAP_IS1_HK_VID] = { 30, 12}, + [VCAP_IS1_HK_DEI] = { 42, 1}, + [VCAP_IS1_HK_PCP] = { 43, 3}, + /* Specific Fields for IS1 Half Key S1_NORMAL */ + [VCAP_IS1_HK_L2_SMAC] = { 46, 48}, + [VCAP_IS1_HK_ETYPE_LEN] = { 94, 1}, + [VCAP_IS1_HK_ETYPE] = { 95, 16}, + [VCAP_IS1_HK_IP_SNAP] = {111, 1}, + [VCAP_IS1_HK_IP4] = {112, 1}, + /* Layer-3 Information */ + [VCAP_IS1_HK_L3_FRAGMENT] = {113, 1}, + [VCAP_IS1_HK_L3_FRAG_OFS_GT0] = {114, 1}, + [VCAP_IS1_HK_L3_OPTIONS] = {115, 1}, + [VCAP_IS1_HK_L3_DSCP] = {116, 6}, + [VCAP_IS1_HK_L3_IP4_SIP] = {122, 32}, + /* Layer-4 Information */ + [VCAP_IS1_HK_TCP_UDP] = {154, 1}, + [VCAP_IS1_HK_TCP] = {155, 1}, + [VCAP_IS1_HK_L4_SPORT] = {156, 16}, + [VCAP_IS1_HK_L4_RNG] = {172, 8}, + /* Specific Fields for IS1 Half Key S1_5TUPLE_IP4 */ + [VCAP_IS1_HK_IP4_INNER_TPID] = { 46, 1}, + [VCAP_IS1_HK_IP4_INNER_VID] = { 47, 12}, + [VCAP_IS1_HK_IP4_INNER_DEI] = { 59, 1}, + [VCAP_IS1_HK_IP4_INNER_PCP] = { 60, 3}, + [VCAP_IS1_HK_IP4_IP4] = { 63, 1}, + [VCAP_IS1_HK_IP4_L3_FRAGMENT] = { 64, 1}, + [VCAP_IS1_HK_IP4_L3_FRAG_OFS_GT0] = { 65, 1}, + [VCAP_IS1_HK_IP4_L3_OPTIONS] = { 66, 1}, + [VCAP_IS1_HK_IP4_L3_DSCP] = { 67, 6}, + [VCAP_IS1_HK_IP4_L3_IP4_DIP] = { 73, 32}, + [VCAP_IS1_HK_IP4_L3_IP4_SIP] = {105, 32}, + [VCAP_IS1_HK_IP4_L3_PROTO] = {137, 8}, + [VCAP_IS1_HK_IP4_TCP_UDP] = {145, 1}, + [VCAP_IS1_HK_IP4_TCP] = {146, 1}, + [VCAP_IS1_HK_IP4_L4_RNG] = {147, 8}, + [VCAP_IS1_HK_IP4_IP_PAYLOAD_S1_5TUPLE] = {155, 32}, +}; + +static const struct vcap_field vsc9953_vcap_is1_actions[] = { + [VCAP_IS1_ACT_DSCP_ENA] = { 0, 1}, + [VCAP_IS1_ACT_DSCP_VAL] = { 1, 6}, + [VCAP_IS1_ACT_QOS_ENA] = { 7, 1}, + [VCAP_IS1_ACT_QOS_VAL] = { 8, 3}, + [VCAP_IS1_ACT_DP_ENA] = { 11, 1}, + [VCAP_IS1_ACT_DP_VAL] = { 12, 1}, + [VCAP_IS1_ACT_PAG_OVERRIDE_MASK] = { 13, 8}, + [VCAP_IS1_ACT_PAG_VAL] = { 21, 8}, + [VCAP_IS1_ACT_RSV] = { 29, 11}, + [VCAP_IS1_ACT_VID_REPLACE_ENA] = { 40, 1}, + [VCAP_IS1_ACT_VID_ADD_VAL] = { 41, 12}, + [VCAP_IS1_ACT_FID_SEL] = { 53, 2}, + [VCAP_IS1_ACT_FID_VAL] = { 55, 13}, + [VCAP_IS1_ACT_PCP_DEI_ENA] = { 68, 1}, + [VCAP_IS1_ACT_PCP_VAL] = { 69, 3}, + [VCAP_IS1_ACT_DEI_VAL] = { 72, 1}, + [VCAP_IS1_ACT_VLAN_POP_CNT_ENA] = { 73, 1}, + [VCAP_IS1_ACT_VLAN_POP_CNT] = { 74, 2}, + [VCAP_IS1_ACT_CUSTOM_ACE_TYPE_ENA] = { 76, 4}, + [VCAP_IS1_ACT_HIT_STICKY] = { 80, 1}, +}; + static struct vcap_field vsc9953_vcap_is2_keys[] = { /* Common: 41 bits */ [VCAP_IS2_TYPE] = { 0, 4}, @@ -684,15 +811,32 @@ static struct vcap_field vsc9953_vcap_is2_actions[] = { [VCAP_IS2_ACT_HIT_CNT] = { 50, 32}, }; -static const struct vcap_props vsc9953_vcap_props[] = { +static struct vcap_props vsc9953_vcap_props[] = { + [VCAP_ES0] = { + .action_type_width = 0, + .action_table = { + [ES0_ACTION_TYPE_NORMAL] = { + .width = 73, /* HIT_STICKY not included */ + .count = 1, + }, + }, + .target = S0, + .keys = vsc9953_vcap_es0_keys, + .actions = vsc9953_vcap_es0_actions, + }, + [VCAP_IS1] = { + .action_type_width = 0, + .action_table = { + [IS1_ACTION_TYPE_NORMAL] = { + .width = 80, /* HIT_STICKY not included */ + .count = 4, + }, + }, + .target = S1, + .keys = vsc9953_vcap_is1_keys, + .actions = vsc9953_vcap_is1_actions, + }, [VCAP_IS2] = { - .tg_width = 2, - .sw_count = 4, - .entry_count = VSC9953_VCAP_IS2_CNT, - .entry_width = VSC9953_VCAP_IS2_ENTRY_WIDTH, - .action_count = VSC9953_VCAP_IS2_CNT + - VSC9953_VCAP_PORT_CNT + 2, - .action_width = 101, .action_type_width = 1, .action_table = { [IS2_ACTION_TYPE_NORMAL] = { @@ -704,8 +848,9 @@ static const struct vcap_props vsc9953_vcap_props[] = { .count = 4 }, }, - .counter_words = 4, - .counter_width = 32, + .target = S2, + .keys = vsc9953_vcap_is2_keys, + .actions = vsc9953_vcap_is2_actions, }, }; @@ -913,6 +1058,8 @@ static u16 vsc9953_wm_enc(u16 value) static const struct ocelot_ops vsc9953_ops = { .reset = vsc9953_reset, .wm_enc = vsc9953_wm_enc, + .port_to_netdev = felix_port_to_netdev, + .netdev_to_port = felix_netdev_to_port, }; static int vsc9953_mdio_bus_alloc(struct ocelot *ocelot) @@ -1031,8 +1178,6 @@ static const struct felix_info seville_info_vsc9953 = { .ops = &vsc9953_ops, .stats_layout = vsc9953_stats_layout, .num_stats = ARRAY_SIZE(vsc9953_stats_layout), - .vcap_is2_keys = vsc9953_vcap_is2_keys, - .vcap_is2_actions = vsc9953_vcap_is2_actions, .vcap = vsc9953_vcap_props, .shared_queue_sz = 2048 * 1024, .num_mact_rows = 2048, diff --git a/drivers/net/ethernet/8390/lib8390.c b/drivers/net/ethernet/8390/lib8390.c index 1f48d7f6365c..e84021282edf 100644 --- a/drivers/net/ethernet/8390/lib8390.c +++ b/drivers/net/ethernet/8390/lib8390.c @@ -50,6 +50,7 @@ */ +#include <linux/build_bug.h> #include <linux/module.h> #include <linux/kernel.h> #include <linux/jiffies.h> @@ -112,8 +113,10 @@ static void do_set_multicast_list(struct net_device *dev); static void __NS8390_init(struct net_device *dev, int startp); static unsigned version_printed; -static u32 msg_enable; -module_param(msg_enable, uint, 0444); +static int msg_enable; +static const int default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_RX_ERR | + NETIF_MSG_TX_ERR); +module_param(msg_enable, int, 0444); MODULE_PARM_DESC(msg_enable, "Debug message level (see linux/netdevice.h for bitmap)"); /* @@ -973,14 +976,14 @@ static void ethdev_setup(struct net_device *dev) { struct ei_device *ei_local = netdev_priv(dev); - if ((msg_enable & NETIF_MSG_DRV) && (version_printed++ == 0)) - pr_info("%s", version); - ether_setup(dev); spin_lock_init(&ei_local->page_lock); - ei_local->msg_enable = msg_enable; + ei_local->msg_enable = netif_msg_init(msg_enable, default_msg_level); + + if (netif_msg_drv(ei_local) && (version_printed++ == 0)) + pr_info("%s", version); } /** @@ -1018,8 +1021,7 @@ static void __NS8390_init(struct net_device *dev, int startp) ? (0x48 | ENDCFG_WTS | (ei_local->bigendian ? ENDCFG_BOS : 0)) : 0x48; - if (sizeof(struct e8390_pkt_hdr) != 4) - panic("8390.c: header struct mispacked\n"); + BUILD_BUG_ON(sizeof(struct e8390_pkt_hdr) != 4); /* Follow National Semi's recommendations for initing the DP83902. */ ei_outb_p(E8390_NODMA+E8390_PAGE0+E8390_STOP, e8390_base+E8390_CMD); /* 0x21 */ ei_outb_p(endcfg, e8390_base + EN0_DCFG); /* 0x48 or 0x49 */ diff --git a/drivers/net/ethernet/amd/sun3lance.c b/drivers/net/ethernet/amd/sun3lance.c index e1fde585fd0d..00ae1081254d 100644 --- a/drivers/net/ethernet/amd/sun3lance.c +++ b/drivers/net/ethernet/amd/sun3lance.c @@ -657,16 +657,6 @@ static irqreturn_t lance_interrupt( int irq, void *dev_id) struct net_device *dev = dev_id; struct lance_private *lp = netdev_priv(dev); int csr0; - static int in_interrupt; - - if (dev == NULL) { - DPRINTK( 1, ( "lance_interrupt(): invalid dev_id\n" )); - return IRQ_NONE; - } - - if (in_interrupt) - DPRINTK( 2, ( "%s: Re-entering the interrupt handler.\n", dev->name )); - in_interrupt = 1; still_more: flush_cache_all(); @@ -774,7 +764,6 @@ static irqreturn_t lance_interrupt( int irq, void *dev_id) DPRINTK( 2, ( "%s: exiting interrupt, csr0=%#04x.\n", dev->name, DREG )); - in_interrupt = 0; return IRQ_HANDLED; } diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c index 00de463268a8..0c12cf7bda50 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c +++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c @@ -220,7 +220,6 @@ static void atl1c_phy_config(struct timer_list *t) void atl1c_reinit_locked(struct atl1c_adapter *adapter) { - WARN_ON(in_interrupt()); atl1c_down(adapter); atl1c_up(adapter); clear_bit(__AT_RESETTING, &adapter->flags); diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c index e43f71849daf..098b0328e3cb 100644 --- a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c +++ b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c @@ -127,8 +127,6 @@ static void atl1e_phy_config(struct timer_list *t) void atl1e_reinit_locked(struct atl1e_adapter *adapter) { - - WARN_ON(in_interrupt()); while (test_and_set_bit(__AT_RESETTING, &adapter->flags)) msleep(1); atl1e_down(adapter); diff --git a/drivers/net/ethernet/atheros/atlx/atl2.c b/drivers/net/ethernet/atheros/atlx/atl2.c index 021fb0e6417a..7b80d924632a 100644 --- a/drivers/net/ethernet/atheros/atlx/atl2.c +++ b/drivers/net/ethernet/atheros/atlx/atl2.c @@ -1086,7 +1086,6 @@ err_up: static void atl2_reinit_locked(struct atl2_adapter *adapter) { - WARN_ON(in_interrupt()); while (test_and_set_bit(__ATL2_RESETTING, &adapter->flags)) msleep(1); atl2_down(adapter); diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index 4f1b41569260..3fd5c6cc23af 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -7,6 +7,7 @@ #ifndef _MACB_H #define _MACB_H +#include <linux/clk.h> #include <linux/phylink.h> #include <linux/ptp_clock_kernel.h> #include <linux/net_tstamp.h> @@ -1298,4 +1299,14 @@ static inline bool gem_has_ptp(struct macb *bp) return !!(bp->caps & MACB_CAPS_GEM_HAS_PTP); } +/** + * struct macb_platform_data - platform data for MACB Ethernet used for PCI registration + * @pclk: platform clock + * @hclk: AHB clock + */ +struct macb_platform_data { + struct clk *pclk; + struct clk *hclk; +}; + #endif /* _MACB_H */ diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index f00ad73ea8e3..4b42b2d6398c 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -23,7 +23,6 @@ #include <linux/netdevice.h> #include <linux/etherdevice.h> #include <linux/dma-mapping.h> -#include <linux/platform_data/macb.h> #include <linux/platform_device.h> #include <linux/phylink.h> #include <linux/of.h> diff --git a/drivers/net/ethernet/cadence/macb_pci.c b/drivers/net/ethernet/cadence/macb_pci.c index 35316c91f523..353393dea639 100644 --- a/drivers/net/ethernet/cadence/macb_pci.c +++ b/drivers/net/ethernet/cadence/macb_pci.c @@ -13,7 +13,6 @@ #include <linux/etherdevice.h> #include <linux/module.h> #include <linux/pci.h> -#include <linux/platform_data/macb.h> #include <linux/platform_device.h> #include "macb.h" diff --git a/drivers/net/ethernet/chelsio/cxgb3/adapter.h b/drivers/net/ethernet/chelsio/cxgb3/adapter.h index 087ff0ffb597..f80fbd81b609 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/adapter.h +++ b/drivers/net/ethernet/chelsio/cxgb3/adapter.h @@ -313,6 +313,7 @@ void t3_os_link_fault(struct adapter *adapter, int port_id, int state); void t3_os_link_fault_handler(struct adapter *adapter, int port_id); void t3_sge_start(struct adapter *adap); +void t3_sge_stop_dma(struct adapter *adap); void t3_sge_stop(struct adapter *adap); void t3_start_sge_timers(struct adapter *adap); void t3_stop_sge_timers(struct adapter *adap); diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c index 6ed58bc12020..84ad7261e243 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c @@ -2996,7 +2996,7 @@ void t3_fatal_err(struct adapter *adapter) unsigned int fw_status[4]; if (adapter->flags & FULL_INIT_DONE) { - t3_sge_stop(adapter); + t3_sge_stop_dma(adapter); t3_write_reg(adapter, A_XGM_TX_CTRL, 0); t3_write_reg(adapter, A_XGM_RX_CTRL, 0); t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0); diff --git a/drivers/net/ethernet/chelsio/cxgb3/sge.c b/drivers/net/ethernet/chelsio/cxgb3/sge.c index 5bb3f0cb36ed..e18e9ce27f94 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb3/sge.c @@ -3270,30 +3270,40 @@ void t3_sge_start(struct adapter *adap) } /** - * t3_sge_stop - disable SGE operation + * t3_sge_stop_dma - Disable SGE DMA engine operation * @adap: the adapter * - * Disables the DMA engine. This can be called in emeregencies (e.g., - * from error interrupts) or from normal process context. In the latter - * case it also disables any pending queue restart tasklets. Note that - * if it is called in interrupt context it cannot disable the restart - * tasklets as it cannot wait, however the tasklets will have no effect - * since the doorbells are disabled and the driver will call this again - * later from process context, at which time the tasklets will be stopped - * if they are still running. + * Can be invoked from interrupt context e.g. error handler. + * + * Note that this function cannot disable the restart of tasklets as + * it cannot wait if called from interrupt context, however the + * tasklets will have no effect since the doorbells are disabled. The + * driver will call tg3_sge_stop() later from process context, at + * which time the tasklets will be stopped if they are still running. */ -void t3_sge_stop(struct adapter *adap) +void t3_sge_stop_dma(struct adapter *adap) { t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, 0); - if (!in_interrupt()) { - int i; +} + +/** + * t3_sge_stop - disable SGE operation completly + * @adap: the adapter + * + * Called from process context. Disables the DMA engine and any + * pending queue restart tasklets. + */ +void t3_sge_stop(struct adapter *adap) +{ + int i; - for (i = 0; i < SGE_QSETS; ++i) { - struct sge_qset *qs = &adap->sge.qs[i]; + t3_sge_stop_dma(adap); - tasklet_kill(&qs->txq[TXQ_OFLD].qresume_tsk); - tasklet_kill(&qs->txq[TXQ_CTRL].qresume_tsk); - } + for (i = 0; i < SGE_QSETS; ++i) { + struct sge_qset *qs = &adap->sge.qs[i]; + + tasklet_kill(&qs->txq[TXQ_OFLD].qresume_tsk); + tasklet_kill(&qs->txq[TXQ_CTRL].qresume_tsk); } } diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c index 5491a41fd1be..0273f40b85f7 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c @@ -3527,6 +3527,10 @@ DEFINE_SHOW_ATTRIBUTE(meminfo); static int chcr_stats_show(struct seq_file *seq, void *v) { +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + struct ch_ktls_port_stats_debug *ktls_port; + int i = 0; +#endif struct adapter *adap = seq->private; seq_puts(seq, "Chelsio Crypto Accelerator Stats \n"); @@ -3557,18 +3561,6 @@ static int chcr_stats_show(struct seq_file *seq, void *v) seq_puts(seq, "\nChelsio KTLS Crypto Accelerator Stats\n"); seq_printf(seq, "Tx TLS offload refcount: %20u\n", refcount_read(&adap->chcr_ktls.ktls_refcount)); - seq_printf(seq, "Tx HW offload contexts added: %20llu\n", - atomic64_read(&adap->ch_ktls_stats.ktls_tx_ctx)); - seq_printf(seq, "Tx connection created: %20llu\n", - atomic64_read(&adap->ch_ktls_stats.ktls_tx_connection_open)); - seq_printf(seq, "Tx connection failed: %20llu\n", - atomic64_read(&adap->ch_ktls_stats.ktls_tx_connection_fail)); - seq_printf(seq, "Tx connection closed: %20llu\n", - atomic64_read(&adap->ch_ktls_stats.ktls_tx_connection_close)); - seq_printf(seq, "Packets passed for encryption : %20llu\n", - atomic64_read(&adap->ch_ktls_stats.ktls_tx_encrypted_packets)); - seq_printf(seq, "Bytes passed for encryption : %20llu\n", - atomic64_read(&adap->ch_ktls_stats.ktls_tx_encrypted_bytes)); seq_printf(seq, "Tx records send: %20llu\n", atomic64_read(&adap->ch_ktls_stats.ktls_tx_send_records)); seq_printf(seq, "Tx partial start of records: %20llu\n", @@ -3581,14 +3573,17 @@ static int chcr_stats_show(struct seq_file *seq, void *v) atomic64_read(&adap->ch_ktls_stats.ktls_tx_complete_pkts)); seq_printf(seq, "TX trim pkts : %20llu\n", atomic64_read(&adap->ch_ktls_stats.ktls_tx_trimmed_pkts)); - seq_printf(seq, "Tx out of order packets: %20llu\n", - atomic64_read(&adap->ch_ktls_stats.ktls_tx_ooo)); - seq_printf(seq, "Tx drop pkts before HW offload: %20llu\n", - atomic64_read(&adap->ch_ktls_stats.ktls_tx_skip_no_sync_data)); - seq_printf(seq, "Tx drop not synced packets: %20llu\n", - atomic64_read(&adap->ch_ktls_stats.ktls_tx_drop_no_sync_data)); - seq_printf(seq, "Tx drop bypass req: %20llu\n", - atomic64_read(&adap->ch_ktls_stats.ktls_tx_drop_bypass_req)); + while (i < MAX_NPORTS) { + ktls_port = &adap->ch_ktls_stats.ktls_port[i]; + seq_printf(seq, "Port %d\n", i); + seq_printf(seq, "Tx connection created: %20llu\n", + atomic64_read(&ktls_port->ktls_tx_connection_open)); + seq_printf(seq, "Tx connection failed: %20llu\n", + atomic64_read(&ktls_port->ktls_tx_connection_fail)); + seq_printf(seq, "Tx connection closed: %20llu\n", + atomic64_read(&ktls_port->ktls_tx_connection_close)); + i++; + } #endif return 0; } diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c index 8c6c9bedcba1..61ea3ec5c3fc 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c @@ -117,14 +117,6 @@ static const char stats_strings[][ETH_GSTRING_LEN] = { "vlan_insertions ", "gro_packets ", "gro_merged ", -}; - -static char adapter_stats_strings[][ETH_GSTRING_LEN] = { - "db_drop ", - "db_full ", - "db_empty ", - "write_coal_success ", - "write_coal_fail ", #if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) "tx_tls_encrypted_packets", "tx_tls_encrypted_bytes ", @@ -136,6 +128,14 @@ static char adapter_stats_strings[][ETH_GSTRING_LEN] = { #endif }; +static char adapter_stats_strings[][ETH_GSTRING_LEN] = { + "db_drop ", + "db_full ", + "db_empty ", + "write_coal_success ", + "write_coal_fail ", +}; + static char loopback_stats_strings[][ETH_GSTRING_LEN] = { "-------Loopback----------- ", "octets_ok ", @@ -257,14 +257,6 @@ struct queue_port_stats { u64 vlan_ins; u64 gro_pkts; u64 gro_merged; -}; - -struct adapter_stats { - u64 db_drop; - u64 db_full; - u64 db_empty; - u64 wc_success; - u64 wc_fail; #if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) u64 tx_tls_encrypted_packets; u64 tx_tls_encrypted_bytes; @@ -276,12 +268,23 @@ struct adapter_stats { #endif }; +struct adapter_stats { + u64 db_drop; + u64 db_full; + u64 db_empty; + u64 wc_success; + u64 wc_fail; +}; + static void collect_sge_port_stats(const struct adapter *adap, const struct port_info *p, struct queue_port_stats *s) { const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset]; const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset]; +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + const struct ch_ktls_port_stats_debug *ktls_stats; +#endif struct sge_eohw_txq *eohw_tx; unsigned int i; @@ -306,6 +309,21 @@ static void collect_sge_port_stats(const struct adapter *adap, s->vlan_ins += eohw_tx->vlan_ins; } } +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + ktls_stats = &adap->ch_ktls_stats.ktls_port[p->port_id]; + s->tx_tls_encrypted_packets = + atomic64_read(&ktls_stats->ktls_tx_encrypted_packets); + s->tx_tls_encrypted_bytes = + atomic64_read(&ktls_stats->ktls_tx_encrypted_bytes); + s->tx_tls_ctx = atomic64_read(&ktls_stats->ktls_tx_ctx); + s->tx_tls_ooo = atomic64_read(&ktls_stats->ktls_tx_ooo); + s->tx_tls_skip_no_sync_data = + atomic64_read(&ktls_stats->ktls_tx_skip_no_sync_data); + s->tx_tls_drop_no_sync_data = + atomic64_read(&ktls_stats->ktls_tx_drop_no_sync_data); + s->tx_tls_drop_bypass_req = + atomic64_read(&ktls_stats->ktls_tx_drop_bypass_req); +#endif } static void collect_adapter_stats(struct adapter *adap, struct adapter_stats *s) diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c index b154190e1ee2..743af9e654aa 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c @@ -690,8 +690,8 @@ int cxgb4_set_ktls_feature(struct adapter *adap, bool enable) * ULD is/are already active, return failure. */ if (cxgb4_uld_in_use(adap)) { - dev_warn(adap->pdev_dev, - "ULD connections (tid/stid) active. Can't enable kTLS\n"); + dev_dbg(adap->pdev_dev, + "ULD connections (tid/stid) active. Can't enable kTLS\n"); return -EINVAL; } ret = t4_set_params(adap, adap->mbox, adap->pf, @@ -699,7 +699,7 @@ int cxgb4_set_ktls_feature(struct adapter *adap, bool enable) if (ret) return ret; refcount_set(&adap->chcr_ktls.ktls_refcount, 1); - pr_info("kTLS has been enabled. Restrictions placed on ULD support\n"); + pr_debug("kTLS has been enabled. Restrictions placed on ULD support\n"); } else { /* ktls settings already up, just increment refcount. */ refcount_inc(&adap->chcr_ktls.ktls_refcount); @@ -716,7 +716,7 @@ int cxgb4_set_ktls_feature(struct adapter *adap, bool enable) 0, 1, ¶ms, ¶ms); if (ret) return ret; - pr_info("kTLS is disabled. Restrictions on ULD support removed\n"); + pr_debug("kTLS is disabled. Restrictions on ULD support removed\n"); } } diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h index ea2fabbdd934..b169776ab484 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h @@ -44,6 +44,7 @@ #include "cxgb4.h" #define MAX_ULD_QSETS 16 +#define MAX_ULD_NPORTS 4 /* CPL message priority levels */ enum { @@ -365,17 +366,10 @@ struct cxgb4_virt_res { /* virtualized HW resources */ }; #if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) -struct ch_ktls_stats_debug { +struct ch_ktls_port_stats_debug { atomic64_t ktls_tx_connection_open; atomic64_t ktls_tx_connection_fail; atomic64_t ktls_tx_connection_close; - atomic64_t ktls_tx_send_records; - atomic64_t ktls_tx_end_pkts; - atomic64_t ktls_tx_start_pkts; - atomic64_t ktls_tx_middle_pkts; - atomic64_t ktls_tx_retransmit_pkts; - atomic64_t ktls_tx_complete_pkts; - atomic64_t ktls_tx_trimmed_pkts; atomic64_t ktls_tx_encrypted_packets; atomic64_t ktls_tx_encrypted_bytes; atomic64_t ktls_tx_ctx; @@ -384,6 +378,17 @@ struct ch_ktls_stats_debug { atomic64_t ktls_tx_drop_no_sync_data; atomic64_t ktls_tx_drop_bypass_req; }; + +struct ch_ktls_stats_debug { + struct ch_ktls_port_stats_debug ktls_port[MAX_ULD_NPORTS]; + atomic64_t ktls_tx_send_records; + atomic64_t ktls_tx_end_pkts; + atomic64_t ktls_tx_start_pkts; + atomic64_t ktls_tx_middle_pkts; + atomic64_t ktls_tx_retransmit_pkts; + atomic64_t ktls_tx_complete_pkts; + atomic64_t ktls_tx_trimmed_pkts; +}; #endif struct chcr_stats_debug { diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index 2b942b4825d4..54e22f17af3d 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -4872,9 +4872,6 @@ void t4_sge_stop(struct adapter *adap) int i; struct sge *s = &adap->sge; - if (in_interrupt()) /* actions below require waiting */ - return; - if (s->rx_timer.function) del_timer_sync(&s->rx_timer); if (s->tx_timer.function) diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c index 4609f1f78426..5195f692f14d 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c @@ -125,60 +125,6 @@ out: return ret; } -static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info, - int new_state) -{ - /* This function can be called from both rx (interrupt context) and tx - * queue contexts. - */ - spin_lock_bh(&tx_info->lock); - switch (tx_info->connection_state) { - case KTLS_CONN_CLOSED: - tx_info->connection_state = new_state; - break; - - case KTLS_CONN_ACT_OPEN_REQ: - /* only go forward if state is greater than current state. */ - if (new_state <= tx_info->connection_state) - break; - /* update to the next state and also initialize TCB */ - tx_info->connection_state = new_state; - fallthrough; - case KTLS_CONN_ACT_OPEN_RPL: - /* if we are stuck in this state, means tcb init might not - * received by HW, try sending it again. - */ - if (!chcr_init_tcb_fields(tx_info)) - tx_info->connection_state = KTLS_CONN_SET_TCB_REQ; - break; - - case KTLS_CONN_SET_TCB_REQ: - /* only go forward if state is greater than current state. */ - if (new_state <= tx_info->connection_state) - break; - /* update to the next state and check if l2t_state is valid */ - tx_info->connection_state = new_state; - fallthrough; - case KTLS_CONN_SET_TCB_RPL: - /* Check if l2t state is valid, then move to ready state. */ - if (cxgb4_check_l2t_valid(tx_info->l2te)) { - tx_info->connection_state = KTLS_CONN_TX_READY; - atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_ctx); - } - break; - - case KTLS_CONN_TX_READY: - /* nothing to be done here */ - break; - - default: - pr_err("unknown KTLS connection state\n"); - break; - } - spin_unlock_bh(&tx_info->lock); - - return tx_info->connection_state; -} /* * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection. * @sk - tcp socket. @@ -298,27 +244,17 @@ static int chcr_setup_connection(struct sock *sk, return -EINVAL; tx_info->atid = atid; - tx_info->ip_family = sk->sk_family; - if (sk->sk_family == AF_INET) { - tx_info->ip_family = AF_INET; + if (tx_info->ip_family == AF_INET) { ret = chcr_ktls_act_open_req(sk, tx_info, atid); #if IS_ENABLED(CONFIG_IPV6) } else { - if (!sk->sk_ipv6only && - ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { - tx_info->ip_family = AF_INET; - ret = chcr_ktls_act_open_req(sk, tx_info, atid); - } else { - tx_info->ip_family = AF_INET6; - ret = cxgb4_clip_get(tx_info->netdev, - (const u32 *) - &sk->sk_v6_rcv_saddr.s6_addr, - 1); - if (ret) - goto out; - ret = chcr_ktls_act_open_req6(sk, tx_info, atid); - } + ret = cxgb4_clip_get(tx_info->netdev, (const u32 *) + &sk->sk_v6_rcv_saddr, + 1); + if (ret) + return ret; + ret = chcr_ktls_act_open_req6(sk, tx_info, atid); #endif } @@ -326,16 +262,21 @@ static int chcr_setup_connection(struct sock *sk, * success, if any other return type clear atid and return that failure. */ if (ret) { - if (ret == NET_XMIT_CN) + if (ret == NET_XMIT_CN) { ret = 0; - else + } else { +#if IS_ENABLED(CONFIG_IPV6) + /* clear clip entry */ + if (tx_info->ip_family == AF_INET6) + cxgb4_clip_release(tx_info->netdev, + (const u32 *) + &sk->sk_v6_rcv_saddr, + 1); +#endif cxgb4_free_atid(t, atid); - goto out; + } } - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ); -out: return ret; } @@ -396,15 +337,10 @@ static void chcr_ktls_dev_del(struct net_device *netdev, struct chcr_ktls_ofld_ctx_tx *tx_ctx = chcr_get_ktls_tx_context(tls_ctx); struct chcr_ktls_info *tx_info = tx_ctx->chcr_info; - struct sock *sk; + struct ch_ktls_port_stats_debug *port_stats; if (!tx_info) return; - sk = tx_info->sk; - - spin_lock(&tx_info->lock); - tx_info->connection_state = KTLS_CONN_CLOSED; - spin_unlock(&tx_info->lock); /* clear l2t entry */ if (tx_info->l2te) @@ -413,8 +349,8 @@ static void chcr_ktls_dev_del(struct net_device *netdev, #if IS_ENABLED(CONFIG_IPV6) /* clear clip entry */ if (tx_info->ip_family == AF_INET6) - cxgb4_clip_release(netdev, - (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8, + cxgb4_clip_release(netdev, (const u32 *) + &tx_info->sk->sk_v6_rcv_saddr, 1); #endif @@ -426,7 +362,8 @@ static void chcr_ktls_dev_del(struct net_device *netdev, tx_info->tid, tx_info->ip_family); } - atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_connection_close); + port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; + atomic64_inc(&port_stats->ktls_tx_connection_close); kvfree(tx_info); tx_ctx->chcr_info = NULL; /* release module refcount */ @@ -448,6 +385,7 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, u32 start_offload_tcp_sn) { struct tls_context *tls_ctx = tls_get_ctx(sk); + struct ch_ktls_port_stats_debug *port_stats; struct chcr_ktls_ofld_ctx_tx *tx_ctx; struct chcr_ktls_info *tx_info; struct dst_entry *dst; @@ -461,30 +399,23 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, pi = netdev_priv(netdev); adap = pi->adapter; + port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id]; + atomic64_inc(&port_stats->ktls_tx_connection_open); + if (direction == TLS_OFFLOAD_CTX_DIR_RX) { pr_err("not expecting for RX direction\n"); - ret = -EINVAL; goto out; } - if (tx_ctx->chcr_info) { - ret = -EINVAL; + + if (tx_ctx->chcr_info) goto out; - } tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL); - if (!tx_info) { - ret = -ENOMEM; + if (!tx_info) goto out; - } - - spin_lock_init(&tx_info->lock); - - /* clear connection state */ - spin_lock(&tx_info->lock); - tx_info->connection_state = KTLS_CONN_CLOSED; - spin_unlock(&tx_info->lock); tx_info->sk = sk; + spin_lock_init(&tx_info->lock); /* initialize tid and atid to -1, 0 is a also a valid id. */ tx_info->tid = -1; tx_info->atid = -1; @@ -495,10 +426,12 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, tx_info->tx_chan = pi->tx_chan; tx_info->smt_idx = pi->smt_idx; tx_info->port_id = pi->port_id; + tx_info->prev_ack = 0; + tx_info->prev_win = 0; tx_info->rx_qid = chcr_get_first_rx_qid(adap); if (unlikely(tx_info->rx_qid < 0)) - goto out2; + goto free_tx_info; tx_info->prev_seq = start_offload_tcp_sn; tx_info->tcp_start_seq_number = start_offload_tcp_sn; @@ -506,18 +439,22 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, /* save crypto keys */ ret = chcr_ktls_save_keys(tx_info, crypto_info, direction); if (ret < 0) - goto out2; + goto free_tx_info; /* get peer ip */ if (sk->sk_family == AF_INET) { memcpy(daaddr, &sk->sk_daddr, 4); + tx_info->ip_family = AF_INET; #if IS_ENABLED(CONFIG_IPV6) } else { if (!sk->sk_ipv6only && - ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) + ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { memcpy(daaddr, &sk->sk_daddr, 4); - else + tx_info->ip_family = AF_INET; + } else { memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16); + tx_info->ip_family = AF_INET6; + } #endif } @@ -525,13 +462,13 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, dst = sk_dst_get(sk); if (!dst) { pr_err("DST entry not found\n"); - goto out2; + goto free_tx_info; } n = dst_neigh_lookup(dst, daaddr); if (!n || !n->dev) { pr_err("neighbour not found\n"); dst_release(dst); - goto out2; + goto free_tx_info; } tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0); @@ -540,31 +477,86 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, if (!tx_info->l2te) { pr_err("l2t entry not found\n"); - goto out2; + goto free_tx_info; } - tx_ctx->chcr_info = tx_info; + /* Driver shouldn't be removed until any single connection exists */ + if (!try_module_get(THIS_MODULE)) + goto free_l2t; + init_completion(&tx_info->completion); /* create a filter and call cxgb4_l2t_send to send the packet out, which * will take care of updating l2t entry in hw if not already done. */ - ret = chcr_setup_connection(sk, tx_info); - if (ret) - goto out2; + tx_info->open_state = CH_KTLS_OPEN_PENDING; - /* Driver shouldn't be removed until any single connection exists */ - if (!try_module_get(THIS_MODULE)) { - ret = -EINVAL; - goto out2; + if (chcr_setup_connection(sk, tx_info)) + goto put_module; + + /* Wait for reply */ + wait_for_completion_timeout(&tx_info->completion, 30 * HZ); + spin_lock_bh(&tx_info->lock); + if (tx_info->open_state) { + /* need to wait for hw response, can't free tx_info yet. */ + if (tx_info->open_state == CH_KTLS_OPEN_PENDING) + tx_info->pending_close = true; + /* free the lock after the cleanup */ + goto put_module; + } + spin_unlock_bh(&tx_info->lock); + + /* initialize tcb */ + reinit_completion(&tx_info->completion); + /* mark it pending for hw response */ + tx_info->open_state = CH_KTLS_OPEN_PENDING; + + if (chcr_init_tcb_fields(tx_info)) + goto free_tid; + + /* Wait for reply */ + wait_for_completion_timeout(&tx_info->completion, 30 * HZ); + spin_lock_bh(&tx_info->lock); + if (tx_info->open_state) { + /* need to wait for hw response, can't free tx_info yet. */ + tx_info->pending_close = true; + /* free the lock after cleanup */ + goto free_tid; } + spin_unlock_bh(&tx_info->lock); + + if (!cxgb4_check_l2t_valid(tx_info->l2te)) + goto free_tid; + + atomic64_inc(&port_stats->ktls_tx_ctx); + tx_ctx->chcr_info = tx_info; - atomic64_inc(&adap->ch_ktls_stats.ktls_tx_connection_open); return 0; -out2: - kvfree(tx_info); + +free_tid: + chcr_ktls_mark_tcb_close(tx_info); +#if IS_ENABLED(CONFIG_IPV6) + /* clear clip entry */ + if (tx_info->ip_family == AF_INET6) + cxgb4_clip_release(netdev, (const u32 *) + &sk->sk_v6_rcv_saddr, + 1); +#endif + cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, + tx_info->tid, tx_info->ip_family); + +put_module: + /* release module refcount */ + module_put(THIS_MODULE); +free_l2t: + cxgb4_l2t_release(tx_info->l2te); +free_tx_info: + if (tx_info->pending_close) + spin_unlock_bh(&tx_info->lock); + else + kvfree(tx_info); out: - atomic64_inc(&adap->ch_ktls_stats.ktls_tx_connection_fail); - return ret; + atomic64_inc(&port_stats->ktls_tx_connection_fail); + return -1; } /* @@ -627,20 +619,39 @@ static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, tx_info = lookup_atid(t, atid); if (!tx_info || tx_info->atid != atid) { - pr_err("tx_info or atid is not correct\n"); + pr_err("%s: incorrect tx_info or atid\n", __func__); return -1; } + cxgb4_free_atid(t, atid); + tx_info->atid = -1; + + spin_lock(&tx_info->lock); + /* HW response is very close, finish pending cleanup */ + if (tx_info->pending_close) { + spin_unlock(&tx_info->lock); + if (!status) { + /* it's a late success, tcb status is establised, + * mark it close. + */ + chcr_ktls_mark_tcb_close(tx_info); + cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, + tid, tx_info->ip_family); + } + kvfree(tx_info); + return 0; + } + if (!status) { tx_info->tid = tid; cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family); - - cxgb4_free_atid(t, atid); - tx_info->atid = -1; - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, - KTLS_CONN_ACT_OPEN_RPL); + tx_info->open_state = CH_KTLS_OPEN_SUCCESS; + } else { + tx_info->open_state = CH_KTLS_OPEN_FAILURE; } + spin_unlock(&tx_info->lock); + + complete(&tx_info->completion); return 0; } @@ -658,12 +669,22 @@ static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input) t = &adap->tids; tx_info = lookup_tid(t, tid); + if (!tx_info || tx_info->tid != tid) { - pr_err("tx_info or atid is not correct\n"); + pr_err("%s: incorrect tx_info or tid\n", __func__); return -1; } - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL); + + spin_lock(&tx_info->lock); + if (tx_info->pending_close) { + spin_unlock(&tx_info->lock); + kvfree(tx_info); + return 0; + } + tx_info->open_state = false; + spin_unlock(&tx_info->lock); + + complete(&tx_info->completion); return 0; } @@ -765,6 +786,7 @@ static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info, u64 tcp_ack, u64 tcp_win) { bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0)); + struct ch_ktls_port_stats_debug *port_stats; u32 len, cpl = 0, ndesc, wr_len; struct fw_ulptx_wr *wr; int credits; @@ -798,12 +820,14 @@ static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info, /* reset snd una if it's a re-transmit pkt */ if (tcp_seq != tx_info->prev_seq) { /* reset snd_una */ + port_stats = + &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, TCB_SND_UNA_RAW_W, TCB_SND_UNA_RAW_V (TCB_SND_UNA_RAW_M), TCB_SND_UNA_RAW_V(0), 0); - atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_ooo); + atomic64_inc(&port_stats->ktls_tx_ooo); cpl++; } /* update ack */ @@ -1836,6 +1860,7 @@ out: /* nic tls TX handler */ static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) { + struct ch_ktls_port_stats_debug *port_stats; struct chcr_ktls_ofld_ctx_tx *tx_ctx; struct ch_ktls_stats_debug *stats; struct tcphdr *th = tcp_hdr(skb); @@ -1845,7 +1870,6 @@ static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) u32 tls_end_offset, tcp_seq; struct tls_context *tls_ctx; struct sk_buff *local_skb; - int new_connection_state; struct sge_eth_txq *q; struct adapter *adap; unsigned long flags; @@ -1868,15 +1892,6 @@ static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) if (unlikely(!tx_info)) goto out; - /* check the connection state, we don't need to pass new connection - * state, state machine will check and update the new state if it is - * stuck due to responses not received from HW. - * Start the tx handling only if state is KTLS_CONN_TX_READY. - */ - new_connection_state = chcr_ktls_update_connection_state(tx_info, 0); - if (new_connection_state != KTLS_CONN_TX_READY) - goto out; - /* don't touch the original skb, make a new skb to extract each records * and send them separately. */ @@ -1887,6 +1902,7 @@ static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) adap = tx_info->adap; stats = &adap->ch_ktls_stats; + port_stats = &stats->ktls_port[tx_info->port_id]; qidx = skb->queue_mapping; q = &adap->sge.ethtxq[qidx + tx_info->first_qset]; @@ -1932,13 +1948,13 @@ static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) */ if (unlikely(!record)) { spin_unlock_irqrestore(&tx_ctx->base.lock, flags); - atomic64_inc(&stats->ktls_tx_drop_no_sync_data); + atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data); goto out; } if (unlikely(tls_record_is_start_marker(record))) { spin_unlock_irqrestore(&tx_ctx->base.lock, flags); - atomic64_inc(&stats->ktls_tx_skip_no_sync_data); + atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data); goto out; } @@ -2009,9 +2025,8 @@ clear_ref: } while (data_len > 0); tx_info->prev_seq = ntohl(th->seq) + skb->data_len; - - atomic64_inc(&stats->ktls_tx_encrypted_packets); - atomic64_add(skb->data_len, &stats->ktls_tx_encrypted_bytes); + atomic64_inc(&port_stats->ktls_tx_encrypted_packets); + atomic64_add(skb->data_len, &port_stats->ktls_tx_encrypted_bytes); /* tcp finish is set, send a separate tcp msg including all the options * as well. diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h index 4ae6ae38c406..c1651b1431a0 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h @@ -27,22 +27,20 @@ #define CHCR_KTLS_WR_SIZE (CHCR_PLAIN_TX_DATA_LEN +\ sizeof(struct cpl_tx_sec_pdu)) -enum chcr_ktls_conn_state { - KTLS_CONN_CLOSED, - KTLS_CONN_ACT_OPEN_REQ, - KTLS_CONN_ACT_OPEN_RPL, - KTLS_CONN_SET_TCB_REQ, - KTLS_CONN_SET_TCB_RPL, - KTLS_CONN_TX_READY, +enum ch_ktls_open_state { + CH_KTLS_OPEN_SUCCESS = 0, + CH_KTLS_OPEN_PENDING = 1, + CH_KTLS_OPEN_FAILURE = 2, }; struct chcr_ktls_info { struct sock *sk; - spinlock_t lock; /* state machine lock */ + spinlock_t lock; /* lock for pending_close */ struct ktls_key_ctx key_ctx; struct adapter *adap; struct l2t_entry *l2te; struct net_device *netdev; + struct completion completion; u64 iv; u64 record_no; int tid; @@ -58,13 +56,14 @@ struct chcr_ktls_info { u32 tcp_start_seq_number; u32 scmd0_short_seqno_numivs; u32 scmd0_short_ivgen_hdrlen; - enum chcr_ktls_conn_state connection_state; u16 prev_win; u8 tx_chan; u8 smt_idx; u8 port_id; u8 ip_family; u8 first_qset; + enum ch_ktls_open_state open_state; + bool pending_close; }; struct chcr_ktls_ofld_ctx_tx { diff --git a/drivers/net/ethernet/cisco/enic/enic.h b/drivers/net/ethernet/cisco/enic/enic.h index 18f3aeb88f22..c67a16a48d62 100644 --- a/drivers/net/ethernet/cisco/enic/enic.h +++ b/drivers/net/ethernet/cisco/enic/enic.h @@ -169,6 +169,7 @@ struct enic { u16 num_vfs; #endif spinlock_t enic_api_lock; + bool enic_api_busy; struct enic_port_profile *pp; /* work queue cache line section */ diff --git a/drivers/net/ethernet/cisco/enic/enic_api.c b/drivers/net/ethernet/cisco/enic/enic_api.c index 89915681473c..3bdc74fba1e3 100644 --- a/drivers/net/ethernet/cisco/enic/enic_api.c +++ b/drivers/net/ethernet/cisco/enic/enic_api.c @@ -34,6 +34,12 @@ int enic_api_devcmd_proxy_by_index(struct net_device *netdev, int vf, struct vnic_dev *vdev = enic->vdev; spin_lock(&enic->enic_api_lock); + while (enic->enic_api_busy) { + spin_unlock(&enic->enic_api_lock); + cpu_relax(); + spin_lock(&enic->enic_api_lock); + } + spin_lock_bh(&enic->devcmd_lock); vnic_dev_cmd_proxy_by_index_start(vdev, vf); diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c index 4f0329d8778f..fb269d587b74 100644 --- a/drivers/net/ethernet/cisco/enic/enic_main.c +++ b/drivers/net/ethernet/cisco/enic/enic_main.c @@ -2107,8 +2107,6 @@ static int enic_dev_wait(struct vnic_dev *vdev, int done; int err; - BUG_ON(in_interrupt()); - err = start(vdev, arg); if (err) return err; @@ -2297,6 +2295,13 @@ static int enic_set_rss_nic_cfg(struct enic *enic) rss_hash_bits, rss_base_cpu, rss_enable); } +static void enic_set_api_busy(struct enic *enic, bool busy) +{ + spin_lock(&enic->enic_api_lock); + enic->enic_api_busy = busy; + spin_unlock(&enic->enic_api_lock); +} + static void enic_reset(struct work_struct *work) { struct enic *enic = container_of(work, struct enic, reset); @@ -2306,7 +2311,9 @@ static void enic_reset(struct work_struct *work) rtnl_lock(); - spin_lock(&enic->enic_api_lock); + /* Stop any activity from infiniband */ + enic_set_api_busy(enic, true); + enic_stop(enic->netdev); enic_dev_soft_reset(enic); enic_reset_addr_lists(enic); @@ -2314,7 +2321,10 @@ static void enic_reset(struct work_struct *work) enic_set_rss_nic_cfg(enic); enic_dev_set_ig_vlan_rewrite_mode(enic); enic_open(enic->netdev); - spin_unlock(&enic->enic_api_lock); + + /* Allow infiniband to fiddle with the device again */ + enic_set_api_busy(enic, false); + call_netdevice_notifiers(NETDEV_REBOOT, enic->netdev); rtnl_unlock(); @@ -2326,7 +2336,9 @@ static void enic_tx_hang_reset(struct work_struct *work) rtnl_lock(); - spin_lock(&enic->enic_api_lock); + /* Stop any activity from infiniband */ + enic_set_api_busy(enic, true); + enic_dev_hang_notify(enic); enic_stop(enic->netdev); enic_dev_hang_reset(enic); @@ -2335,7 +2347,10 @@ static void enic_tx_hang_reset(struct work_struct *work) enic_set_rss_nic_cfg(enic); enic_dev_set_ig_vlan_rewrite_mode(enic); enic_open(enic->netdev); - spin_unlock(&enic->enic_api_lock); + + /* Allow infiniband to fiddle with the device again */ + enic_set_api_busy(enic, false); + call_netdevice_notifiers(NETDEV_REBOOT, enic->netdev); rtnl_unlock(); diff --git a/drivers/net/ethernet/freescale/fec_mpc52xx.c b/drivers/net/ethernet/freescale/fec_mpc52xx.c index 7a3f066e611d..b3bad429e03b 100644 --- a/drivers/net/ethernet/freescale/fec_mpc52xx.c +++ b/drivers/net/ethernet/freescale/fec_mpc52xx.c @@ -74,7 +74,7 @@ struct mpc52xx_fec_priv { static irqreturn_t mpc52xx_fec_interrupt(int, void *); static irqreturn_t mpc52xx_fec_rx_interrupt(int, void *); static irqreturn_t mpc52xx_fec_tx_interrupt(int, void *); -static void mpc52xx_fec_stop(struct net_device *dev); +static void mpc52xx_fec_stop(struct net_device *dev, bool may_sleep); static void mpc52xx_fec_start(struct net_device *dev); static void mpc52xx_fec_reset(struct net_device *dev); @@ -283,7 +283,7 @@ static int mpc52xx_fec_close(struct net_device *dev) netif_stop_queue(dev); - mpc52xx_fec_stop(dev); + mpc52xx_fec_stop(dev, true); mpc52xx_fec_free_rx_buffers(dev, priv->rx_dmatsk); @@ -693,7 +693,7 @@ static void mpc52xx_fec_start(struct net_device *dev) * * stop all activity on fec and empty dma buffers */ -static void mpc52xx_fec_stop(struct net_device *dev) +static void mpc52xx_fec_stop(struct net_device *dev, bool may_sleep) { struct mpc52xx_fec_priv *priv = netdev_priv(dev); struct mpc52xx_fec __iomem *fec = priv->fec; @@ -706,7 +706,7 @@ static void mpc52xx_fec_stop(struct net_device *dev) bcom_disable(priv->rx_dmatsk); /* Wait for tx queue to drain, but only if we're in process context */ - if (!in_interrupt()) { + if (may_sleep) { timeout = jiffies + msecs_to_jiffies(2000); while (time_before(jiffies, timeout) && !bcom_queue_empty(priv->tx_dmatsk)) @@ -738,7 +738,7 @@ static void mpc52xx_fec_reset(struct net_device *dev) struct mpc52xx_fec_priv *priv = netdev_priv(dev); struct mpc52xx_fec __iomem *fec = priv->fec; - mpc52xx_fec_stop(dev); + mpc52xx_fec_stop(dev, false); out_be32(&fec->rfifo_status, in_be32(&fec->rfifo_status)); out_be32(&fec->reset_cntrl, FEC_RESET_CNTRL_RESET_FIFO); diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index f6d0702a0df7..912c51e327d6 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -130,6 +130,9 @@ enum HNAE3_DEV_CAP_BITS { #define hnae3_dev_stash_supported(hdev) \ test_bit(HNAE3_DEV_SUPPORT_STASH_B, (hdev)->ae_dev->caps) +#define hnae3_ae_dev_tqp_txrx_indep_supported(ae_dev) \ + test_bit(HNAE3_DEV_SUPPORT_TQP_TXRX_INDEP_B, (ae_dev)->caps) + #define ring_ptr_move_fw(ring, p) \ ((ring)->p = ((ring)->p + 1) % (ring)->desc_num) #define ring_ptr_move_bw(ring, p) \ diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c index 15333ec5f035..dc9a85745e62 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c @@ -15,6 +15,7 @@ static struct dentry *hns3_dbgfs_root; static int hns3_dbg_queue_info(struct hnae3_handle *h, const char *cmd_buf) { + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); struct hns3_nic_priv *priv = h->priv; struct hns3_enet_ring *ring; u32 base_add_l, base_add_h; @@ -118,8 +119,25 @@ static int hns3_dbg_queue_info(struct hnae3_handle *h, value = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_PKTNUM_RECORD_REG); - dev_info(&h->pdev->dev, "TX(%u) RING PKTNUM: %u\n\n", i, - value); + dev_info(&h->pdev->dev, "TX(%u) RING PKTNUM: %u\n", i, value); + + value = readl_relaxed(ring->tqp->io_base + HNS3_RING_EN_REG); + dev_info(&h->pdev->dev, "TX/RX(%u) RING EN: %s\n", i, + value ? "enable" : "disable"); + + if (hnae3_ae_dev_tqp_txrx_indep_supported(ae_dev)) { + value = readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_EN_REG); + dev_info(&h->pdev->dev, "TX(%u) RING EN: %s\n", i, + value ? "enable" : "disable"); + + value = readl_relaxed(ring->tqp->io_base + + HNS3_RING_RX_EN_REG); + dev_info(&h->pdev->dev, "RX(%u) RING EN: %s\n", i, + value ? "enable" : "disable"); + } + + dev_info(&h->pdev->dev, "\n"); } return 0; @@ -245,6 +263,7 @@ static void hns3_dbg_help(struct hnae3_handle *h) dev_info(&h->pdev->dev, "queue map\n"); dev_info(&h->pdev->dev, "bd info <q_num> <bd index>\n"); dev_info(&h->pdev->dev, "dev capability\n"); + dev_info(&h->pdev->dev, "dev spec\n"); if (!hns3_is_phys_func(h->pdev)) return; @@ -307,6 +326,31 @@ static void hns3_dbg_dev_caps(struct hnae3_handle *h) test_bit(HNAE3_DEV_SUPPORT_INT_QL_B, caps) ? "yes" : "no"); } +static void hns3_dbg_dev_specs(struct hnae3_handle *h) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); + struct hnae3_dev_specs *dev_specs = &ae_dev->dev_specs; + struct hnae3_knic_private_info *kinfo = &h->kinfo; + struct hns3_nic_priv *priv = h->priv; + + dev_info(priv->dev, "MAC entry num: %u\n", dev_specs->mac_entry_num); + dev_info(priv->dev, "MNG entry num: %u\n", dev_specs->mng_entry_num); + dev_info(priv->dev, "MAX non tso bd num: %u\n", + dev_specs->max_non_tso_bd_num); + dev_info(priv->dev, "RSS ind tbl size: %u\n", + dev_specs->rss_ind_tbl_size); + dev_info(priv->dev, "RSS key size: %u\n", dev_specs->rss_key_size); + dev_info(priv->dev, "RSS size: %u\n", kinfo->rss_size); + dev_info(priv->dev, "Allocated RSS size: %u\n", kinfo->req_rss_size); + dev_info(priv->dev, "Task queue pairs numbers: %u\n", kinfo->num_tqps); + + dev_info(priv->dev, "RX buffer length: %u\n", kinfo->rx_buf_len); + dev_info(priv->dev, "Desc num per TX queue: %u\n", kinfo->num_tx_desc); + dev_info(priv->dev, "Desc num per RX queue: %u\n", kinfo->num_rx_desc); + dev_info(priv->dev, "Total number of enabled TCs: %u\n", kinfo->num_tc); + dev_info(priv->dev, "MAX INT QL: %u\n", dev_specs->int_ql_max); +} + static ssize_t hns3_dbg_cmd_read(struct file *filp, char __user *buffer, size_t count, loff_t *ppos) { @@ -384,6 +428,8 @@ static ssize_t hns3_dbg_cmd_write(struct file *filp, const char __user *buffer, ret = hns3_dbg_bd_info(handle, cmd_buf); else if (strncmp(cmd_buf, "dev capability", 14) == 0) hns3_dbg_dev_caps(handle); + else if (strncmp(cmd_buf, "dev spec", 8) == 0) + hns3_dbg_dev_specs(handle); else if (handle->ae_algo->ops->dbg_run_cmd) ret = handle->ae_algo->ops->dbg_run_cmd(handle, cmd_buf); else diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 1c4e820e21b2..a362516a3185 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -696,12 +696,19 @@ static int hns3_set_tso(struct sk_buff *skb, u32 *paylen, /* normal or tunnel packet */ l4_offset = l4.hdr - skb->data; - hdr_len = (l4.tcp->doff << 2) + l4_offset; /* remove payload length from inner pseudo checksum when tso */ l4_paylen = skb->len - l4_offset; - csum_replace_by_diff(&l4.tcp->check, - (__force __wsum)htonl(l4_paylen)); + + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { + hdr_len = sizeof(*l4.udp) + l4_offset; + csum_replace_by_diff(&l4.udp->check, + (__force __wsum)htonl(l4_paylen)); + } else { + hdr_len = (l4.tcp->doff << 2) + l4_offset; + csum_replace_by_diff(&l4.tcp->check, + (__force __wsum)htonl(l4_paylen)); + } /* find the txbd field values */ *paylen = skb->len - hdr_len; @@ -1184,21 +1191,23 @@ static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size, return bd_num; } -static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size) +static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size, + u8 max_non_tso_bd_num) { struct sk_buff *frag_skb; unsigned int bd_num = 0; /* If the total len is within the max bd limit */ if (likely(skb->len <= HNS3_MAX_BD_SIZE && !skb_has_frag_list(skb) && - skb_shinfo(skb)->nr_frags < HNS3_MAX_NON_TSO_BD_NUM)) + skb_shinfo(skb)->nr_frags < max_non_tso_bd_num)) return skb_shinfo(skb)->nr_frags + 1U; /* The below case will always be linearized, return * HNS3_MAX_BD_NUM_TSO + 1U to make sure it is linearized. */ if (unlikely(skb->len > HNS3_MAX_TSO_SIZE || - (!skb_is_gso(skb) && skb->len > HNS3_MAX_NON_TSO_SIZE))) + (!skb_is_gso(skb) && skb->len > + HNS3_MAX_NON_TSO_SIZE(max_non_tso_bd_num)))) return HNS3_MAX_TSO_BD_NUM + 1U; bd_num = hns3_skb_bd_num(skb, bd_size, bd_num); @@ -1223,31 +1232,34 @@ static unsigned int hns3_gso_hdr_len(struct sk_buff *skb) return skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb); } -/* HW need every continuous 8 buffer data to be larger than MSS, - * we simplify it by ensuring skb_headlen + the first continuous - * 7 frags to to be larger than gso header len + mss, and the remaining - * continuous 7 frags to be larger than MSS except the last 7 frags. +/* HW need every continuous max_non_tso_bd_num buffer data to be larger + * than MSS, we simplify it by ensuring skb_headlen + the first continuous + * max_non_tso_bd_num - 1 frags to be larger than gso header len + mss, + * and the remaining continuous max_non_tso_bd_num - 1 frags to be larger + * than MSS except the last max_non_tso_bd_num - 1 frags. */ static bool hns3_skb_need_linearized(struct sk_buff *skb, unsigned int *bd_size, - unsigned int bd_num) + unsigned int bd_num, u8 max_non_tso_bd_num) { unsigned int tot_len = 0; int i; - for (i = 0; i < HNS3_MAX_NON_TSO_BD_NUM - 1U; i++) + for (i = 0; i < max_non_tso_bd_num - 1U; i++) tot_len += bd_size[i]; - /* ensure the first 8 frags is greater than mss + header */ - if (tot_len + bd_size[HNS3_MAX_NON_TSO_BD_NUM - 1U] < + /* ensure the first max_non_tso_bd_num frags is greater than + * mss + header + */ + if (tot_len + bd_size[max_non_tso_bd_num - 1U] < skb_shinfo(skb)->gso_size + hns3_gso_hdr_len(skb)) return true; - /* ensure every continuous 7 buffer is greater than mss - * except the last one. + /* ensure every continuous max_non_tso_bd_num - 1 buffer is greater + * than mss except the last one. */ - for (i = 0; i < bd_num - HNS3_MAX_NON_TSO_BD_NUM; i++) { + for (i = 0; i < bd_num - max_non_tso_bd_num; i++) { tot_len -= bd_size[i]; - tot_len += bd_size[i + HNS3_MAX_NON_TSO_BD_NUM - 1U]; + tot_len += bd_size[i + max_non_tso_bd_num - 1U]; if (tot_len < skb_shinfo(skb)->gso_size) return true; @@ -1269,14 +1281,16 @@ static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, struct sk_buff *skb) { struct hns3_nic_priv *priv = netdev_priv(netdev); + u8 max_non_tso_bd_num = priv->max_non_tso_bd_num; unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U]; unsigned int bd_num; - bd_num = hns3_tx_bd_num(skb, bd_size); - if (unlikely(bd_num > HNS3_MAX_NON_TSO_BD_NUM)) { + bd_num = hns3_tx_bd_num(skb, bd_size, max_non_tso_bd_num); + if (unlikely(bd_num > max_non_tso_bd_num)) { if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) && - !hns3_skb_need_linearized(skb, bd_size, bd_num)) { - trace_hns3_over_8bd(skb); + !hns3_skb_need_linearized(skb, bd_size, bd_num, + max_non_tso_bd_num)) { + trace_hns3_over_max_bd(skb); goto out; } @@ -1286,8 +1300,8 @@ static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, bd_num = hns3_tx_bd_count(skb->len); if ((skb_is_gso(skb) && bd_num > HNS3_MAX_TSO_BD_NUM) || (!skb_is_gso(skb) && - bd_num > HNS3_MAX_NON_TSO_BD_NUM)) { - trace_hns3_over_8bd(skb); + bd_num > max_non_tso_bd_num)) { + trace_hns3_over_max_bd(skb); return -ENOMEM; } @@ -2304,6 +2318,13 @@ static void hns3_set_default_feature(struct net_device *netdev) netdev->features |= NETIF_F_NTUPLE; } } + + if (test_bit(HNAE3_DEV_SUPPORT_UDP_GSO_B, ae_dev->caps)) { + netdev->hw_features |= NETIF_F_GSO_UDP_L4; + netdev->features |= NETIF_F_GSO_UDP_L4; + netdev->vlan_features |= NETIF_F_GSO_UDP_L4; + netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4; + } } static int hns3_alloc_buffer(struct hns3_enet_ring *ring, @@ -3991,6 +4012,7 @@ static void hns3_info_show(struct hns3_nic_priv *priv) static int hns3_client_init(struct hnae3_handle *handle) { struct pci_dev *pdev = handle->pdev; + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); u16 alloc_tqps, max_rss_size; struct hns3_nic_priv *priv; struct net_device *netdev; @@ -4007,6 +4029,7 @@ static int hns3_client_init(struct hnae3_handle *handle) priv->netdev = netdev; priv->ae_handle = handle; priv->tx_timeout_count = 0; + priv->max_non_tso_bd_num = ae_dev->dev_specs.max_non_tso_bd_num; set_bit(HNS3_NIC_STATE_DOWN, &priv->state); handle->msg_enable = netif_msg_init(debug, DEFAULT_MSG_LEVEL); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index 8a6c8bae8fa6..1c81dea0da1e 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -43,6 +43,8 @@ enum hns3_nic_state { #define HNS3_RING_TX_RING_EBD_OFFSET_REG 0x00070 #define HNS3_RING_TX_RING_BD_ERR_REG 0x00074 #define HNS3_RING_EN_REG 0x00090 +#define HNS3_RING_RX_EN_REG 0x00098 +#define HNS3_RING_TX_EN_REG 0x000D4 #define HNS3_RX_HEAD_SIZE 256 @@ -167,13 +169,12 @@ enum hns3_nic_state { #define HNS3_VECTOR_INITED 1 #define HNS3_MAX_BD_SIZE 65535 -#define HNS3_MAX_NON_TSO_BD_NUM 8U #define HNS3_MAX_TSO_BD_NUM 63U #define HNS3_MAX_TSO_SIZE \ (HNS3_MAX_BD_SIZE * HNS3_MAX_TSO_BD_NUM) -#define HNS3_MAX_NON_TSO_SIZE \ - (HNS3_MAX_BD_SIZE * HNS3_MAX_NON_TSO_BD_NUM) +#define HNS3_MAX_NON_TSO_SIZE(max_non_tso_bd_num) \ + (HNS3_MAX_BD_SIZE * (max_non_tso_bd_num)) #define HNS3_VECTOR_GL0_OFFSET 0x100 #define HNS3_VECTOR_GL1_OFFSET 0x200 @@ -475,6 +476,7 @@ struct hns3_nic_priv { struct hns3_enet_ring *ring; struct hns3_enet_tqp_vector *tqp_vector; u16 vector_num; + u8 max_non_tso_bd_num; u64 tx_timeout_count; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index bf3504d6d70a..6b07b2771172 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -311,9 +311,6 @@ static void hns3_self_test(struct net_device *ndev, struct hnae3_handle *h = priv->ae_handle; int st_param[HNS3_SELF_TEST_TYPE_NUM][2]; bool if_running = netif_running(ndev); -#if IS_ENABLED(CONFIG_VLAN_8021Q) - bool dis_vlan_filter; -#endif int test_index = 0; u32 i; @@ -350,9 +347,7 @@ static void hns3_self_test(struct net_device *ndev, #if IS_ENABLED(CONFIG_VLAN_8021Q) /* Disable the vlan filter for selftest does not support it */ - dis_vlan_filter = (ndev->features & NETIF_F_HW_VLAN_CTAG_FILTER) && - h->ae_algo->ops->enable_vlan_filter; - if (dis_vlan_filter) + if (h->ae_algo->ops->enable_vlan_filter) h->ae_algo->ops->enable_vlan_filter(h, false); #endif @@ -389,7 +384,7 @@ static void hns3_self_test(struct net_device *ndev, h->ae_algo->ops->halt_autoneg(h, false); #if IS_ENABLED(CONFIG_VLAN_8021Q) - if (dis_vlan_filter) + if (h->ae_algo->ops->enable_vlan_filter) h->ae_algo->ops->enable_vlan_filter(h, true); #endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_trace.h b/drivers/net/ethernet/hisilicon/hns3/hns3_trace.h index 7bddcca148a5..5153e5d41bbd 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_trace.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_trace.h @@ -53,7 +53,7 @@ DECLARE_EVENT_CLASS(hns3_skb_template, ) ); -DEFINE_EVENT(hns3_skb_template, hns3_over_8bd, +DEFINE_EVENT(hns3_skb_template, hns3_over_max_bd, TP_PROTO(struct sk_buff *skb), TP_ARGS(skb)); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 8c8e666b6152..50c84c5e65d2 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -1702,6 +1702,26 @@ static int hclgevf_notify_client(struct hclgevf_dev *hdev, return ret; } +static int hclgevf_notify_roce_client(struct hclgevf_dev *hdev, + enum hnae3_reset_notify_type type) +{ + struct hnae3_client *client = hdev->roce_client; + struct hnae3_handle *handle = &hdev->roce; + int ret; + + if (!test_bit(HCLGEVF_STATE_ROCE_REGISTERED, &hdev->state) || !client) + return 0; + + if (!client->ops->reset_notify) + return -EOPNOTSUPP; + + ret = client->ops->reset_notify(handle, type); + if (ret) + dev_err(&hdev->pdev->dev, "notify roce client failed %d(%d)", + type, ret); + return ret; +} + static int hclgevf_reset_wait(struct hclgevf_dev *hdev) { #define HCLGEVF_RESET_WAIT_US 20000 @@ -1865,6 +1885,11 @@ static int hclgevf_reset_prepare(struct hclgevf_dev *hdev) hdev->rst_stats.rst_cnt++; + /* perform reset of the stack & ae device for a client */ + ret = hclgevf_notify_roce_client(hdev, HNAE3_DOWN_CLIENT); + if (ret) + return ret; + rtnl_lock(); /* bring down the nic to stop any ongoing TX/RX */ ret = hclgevf_notify_client(hdev, HNAE3_DOWN_CLIENT); @@ -1880,6 +1905,9 @@ static int hclgevf_reset_rebuild(struct hclgevf_dev *hdev) int ret; hdev->rst_stats.hw_rst_done_cnt++; + ret = hclgevf_notify_roce_client(hdev, HNAE3_UNINIT_CLIENT); + if (ret) + return ret; rtnl_lock(); /* now, re-initialize the nic client and ae device */ @@ -1890,6 +1918,18 @@ static int hclgevf_reset_rebuild(struct hclgevf_dev *hdev) return ret; } + ret = hclgevf_notify_roce_client(hdev, HNAE3_INIT_CLIENT); + /* ignore RoCE notify error if it fails HCLGEVF_RESET_MAX_FAIL_CNT - 1 + * times + */ + if (ret && + hdev->rst_stats.rst_fail_cnt < HCLGEVF_RESET_MAX_FAIL_CNT - 1) + return ret; + + ret = hclgevf_notify_roce_client(hdev, HNAE3_UP_CLIENT); + if (ret) + return ret; + hdev->last_reset_time = jiffies; hdev->rst_stats.rst_done_cnt++; hdev->rst_stats.rst_fail_cnt = 0; @@ -2757,6 +2797,7 @@ static int hclgevf_init_roce_client_instance(struct hnae3_ae_dev *ae_dev, if (ret) return ret; + set_bit(HCLGEVF_STATE_ROCE_REGISTERED, &hdev->state); hnae3_set_client_init_flag(client, ae_dev, 1); return 0; @@ -2817,6 +2858,7 @@ static void hclgevf_uninit_client_instance(struct hnae3_client *client, /* un-init roce, if it exists */ if (hdev->roce_client) { + clear_bit(HCLGEVF_STATE_ROCE_REGISTERED, &hdev->state); hdev->roce_client->ops->uninit_instance(&hdev->roce, 0); hdev->roce_client = NULL; hdev->roce.client = NULL; @@ -3419,6 +3461,13 @@ static bool hclgevf_get_hw_reset_stat(struct hnae3_handle *handle) return !!hclgevf_read_dev(&hdev->hw, HCLGEVF_RST_ING); } +static bool hclgevf_get_cmdq_stat(struct hnae3_handle *handle) +{ + struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); + + return test_bit(HCLGEVF_STATE_CMD_DISABLE, &hdev->state); +} + static bool hclgevf_ae_dev_resetting(struct hnae3_handle *handle) { struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); @@ -3604,6 +3653,7 @@ static const struct hnae3_ae_ops hclgevf_ops = { .get_link_mode = hclgevf_get_link_mode, .set_promisc_mode = hclgevf_set_promisc_mode, .request_update_promisc_mode = hclgevf_request_update_promisc_mode, + .get_cmdq_stat = hclgevf_get_cmdq_stat, }; static struct hnae3_ae_algo ae_algovf = { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h index c1fac8920ae3..c5bcc3894fd5 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h @@ -139,6 +139,7 @@ enum hclgevf_states { HCLGEVF_STATE_IRQ_INITED, HCLGEVF_STATE_REMOVING, HCLGEVF_STATE_NIC_REGISTERED, + HCLGEVF_STATE_ROCE_REGISTERED, /* task states */ HCLGEVF_STATE_RST_SERVICE_SCHED, HCLGEVF_STATE_RST_HANDLING, diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index a151ff37fda2..1f7fe6b3dd5a 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -97,11 +97,11 @@ static int pending_scrq(struct ibmvnic_adapter *, static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *, struct ibmvnic_sub_crq_queue *); static int ibmvnic_poll(struct napi_struct *napi, int data); -static void send_map_query(struct ibmvnic_adapter *adapter); +static void send_query_map(struct ibmvnic_adapter *adapter); static int send_request_map(struct ibmvnic_adapter *, dma_addr_t, __be32, u8); static int send_request_unmap(struct ibmvnic_adapter *, u8); static int send_login(struct ibmvnic_adapter *adapter); -static void send_cap_queries(struct ibmvnic_adapter *adapter); +static void send_query_cap(struct ibmvnic_adapter *adapter); static int init_sub_crqs(struct ibmvnic_adapter *); static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter); static int ibmvnic_reset_init(struct ibmvnic_adapter *, bool reset); @@ -882,7 +882,7 @@ static int ibmvnic_login(struct net_device *netdev) "Received partial success, retrying...\n"); adapter->init_done_rc = 0; reinit_completion(&adapter->init_done); - send_cap_queries(adapter); + send_query_cap(adapter); if (!wait_for_completion_timeout(&adapter->init_done, timeout)) { netdev_warn(netdev, @@ -1113,7 +1113,7 @@ static int init_resources(struct ibmvnic_adapter *adapter) if (rc) return rc; - send_map_query(adapter); + send_query_map(adapter); rc = init_rx_pools(netdev); if (rc) @@ -3299,7 +3299,7 @@ tx_failed: return -1; } -static void ibmvnic_send_req_caps(struct ibmvnic_adapter *adapter, int retry) +static void send_request_cap(struct ibmvnic_adapter *adapter, int retry) { struct device *dev = &adapter->vdev->dev; union ibmvnic_crq crq; @@ -3825,7 +3825,7 @@ static int send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id) return ibmvnic_send_crq(adapter, &crq); } -static void send_map_query(struct ibmvnic_adapter *adapter) +static void send_query_map(struct ibmvnic_adapter *adapter) { union ibmvnic_crq crq; @@ -3836,7 +3836,7 @@ static void send_map_query(struct ibmvnic_adapter *adapter) } /* Send a series of CRQs requesting various capabilities of the VNIC server */ -static void send_cap_queries(struct ibmvnic_adapter *adapter) +static void send_query_cap(struct ibmvnic_adapter *adapter) { union ibmvnic_crq crq; @@ -3953,6 +3953,113 @@ static void send_cap_queries(struct ibmvnic_adapter *adapter) ibmvnic_send_crq(adapter, &crq); } +static void send_query_ip_offload(struct ibmvnic_adapter *adapter) +{ + int buf_sz = sizeof(struct ibmvnic_query_ip_offload_buffer); + struct device *dev = &adapter->vdev->dev; + union ibmvnic_crq crq; + + adapter->ip_offload_tok = + dma_map_single(dev, + &adapter->ip_offload_buf, + buf_sz, + DMA_FROM_DEVICE); + + if (dma_mapping_error(dev, adapter->ip_offload_tok)) { + if (!firmware_has_feature(FW_FEATURE_CMO)) + dev_err(dev, "Couldn't map offload buffer\n"); + return; + } + + memset(&crq, 0, sizeof(crq)); + crq.query_ip_offload.first = IBMVNIC_CRQ_CMD; + crq.query_ip_offload.cmd = QUERY_IP_OFFLOAD; + crq.query_ip_offload.len = cpu_to_be32(buf_sz); + crq.query_ip_offload.ioba = + cpu_to_be32(adapter->ip_offload_tok); + + ibmvnic_send_crq(adapter, &crq); +} + +static void send_control_ip_offload(struct ibmvnic_adapter *adapter) +{ + struct ibmvnic_control_ip_offload_buffer *ctrl_buf = &adapter->ip_offload_ctrl; + struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf; + struct device *dev = &adapter->vdev->dev; + netdev_features_t old_hw_features = 0; + union ibmvnic_crq crq; + + adapter->ip_offload_ctrl_tok = + dma_map_single(dev, + ctrl_buf, + sizeof(adapter->ip_offload_ctrl), + DMA_TO_DEVICE); + + if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) { + dev_err(dev, "Couldn't map ip offload control buffer\n"); + return; + } + + ctrl_buf->len = cpu_to_be32(sizeof(adapter->ip_offload_ctrl)); + ctrl_buf->version = cpu_to_be32(INITIAL_VERSION_IOB); + ctrl_buf->ipv4_chksum = buf->ipv4_chksum; + ctrl_buf->ipv6_chksum = buf->ipv6_chksum; + ctrl_buf->tcp_ipv4_chksum = buf->tcp_ipv4_chksum; + ctrl_buf->udp_ipv4_chksum = buf->udp_ipv4_chksum; + ctrl_buf->tcp_ipv6_chksum = buf->tcp_ipv6_chksum; + ctrl_buf->udp_ipv6_chksum = buf->udp_ipv6_chksum; + ctrl_buf->large_tx_ipv4 = buf->large_tx_ipv4; + ctrl_buf->large_tx_ipv6 = buf->large_tx_ipv6; + + /* large_rx disabled for now, additional features needed */ + ctrl_buf->large_rx_ipv4 = 0; + ctrl_buf->large_rx_ipv6 = 0; + + if (adapter->state != VNIC_PROBING) { + old_hw_features = adapter->netdev->hw_features; + adapter->netdev->hw_features = 0; + } + + adapter->netdev->hw_features = NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO; + + if (buf->tcp_ipv4_chksum || buf->udp_ipv4_chksum) + adapter->netdev->hw_features |= NETIF_F_IP_CSUM; + + if (buf->tcp_ipv6_chksum || buf->udp_ipv6_chksum) + adapter->netdev->hw_features |= NETIF_F_IPV6_CSUM; + + if ((adapter->netdev->features & + (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) + adapter->netdev->hw_features |= NETIF_F_RXCSUM; + + if (buf->large_tx_ipv4) + adapter->netdev->hw_features |= NETIF_F_TSO; + if (buf->large_tx_ipv6) + adapter->netdev->hw_features |= NETIF_F_TSO6; + + if (adapter->state == VNIC_PROBING) { + adapter->netdev->features |= adapter->netdev->hw_features; + } else if (old_hw_features != adapter->netdev->hw_features) { + netdev_features_t tmp = 0; + + /* disable features no longer supported */ + adapter->netdev->features &= adapter->netdev->hw_features; + /* turn on features now supported if previously enabled */ + tmp = (old_hw_features ^ adapter->netdev->hw_features) & + adapter->netdev->hw_features; + adapter->netdev->features |= + tmp & adapter->netdev->wanted_features; + } + + memset(&crq, 0, sizeof(crq)); + crq.control_ip_offload.first = IBMVNIC_CRQ_CMD; + crq.control_ip_offload.cmd = CONTROL_IP_OFFLOAD; + crq.control_ip_offload.len = + cpu_to_be32(sizeof(adapter->ip_offload_ctrl)); + crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok); + ibmvnic_send_crq(adapter, &crq); +} + static void handle_vpd_size_rsp(union ibmvnic_crq *crq, struct ibmvnic_adapter *adapter) { @@ -4022,8 +4129,6 @@ static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter) { struct device *dev = &adapter->vdev->dev; struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf; - netdev_features_t old_hw_features = 0; - union ibmvnic_crq crq; int i; dma_unmap_single(dev, adapter->ip_offload_tok, @@ -4073,74 +4178,7 @@ static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter) netdev_dbg(adapter->netdev, "off_ipv6_ext_hd = %d\n", buf->off_ipv6_ext_headers); - adapter->ip_offload_ctrl_tok = - dma_map_single(dev, &adapter->ip_offload_ctrl, - sizeof(adapter->ip_offload_ctrl), DMA_TO_DEVICE); - - if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) { - dev_err(dev, "Couldn't map ip offload control buffer\n"); - return; - } - - adapter->ip_offload_ctrl.len = - cpu_to_be32(sizeof(adapter->ip_offload_ctrl)); - adapter->ip_offload_ctrl.version = cpu_to_be32(INITIAL_VERSION_IOB); - adapter->ip_offload_ctrl.ipv4_chksum = buf->ipv4_chksum; - adapter->ip_offload_ctrl.ipv6_chksum = buf->ipv6_chksum; - adapter->ip_offload_ctrl.tcp_ipv4_chksum = buf->tcp_ipv4_chksum; - adapter->ip_offload_ctrl.udp_ipv4_chksum = buf->udp_ipv4_chksum; - adapter->ip_offload_ctrl.tcp_ipv6_chksum = buf->tcp_ipv6_chksum; - adapter->ip_offload_ctrl.udp_ipv6_chksum = buf->udp_ipv6_chksum; - adapter->ip_offload_ctrl.large_tx_ipv4 = buf->large_tx_ipv4; - adapter->ip_offload_ctrl.large_tx_ipv6 = buf->large_tx_ipv6; - - /* large_rx disabled for now, additional features needed */ - adapter->ip_offload_ctrl.large_rx_ipv4 = 0; - adapter->ip_offload_ctrl.large_rx_ipv6 = 0; - - if (adapter->state != VNIC_PROBING) { - old_hw_features = adapter->netdev->hw_features; - adapter->netdev->hw_features = 0; - } - - adapter->netdev->hw_features = NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO; - - if (buf->tcp_ipv4_chksum || buf->udp_ipv4_chksum) - adapter->netdev->hw_features |= NETIF_F_IP_CSUM; - - if (buf->tcp_ipv6_chksum || buf->udp_ipv6_chksum) - adapter->netdev->hw_features |= NETIF_F_IPV6_CSUM; - - if ((adapter->netdev->features & - (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) - adapter->netdev->hw_features |= NETIF_F_RXCSUM; - - if (buf->large_tx_ipv4) - adapter->netdev->hw_features |= NETIF_F_TSO; - if (buf->large_tx_ipv6) - adapter->netdev->hw_features |= NETIF_F_TSO6; - - if (adapter->state == VNIC_PROBING) { - adapter->netdev->features |= adapter->netdev->hw_features; - } else if (old_hw_features != adapter->netdev->hw_features) { - netdev_features_t tmp = 0; - - /* disable features no longer supported */ - adapter->netdev->features &= adapter->netdev->hw_features; - /* turn on features now supported if previously enabled */ - tmp = (old_hw_features ^ adapter->netdev->hw_features) & - adapter->netdev->hw_features; - adapter->netdev->features |= - tmp & adapter->netdev->wanted_features; - } - - memset(&crq, 0, sizeof(crq)); - crq.control_ip_offload.first = IBMVNIC_CRQ_CMD; - crq.control_ip_offload.cmd = CONTROL_IP_OFFLOAD; - crq.control_ip_offload.len = - cpu_to_be32(sizeof(adapter->ip_offload_ctrl)); - crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok); - ibmvnic_send_crq(adapter, &crq); + send_control_ip_offload(adapter); } static const char *ibmvnic_fw_err_cause(u16 cause) @@ -4266,7 +4304,7 @@ static void handle_request_cap_rsp(union ibmvnic_crq *crq, be64_to_cpu(crq->request_capability_rsp.number); } - ibmvnic_send_req_caps(adapter, 1); + send_request_cap(adapter, 1); return; default: dev_err(dev, "Error %d in request cap rsp\n", @@ -4276,30 +4314,8 @@ static void handle_request_cap_rsp(union ibmvnic_crq *crq, /* Done receiving requested capabilities, query IP offload support */ if (atomic_read(&adapter->running_cap_crqs) == 0) { - union ibmvnic_crq newcrq; - int buf_sz = sizeof(struct ibmvnic_query_ip_offload_buffer); - struct ibmvnic_query_ip_offload_buffer *ip_offload_buf = - &adapter->ip_offload_buf; - adapter->wait_capability = false; - adapter->ip_offload_tok = dma_map_single(dev, ip_offload_buf, - buf_sz, - DMA_FROM_DEVICE); - - if (dma_mapping_error(dev, adapter->ip_offload_tok)) { - if (!firmware_has_feature(FW_FEATURE_CMO)) - dev_err(dev, "Couldn't map offload buffer\n"); - return; - } - - memset(&newcrq, 0, sizeof(newcrq)); - newcrq.query_ip_offload.first = IBMVNIC_CRQ_CMD; - newcrq.query_ip_offload.cmd = QUERY_IP_OFFLOAD; - newcrq.query_ip_offload.len = cpu_to_be32(buf_sz); - newcrq.query_ip_offload.ioba = - cpu_to_be32(adapter->ip_offload_tok); - - ibmvnic_send_crq(adapter, &newcrq); + send_query_ip_offload(adapter); } } @@ -4582,7 +4598,7 @@ static void handle_query_cap_rsp(union ibmvnic_crq *crq, out: if (atomic_read(&adapter->running_cap_crqs) == 0) { adapter->wait_capability = false; - ibmvnic_send_req_caps(adapter, 0); + send_request_cap(adapter, 0); } } @@ -4637,7 +4653,7 @@ static int handle_query_phys_parms_rsp(union ibmvnic_crq *crq, case IBMVNIC_1GBPS: adapter->speed = SPEED_1000; break; - case IBMVNIC_10GBP: + case IBMVNIC_10GBPS: adapter->speed = SPEED_10000; break; case IBMVNIC_25GBPS: @@ -4652,6 +4668,9 @@ static int handle_query_phys_parms_rsp(union ibmvnic_crq *crq, case IBMVNIC_100GBPS: adapter->speed = SPEED_100000; break; + case IBMVNIC_200GBPS: + adapter->speed = SPEED_200000; + break; default: if (netif_carrier_ok(netdev)) netdev_warn(netdev, "Unknown speed 0x%08x\n", rspeed); @@ -4747,7 +4766,7 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq, be16_to_cpu(crq->version_exchange_rsp.version); dev_info(dev, "Partner protocol version is %d\n", ibmvnic_version); - send_cap_queries(adapter); + send_query_cap(adapter); break; case QUERY_CAPABILITY_RSP: handle_query_cap_rsp(crq, adapter); diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h index 8da98794eda9..217dcc7ded70 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.h +++ b/drivers/net/ethernet/ibm/ibmvnic.h @@ -373,7 +373,7 @@ struct ibmvnic_phys_parms { #define IBMVNIC_10MBPS 0x40000000 #define IBMVNIC_100MBPS 0x20000000 #define IBMVNIC_1GBPS 0x10000000 -#define IBMVNIC_10GBP 0x08000000 +#define IBMVNIC_10GBPS 0x08000000 #define IBMVNIC_40GBPS 0x04000000 #define IBMVNIC_100GBPS 0x02000000 #define IBMVNIC_25GBPS 0x01000000 diff --git a/drivers/net/ethernet/intel/e100.c b/drivers/net/ethernet/intel/e100.c index 76bb77b4607a..8cc651d37a7f 100644 --- a/drivers/net/ethernet/intel/e100.c +++ b/drivers/net/ethernet/intel/e100.c @@ -1531,7 +1531,7 @@ static int e100_hw_init(struct nic *nic) e100_hw_reset(nic); netif_err(nic, hw, nic->netdev, "e100_hw_init\n"); - if (!in_interrupt() && (err = e100_self_test(nic))) + if ((err = e100_self_test(nic))) return err; if ((err = e100_phy_init(nic))) @@ -2155,7 +2155,7 @@ static int e100_rx_alloc_list(struct nic *nic) nic->rx_to_use = nic->rx_to_clean = NULL; nic->ru_running = RU_UNINITIALIZED; - if (!(nic->rxs = kcalloc(count, sizeof(struct rx), GFP_ATOMIC))) + if (!(nic->rxs = kcalloc(count, sizeof(struct rx), GFP_KERNEL))) return -ENOMEM; for (rx = nic->rxs, i = 0; i < count; rx++, i++) { diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 906591614553..5e28cf4fa2cd 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -536,7 +536,6 @@ void e1000_down(struct e1000_adapter *adapter) void e1000_reinit_locked(struct e1000_adapter *adapter) { - WARN_ON(in_interrupt()); while (test_and_set_bit(__E1000_RESETTING, &adapter->flags)) msleep(1); diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c index a8fc9208382c..03215b0aee4b 100644 --- a/drivers/net/ethernet/intel/e1000e/ethtool.c +++ b/drivers/net/ethernet/intel/e1000e/ethtool.c @@ -895,6 +895,7 @@ static int e1000_reg_test(struct e1000_adapter *adapter, u64 *data) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: mask |= BIT(18); break; default: @@ -1560,6 +1561,7 @@ static void e1000_loopback_cleanup(struct e1000_adapter *adapter) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: fext_nvm11 = er32(FEXTNVM11); fext_nvm11 &= ~E1000_FEXTNVM11_DISABLE_MULR_FIX; ew32(FEXTNVM11, fext_nvm11); diff --git a/drivers/net/ethernet/intel/e1000e/hw.h b/drivers/net/ethernet/intel/e1000e/hw.h index b1447221669e..69a2329ea463 100644 --- a/drivers/net/ethernet/intel/e1000e/hw.h +++ b/drivers/net/ethernet/intel/e1000e/hw.h @@ -102,6 +102,10 @@ struct e1000_hw; #define E1000_DEV_ID_PCH_ADP_I219_V16 0x1A1F #define E1000_DEV_ID_PCH_ADP_I219_LM17 0x1A1C #define E1000_DEV_ID_PCH_ADP_I219_V17 0x1A1D +#define E1000_DEV_ID_PCH_MTP_I219_LM18 0x550A +#define E1000_DEV_ID_PCH_MTP_I219_V18 0x550B +#define E1000_DEV_ID_PCH_MTP_I219_LM19 0x550C +#define E1000_DEV_ID_PCH_MTP_I219_V19 0x550D #define E1000_REVISION_4 4 @@ -127,6 +131,7 @@ enum e1000_mac_type { e1000_pch_cnp, e1000_pch_tgp, e1000_pch_adp, + e1000_pch_mtp, }; enum e1000_media_type { diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c index ded74304e8cf..9aa6fad8ed47 100644 --- a/drivers/net/ethernet/intel/e1000e/ich8lan.c +++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c @@ -320,6 +320,7 @@ static s32 e1000_init_phy_workarounds_pchlan(struct e1000_hw *hw) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: if (e1000_phy_is_accessible_pchlan(hw)) break; @@ -464,6 +465,7 @@ static s32 e1000_init_phy_params_pchlan(struct e1000_hw *hw) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: /* In case the PHY needs to be in mdio slow mode, * set slow mode and try to get the PHY id again. */ @@ -708,6 +710,7 @@ static s32 e1000_init_mac_params_ich8lan(struct e1000_hw *hw) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: case e1000_pchlan: /* check management mode */ mac->ops.check_mng_mode = e1000_check_mng_mode_pchlan; @@ -1648,6 +1651,7 @@ static s32 e1000_get_variants_ich8lan(struct e1000_adapter *adapter) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: rc = e1000_init_phy_params_pchlan(hw); break; default: @@ -2102,6 +2106,7 @@ static s32 e1000_sw_lcd_config_ich8lan(struct e1000_hw *hw) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: sw_cfg_mask = E1000_FEXTNVM_SW_CONFIG_ICH8M; break; default: @@ -3145,6 +3150,7 @@ static s32 e1000_valid_nvm_bank_detect_ich8lan(struct e1000_hw *hw, u32 *bank) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: bank1_offset = nvm->flash_bank_size; act_offset = E1000_ICH_NVM_SIG_WORD; @@ -4090,6 +4096,7 @@ static s32 e1000_validate_nvm_checksum_ich8lan(struct e1000_hw *hw) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: word = NVM_COMPAT; valid_csum_mask = NVM_COMPAT_VALID_CSUM; break; diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 99f4ec9b5696..b30f00891c03 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -3587,6 +3587,7 @@ s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) { /* Stable 24MHz frequency */ incperiod = INCPERIOD_24MHZ; @@ -4104,6 +4105,7 @@ void e1000e_reset(struct e1000_adapter *adapter) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: fc->refresh_time = 0xFFFF; fc->pause_time = 0xFFFF; @@ -7877,6 +7879,10 @@ static const struct pci_device_id e1000_pci_tbl[] = { { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V16), board_pch_cnp }, { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM17), board_pch_cnp }, { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V17), board_pch_cnp }, + { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM18), board_pch_cnp }, + { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V18), board_pch_cnp }, + { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM19), board_pch_cnp }, + { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V19), board_pch_cnp }, { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */ }; diff --git a/drivers/net/ethernet/intel/e1000e/ptp.c b/drivers/net/ethernet/intel/e1000e/ptp.c index 8d21bcb427ec..f3f671311855 100644 --- a/drivers/net/ethernet/intel/e1000e/ptp.c +++ b/drivers/net/ethernet/intel/e1000e/ptp.c @@ -297,6 +297,7 @@ void e1000e_ptp_init(struct e1000_adapter *adapter) case e1000_pch_cnp: case e1000_pch_tgp: case e1000_pch_adp: + case e1000_pch_mtp: if ((hw->mac.type < e1000_pch_lpt) || (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI)) { adapter->ptp_clock_info.max_adj = 24000000 - 1; diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c index 140212bfe08b..9e3103fae723 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c @@ -221,8 +221,6 @@ static bool fm10k_prepare_for_reset(struct fm10k_intfc *interface) { struct net_device *netdev = interface->netdev; - WARN_ON(in_interrupt()); - /* put off any impending NetWatchDogTimeout */ netif_trans_update(netdev); diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index ada0e93c38f0..537300e762f0 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -35,6 +35,7 @@ #include <net/pkt_cls.h> #include <net/tc_act/tc_gact.h> #include <net/tc_act/tc_mirred.h> +#include <net/udp_tunnel.h> #include <net/xdp_sock.h> #include "i40e_type.h" #include "i40e_prototype.h" @@ -133,7 +134,6 @@ enum i40e_state_t { __I40E_PORT_SUSPENDED, __I40E_VF_DISABLE, __I40E_MACVLAN_SYNC_PENDING, - __I40E_UDP_FILTER_SYNC_PENDING, __I40E_TEMP_LINK_POLLING, __I40E_CLIENT_SERVICE_REQUESTED, __I40E_CLIENT_L2_CHANGE, @@ -478,8 +478,8 @@ struct i40e_pf { struct list_head l3_flex_pit_list; struct list_head l4_flex_pit_list; - struct i40e_udp_port_config udp_ports[I40E_MAX_PF_UDP_OFFLOAD_PORTS]; - u16 pending_udp_bitmap; + struct udp_tunnel_nic_shared udp_tunnel_shared; + struct udp_tunnel_nic_info udp_tunnel_nic; struct hlist_head cloud_filter_list; u16 num_cloud_filters; diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index cb366daa1ad7..929c64789119 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -6691,7 +6691,6 @@ static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) { struct i40e_pf *pf = vsi->back; - WARN_ON(in_interrupt()); while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) usleep_range(1000, 2000); i40e_down(vsi); @@ -8464,9 +8463,6 @@ void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) { u32 val; - WARN_ON(in_interrupt()); - - /* do the biggest reset indicated */ if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { @@ -10386,106 +10382,6 @@ static void i40e_handle_mdd_event(struct i40e_pf *pf) i40e_flush(hw); } -static const char *i40e_tunnel_name(u8 type) -{ - switch (type) { - case UDP_TUNNEL_TYPE_VXLAN: - return "vxlan"; - case UDP_TUNNEL_TYPE_GENEVE: - return "geneve"; - default: - return "unknown"; - } -} - -/** - * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters - * @pf: board private structure - **/ -static void i40e_sync_udp_filters(struct i40e_pf *pf) -{ - int i; - - /* loop through and set pending bit for all active UDP filters */ - for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { - if (pf->udp_ports[i].port) - pf->pending_udp_bitmap |= BIT_ULL(i); - } - - set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); -} - -/** - * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW - * @pf: board private structure - **/ -static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - u8 filter_index, type; - u16 port; - int i; - - if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state)) - return; - - /* acquire RTNL to maintain state of flags and port requests */ - rtnl_lock(); - - for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { - if (pf->pending_udp_bitmap & BIT_ULL(i)) { - struct i40e_udp_port_config *udp_port; - i40e_status ret = 0; - - udp_port = &pf->udp_ports[i]; - pf->pending_udp_bitmap &= ~BIT_ULL(i); - - port = READ_ONCE(udp_port->port); - type = READ_ONCE(udp_port->type); - filter_index = READ_ONCE(udp_port->filter_index); - - /* release RTNL while we wait on AQ command */ - rtnl_unlock(); - - if (port) - ret = i40e_aq_add_udp_tunnel(hw, port, - type, - &filter_index, - NULL); - else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED) - ret = i40e_aq_del_udp_tunnel(hw, filter_index, - NULL); - - /* reacquire RTNL so we can update filter_index */ - rtnl_lock(); - - if (ret) { - dev_info(&pf->pdev->dev, - "%s %s port %d, index %d failed, err %s aq_err %s\n", - i40e_tunnel_name(type), - port ? "add" : "delete", - port, - filter_index, - i40e_stat_str(&pf->hw, ret), - i40e_aq_str(&pf->hw, - pf->hw.aq.asq_last_status)); - if (port) { - /* failed to add, just reset port, - * drop pending bit for any deletion - */ - udp_port->port = 0; - pf->pending_udp_bitmap &= ~BIT_ULL(i); - } - } else if (port) { - /* record filter index on success */ - udp_port->filter_index = filter_index; - } - } - } - - rtnl_unlock(); -} - /** * i40e_service_task - Run the driver's async subtasks * @work: pointer to work_struct containing our data @@ -10525,7 +10421,6 @@ static void i40e_service_task(struct work_struct *work) pf->vsi[pf->lan_vsi]); } i40e_sync_filters_subtask(pf); - i40e_sync_udp_filters_subtask(pf); } else { i40e_reset_subtask(pf); } @@ -12225,131 +12120,48 @@ static int i40e_set_features(struct net_device *netdev, return 0; } -/** - * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port - * @pf: board private structure - * @port: The UDP port to look up - * - * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found - **/ -static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) -{ - u8 i; - - for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { - /* Do not report ports with pending deletions as - * being available. - */ - if (!port && (pf->pending_udp_bitmap & BIT_ULL(i))) - continue; - if (pf->udp_ports[i].port == port) - return i; - } - - return i; -} - -/** - * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up - * @netdev: This physical port's netdev - * @ti: Tunnel endpoint information - **/ -static void i40e_udp_tunnel_add(struct net_device *netdev, - struct udp_tunnel_info *ti) +static int i40e_udp_tunnel_set_port(struct net_device *netdev, + unsigned int table, unsigned int idx, + struct udp_tunnel_info *ti) { struct i40e_netdev_priv *np = netdev_priv(netdev); - struct i40e_vsi *vsi = np->vsi; - struct i40e_pf *pf = vsi->back; - u16 port = ntohs(ti->port); - u8 next_idx; - u8 idx; - - idx = i40e_get_udp_port_idx(pf, port); - - /* Check if port already exists */ - if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { - netdev_info(netdev, "port %d already offloaded\n", port); - return; - } - - /* Now check if there is space to add the new port */ - next_idx = i40e_get_udp_port_idx(pf, 0); + struct i40e_hw *hw = &np->vsi->back->hw; + u8 type, filter_index; + i40e_status ret; - if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { - netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", - port); - return; - } + type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN : + I40E_AQC_TUNNEL_TYPE_NGE; - switch (ti->type) { - case UDP_TUNNEL_TYPE_VXLAN: - pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; - break; - case UDP_TUNNEL_TYPE_GENEVE: - if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE)) - return; - pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; - break; - default: - return; + ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index, + NULL); + if (ret) { + netdev_info(netdev, "add UDP port failed, err %s aq_err %s\n", + i40e_stat_str(hw, ret), + i40e_aq_str(hw, hw->aq.asq_last_status)); + return -EIO; } - /* New port: add it and mark its index in the bitmap */ - pf->udp_ports[next_idx].port = port; - pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED; - pf->pending_udp_bitmap |= BIT_ULL(next_idx); - set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); + udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index); + return 0; } -/** - * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away - * @netdev: This physical port's netdev - * @ti: Tunnel endpoint information - **/ -static void i40e_udp_tunnel_del(struct net_device *netdev, - struct udp_tunnel_info *ti) +static int i40e_udp_tunnel_unset_port(struct net_device *netdev, + unsigned int table, unsigned int idx, + struct udp_tunnel_info *ti) { struct i40e_netdev_priv *np = netdev_priv(netdev); - struct i40e_vsi *vsi = np->vsi; - struct i40e_pf *pf = vsi->back; - u16 port = ntohs(ti->port); - u8 idx; - - idx = i40e_get_udp_port_idx(pf, port); - - /* Check if port already exists */ - if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) - goto not_found; + struct i40e_hw *hw = &np->vsi->back->hw; + i40e_status ret; - switch (ti->type) { - case UDP_TUNNEL_TYPE_VXLAN: - if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) - goto not_found; - break; - case UDP_TUNNEL_TYPE_GENEVE: - if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) - goto not_found; - break; - default: - goto not_found; + ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL); + if (ret) { + netdev_info(netdev, "delete UDP port failed, err %s aq_err %s\n", + i40e_stat_str(hw, ret), + i40e_aq_str(hw, hw->aq.asq_last_status)); + return -EIO; } - /* if port exists, set it to 0 (mark for deletion) - * and make it pending - */ - pf->udp_ports[idx].port = 0; - - /* Toggle pending bit instead of setting it. This way if we are - * deleting a port that has yet to be added we just clear the pending - * bit and don't have to worry about it. - */ - pf->pending_udp_bitmap ^= BIT_ULL(idx); - set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); - - return; -not_found: - netdev_warn(netdev, "UDP port %d was not found, not deleting\n", - port); + return 0; } static int i40e_get_phys_port_id(struct net_device *netdev, @@ -12955,8 +12767,8 @@ static const struct net_device_ops i40e_netdev_ops = { .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, .ndo_set_vf_trust = i40e_ndo_set_vf_trust, - .ndo_udp_tunnel_add = i40e_udp_tunnel_add, - .ndo_udp_tunnel_del = i40e_udp_tunnel_del, + .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, + .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_get_phys_port_id = i40e_get_phys_port_id, .ndo_fdb_add = i40e_ndo_fdb_add, .ndo_features_check = i40e_features_check, @@ -13020,6 +12832,8 @@ static int i40e_config_netdev(struct i40e_vsi *vsi) if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; + netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic; + netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; netdev->hw_enc_features |= hw_enc_features; @@ -14420,7 +14234,7 @@ static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) i40e_ptp_init(pf); /* repopulate tunnel port filters */ - i40e_sync_udp_filters(pf); + udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev); return ret; } @@ -15149,6 +14963,14 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) goto err_switch_setup; + pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port; + pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port; + pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP; + pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared; + pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS; + pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN | + UDP_TUNNEL_TYPE_GENEVE; + /* The number of VSIs reported by the FW is the minimum guaranteed * to us; HW supports far more and we share the remaining pool with * the other PFs. We allocate space for more than the guarantee with @@ -15158,6 +14980,12 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; else pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; + if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { + dev_warn(&pf->pdev->dev, + "limiting the VSI count due to UDP tunnel limitation %d > %d\n", + pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); + pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; + } /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool_fdir.c b/drivers/net/ethernet/intel/ice/ice_ethtool_fdir.c index d7430ce6af26..2d27f66ac853 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool_fdir.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool_fdir.c @@ -1268,8 +1268,7 @@ ice_fdir_write_all_fltr(struct ice_pf *pf, struct ice_fdir_fltr *input, bool is_tun = tun == ICE_FD_HW_SEG_TUN; int err; - if (is_tun && !ice_get_open_tunnel_port(&pf->hw, TNL_ALL, - &port_num)) + if (is_tun && !ice_get_open_tunnel_port(&pf->hw, &port_num)) continue; err = ice_fdir_write_fltr(pf, input, add, is_tun); if (err) @@ -1647,8 +1646,7 @@ int ice_add_fdir_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd) } /* return error if not an update and no available filters */ - fltrs_needed = ice_get_open_tunnel_port(hw, TNL_ALL, &tunnel_port) ? - 2 : 1; + fltrs_needed = ice_get_open_tunnel_port(hw, &tunnel_port) ? 2 : 1; if (!ice_fdir_find_fltr_by_idx(hw, fsp->location) && ice_fdir_num_avail_fltr(hw, pf->vsi[vsi->idx]) < fltrs_needed) { dev_err(dev, "Failed to add filter. The maximum number of flow director filters has been reached.\n"); diff --git a/drivers/net/ethernet/intel/ice/ice_fdir.c b/drivers/net/ethernet/intel/ice/ice_fdir.c index 6834df14332f..59c0c6a0f8c5 100644 --- a/drivers/net/ethernet/intel/ice/ice_fdir.c +++ b/drivers/net/ethernet/intel/ice/ice_fdir.c @@ -556,7 +556,7 @@ ice_fdir_get_gen_prgm_pkt(struct ice_hw *hw, struct ice_fdir_fltr *input, memcpy(pkt, ice_fdir_pkt[idx].pkt, ice_fdir_pkt[idx].pkt_len); loc = pkt; } else { - if (!ice_get_open_tunnel_port(hw, TNL_ALL, &tnl_port)) + if (!ice_get_open_tunnel_port(hw, &tnl_port)) return ICE_ERR_DOES_NOT_EXIST; if (!ice_fdir_pkt[idx].tun_pkt) return ICE_ERR_PARAM; diff --git a/drivers/net/ethernet/intel/ice/ice_flex_pipe.c b/drivers/net/ethernet/intel/ice/ice_flex_pipe.c index b17ae3e20157..ac448d223e9a 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_pipe.c +++ b/drivers/net/ethernet/intel/ice/ice_flex_pipe.c @@ -489,8 +489,6 @@ static void ice_init_pkg_hints(struct ice_hw *hw, struct ice_seg *ice_seg) if ((label_name[len] - '0') == hw->pf_id) { hw->tnl.tbl[hw->tnl.count].type = tnls[i].type; hw->tnl.tbl[hw->tnl.count].valid = false; - hw->tnl.tbl[hw->tnl.count].in_use = false; - hw->tnl.tbl[hw->tnl.count].marked = false; hw->tnl.tbl[hw->tnl.count].boost_addr = val; hw->tnl.tbl[hw->tnl.count].port = 0; hw->tnl.count++; @@ -505,8 +503,11 @@ static void ice_init_pkg_hints(struct ice_hw *hw, struct ice_seg *ice_seg) for (i = 0; i < hw->tnl.count; i++) { ice_find_boost_entry(ice_seg, hw->tnl.tbl[i].boost_addr, &hw->tnl.tbl[i].boost_entry); - if (hw->tnl.tbl[i].boost_entry) + if (hw->tnl.tbl[i].boost_entry) { hw->tnl.tbl[i].valid = true; + if (hw->tnl.tbl[i].type < __TNL_TYPE_CNT) + hw->tnl.valid_count[hw->tnl.tbl[i].type]++; + } } } @@ -1626,104 +1627,59 @@ static struct ice_buf *ice_pkg_buf(struct ice_buf_build *bld) } /** - * ice_tunnel_port_in_use_hlpr - helper function to determine tunnel usage + * ice_get_open_tunnel_port - retrieve an open tunnel port * @hw: pointer to the HW structure - * @port: port to search for - * @index: optionally returns index - * - * Returns whether a port is already in use as a tunnel, and optionally its - * index + * @port: returns open port */ -static bool ice_tunnel_port_in_use_hlpr(struct ice_hw *hw, u16 port, u16 *index) +bool +ice_get_open_tunnel_port(struct ice_hw *hw, u16 *port) { + bool res = false; u16 i; + mutex_lock(&hw->tnl_lock); + for (i = 0; i < hw->tnl.count && i < ICE_TUNNEL_MAX_ENTRIES; i++) - if (hw->tnl.tbl[i].in_use && hw->tnl.tbl[i].port == port) { - if (index) - *index = i; - return true; + if (hw->tnl.tbl[i].valid && hw->tnl.tbl[i].port) { + *port = hw->tnl.tbl[i].port; + res = true; + break; } - return false; -} - -/** - * ice_tunnel_port_in_use - * @hw: pointer to the HW structure - * @port: port to search for - * @index: optionally returns index - * - * Returns whether a port is already in use as a tunnel, and optionally its - * index - */ -bool ice_tunnel_port_in_use(struct ice_hw *hw, u16 port, u16 *index) -{ - bool res; - - mutex_lock(&hw->tnl_lock); - res = ice_tunnel_port_in_use_hlpr(hw, port, index); mutex_unlock(&hw->tnl_lock); return res; } /** - * ice_find_free_tunnel_entry + * ice_tunnel_idx_to_entry - convert linear index to the sparse one * @hw: pointer to the HW structure - * @type: tunnel type - * @index: optionally returns index + * @type: type of tunnel + * @idx: linear index * - * Returns whether there is a free tunnel entry, and optionally its index + * Stack assumes we have 2 linear tables with indexes [0, count_valid), + * but really the port table may be sprase, and types are mixed, so convert + * the stack index into the device index. */ -static bool -ice_find_free_tunnel_entry(struct ice_hw *hw, enum ice_tunnel_type type, - u16 *index) +static u16 ice_tunnel_idx_to_entry(struct ice_hw *hw, enum ice_tunnel_type type, + u16 idx) { u16 i; for (i = 0; i < hw->tnl.count && i < ICE_TUNNEL_MAX_ENTRIES; i++) - if (hw->tnl.tbl[i].valid && !hw->tnl.tbl[i].in_use && - hw->tnl.tbl[i].type == type) { - if (index) - *index = i; - return true; - } + if (hw->tnl.tbl[i].valid && + hw->tnl.tbl[i].type == type && + idx--) + return i; - return false; -} - -/** - * ice_get_open_tunnel_port - retrieve an open tunnel port - * @hw: pointer to the HW structure - * @type: tunnel type (TNL_ALL will return any open port) - * @port: returns open port - */ -bool -ice_get_open_tunnel_port(struct ice_hw *hw, enum ice_tunnel_type type, - u16 *port) -{ - bool res = false; - u16 i; - - mutex_lock(&hw->tnl_lock); - - for (i = 0; i < hw->tnl.count && i < ICE_TUNNEL_MAX_ENTRIES; i++) - if (hw->tnl.tbl[i].valid && hw->tnl.tbl[i].in_use && - (type == TNL_ALL || hw->tnl.tbl[i].type == type)) { - *port = hw->tnl.tbl[i].port; - res = true; - break; - } - - mutex_unlock(&hw->tnl_lock); - - return res; + WARN_ON_ONCE(1); + return 0; } /** * ice_create_tunnel * @hw: pointer to the HW structure + * @index: device table entry * @type: type of tunnel * @port: port of tunnel to create * @@ -1731,27 +1687,16 @@ ice_get_open_tunnel_port(struct ice_hw *hw, enum ice_tunnel_type type, * creating a package buffer with the tunnel info and issuing an update package * command. */ -enum ice_status -ice_create_tunnel(struct ice_hw *hw, enum ice_tunnel_type type, u16 port) +static enum ice_status +ice_create_tunnel(struct ice_hw *hw, u16 index, + enum ice_tunnel_type type, u16 port) { struct ice_boost_tcam_section *sect_rx, *sect_tx; enum ice_status status = ICE_ERR_MAX_LIMIT; struct ice_buf_build *bld; - u16 index; mutex_lock(&hw->tnl_lock); - if (ice_tunnel_port_in_use_hlpr(hw, port, &index)) { - hw->tnl.tbl[index].ref++; - status = 0; - goto ice_create_tunnel_end; - } - - if (!ice_find_free_tunnel_entry(hw, type, &index)) { - status = ICE_ERR_OUT_OF_RANGE; - goto ice_create_tunnel_end; - } - bld = ice_pkg_buf_alloc(hw); if (!bld) { status = ICE_ERR_NO_MEMORY; @@ -1790,11 +1735,8 @@ ice_create_tunnel(struct ice_hw *hw, enum ice_tunnel_type type, u16 port) memcpy(sect_tx->tcam, sect_rx->tcam, sizeof(*sect_tx->tcam)); status = ice_update_pkg(hw, ice_pkg_buf(bld), 1); - if (!status) { + if (!status) hw->tnl.tbl[index].port = port; - hw->tnl.tbl[index].in_use = true; - hw->tnl.tbl[index].ref = 1; - } ice_create_tunnel_err: ice_pkg_buf_free(hw, bld); @@ -1808,46 +1750,31 @@ ice_create_tunnel_end: /** * ice_destroy_tunnel * @hw: pointer to the HW structure + * @index: device table entry + * @type: type of tunnel * @port: port of tunnel to destroy (ignored if the all parameter is true) - * @all: flag that states to destroy all tunnels * * Destroys a tunnel or all tunnels by creating an update package buffer * targeting the specific updates requested and then performing an update * package. */ -enum ice_status ice_destroy_tunnel(struct ice_hw *hw, u16 port, bool all) +static enum ice_status +ice_destroy_tunnel(struct ice_hw *hw, u16 index, enum ice_tunnel_type type, + u16 port) { struct ice_boost_tcam_section *sect_rx, *sect_tx; enum ice_status status = ICE_ERR_MAX_LIMIT; struct ice_buf_build *bld; - u16 count = 0; - u16 index; - u16 size; - u16 i; mutex_lock(&hw->tnl_lock); - if (!all && ice_tunnel_port_in_use_hlpr(hw, port, &index)) - if (hw->tnl.tbl[index].ref > 1) { - hw->tnl.tbl[index].ref--; - status = 0; - goto ice_destroy_tunnel_end; - } - - /* determine count */ - for (i = 0; i < hw->tnl.count && i < ICE_TUNNEL_MAX_ENTRIES; i++) - if (hw->tnl.tbl[i].valid && hw->tnl.tbl[i].in_use && - (all || hw->tnl.tbl[i].port == port)) - count++; - - if (!count) { - status = ICE_ERR_PARAM; + if (WARN_ON(!hw->tnl.tbl[index].valid || + hw->tnl.tbl[index].type != type || + hw->tnl.tbl[index].port != port)) { + status = ICE_ERR_OUT_OF_RANGE; goto ice_destroy_tunnel_end; } - /* size of section - there is at least one entry */ - size = struct_size(sect_rx, tcam, count); - bld = ice_pkg_buf_alloc(hw); if (!bld) { status = ICE_ERR_NO_MEMORY; @@ -1859,13 +1786,13 @@ enum ice_status ice_destroy_tunnel(struct ice_hw *hw, u16 port, bool all) goto ice_destroy_tunnel_err; sect_rx = ice_pkg_buf_alloc_section(bld, ICE_SID_RXPARSER_BOOST_TCAM, - size); + struct_size(sect_rx, tcam, 1)); if (!sect_rx) goto ice_destroy_tunnel_err; sect_rx->count = cpu_to_le16(1); sect_tx = ice_pkg_buf_alloc_section(bld, ICE_SID_TXPARSER_BOOST_TCAM, - size); + struct_size(sect_tx, tcam, 1)); if (!sect_tx) goto ice_destroy_tunnel_err; sect_tx->count = cpu_to_le16(1); @@ -1873,26 +1800,14 @@ enum ice_status ice_destroy_tunnel(struct ice_hw *hw, u16 port, bool all) /* copy original boost entry to update package buffer, one copy to Rx * section, another copy to the Tx section */ - for (i = 0; i < hw->tnl.count && i < ICE_TUNNEL_MAX_ENTRIES; i++) - if (hw->tnl.tbl[i].valid && hw->tnl.tbl[i].in_use && - (all || hw->tnl.tbl[i].port == port)) { - memcpy(sect_rx->tcam + i, hw->tnl.tbl[i].boost_entry, - sizeof(*sect_rx->tcam)); - memcpy(sect_tx->tcam + i, hw->tnl.tbl[i].boost_entry, - sizeof(*sect_tx->tcam)); - hw->tnl.tbl[i].marked = true; - } + memcpy(sect_rx->tcam, hw->tnl.tbl[index].boost_entry, + sizeof(*sect_rx->tcam)); + memcpy(sect_tx->tcam, hw->tnl.tbl[index].boost_entry, + sizeof(*sect_tx->tcam)); status = ice_update_pkg(hw, ice_pkg_buf(bld), 1); if (!status) - for (i = 0; i < hw->tnl.count && - i < ICE_TUNNEL_MAX_ENTRIES; i++) - if (hw->tnl.tbl[i].marked) { - hw->tnl.tbl[i].ref = 0; - hw->tnl.tbl[i].port = 0; - hw->tnl.tbl[i].in_use = false; - hw->tnl.tbl[i].marked = false; - } + hw->tnl.tbl[index].port = 0; ice_destroy_tunnel_err: ice_pkg_buf_free(hw, bld); @@ -1903,6 +1818,52 @@ ice_destroy_tunnel_end: return status; } +int ice_udp_tunnel_set_port(struct net_device *netdev, unsigned int table, + unsigned int idx, struct udp_tunnel_info *ti) +{ + struct ice_netdev_priv *np = netdev_priv(netdev); + struct ice_vsi *vsi = np->vsi; + struct ice_pf *pf = vsi->back; + enum ice_tunnel_type tnl_type; + enum ice_status status; + u16 index; + + tnl_type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? TNL_VXLAN : TNL_GENEVE; + index = ice_tunnel_idx_to_entry(&pf->hw, idx, tnl_type); + + status = ice_create_tunnel(&pf->hw, index, tnl_type, ntohs(ti->port)); + if (status) { + netdev_err(netdev, "Error adding UDP tunnel - %s\n", + ice_stat_str(status)); + return -EIO; + } + + udp_tunnel_nic_set_port_priv(netdev, table, idx, index); + return 0; +} + +int ice_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table, + unsigned int idx, struct udp_tunnel_info *ti) +{ + struct ice_netdev_priv *np = netdev_priv(netdev); + struct ice_vsi *vsi = np->vsi; + struct ice_pf *pf = vsi->back; + enum ice_tunnel_type tnl_type; + enum ice_status status; + + tnl_type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? TNL_VXLAN : TNL_GENEVE; + + status = ice_destroy_tunnel(&pf->hw, ti->hw_priv, tnl_type, + ntohs(ti->port)); + if (status) { + netdev_err(netdev, "Error removing UDP tunnel - %s\n", + ice_stat_str(status)); + return -EIO; + } + + return 0; +} + /* PTG Management */ /** diff --git a/drivers/net/ethernet/intel/ice/ice_flex_pipe.h b/drivers/net/ethernet/intel/ice/ice_flex_pipe.h index 568ea519af51..20deddb807c5 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_pipe.h +++ b/drivers/net/ethernet/intel/ice/ice_flex_pipe.h @@ -19,12 +19,11 @@ #define ICE_PKG_CNT 4 bool -ice_get_open_tunnel_port(struct ice_hw *hw, enum ice_tunnel_type type, - u16 *port); -enum ice_status -ice_create_tunnel(struct ice_hw *hw, enum ice_tunnel_type type, u16 port); -enum ice_status ice_destroy_tunnel(struct ice_hw *hw, u16 port, bool all); -bool ice_tunnel_port_in_use(struct ice_hw *hw, u16 port, u16 *index); +ice_get_open_tunnel_port(struct ice_hw *hw, u16 *port); +int ice_udp_tunnel_set_port(struct net_device *netdev, unsigned int table, + unsigned int idx, struct udp_tunnel_info *ti); +int ice_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table, + unsigned int idx, struct udp_tunnel_info *ti); enum ice_status ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], diff --git a/drivers/net/ethernet/intel/ice/ice_flex_type.h b/drivers/net/ethernet/intel/ice/ice_flex_type.h index c1c99a267a98..24063c1351b2 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_type.h +++ b/drivers/net/ethernet/intel/ice/ice_flex_type.h @@ -298,6 +298,7 @@ struct ice_pkg_enum { enum ice_tunnel_type { TNL_VXLAN = 0, TNL_GENEVE, + __TNL_TYPE_CNT, TNL_LAST = 0xFF, TNL_ALL = 0xFF, }; @@ -311,11 +312,8 @@ struct ice_tunnel_entry { enum ice_tunnel_type type; u16 boost_addr; u16 port; - u16 ref; struct ice_boost_tcam_entry *boost_entry; u8 valid; - u8 in_use; - u8 marked; }; #define ICE_TUNNEL_MAX_ENTRIES 16 @@ -323,6 +321,7 @@ struct ice_tunnel_entry { struct ice_tunnel_table { struct ice_tunnel_entry tbl[ICE_TUNNEL_MAX_ENTRIES]; u16 count; + u16 valid_count[__TNL_TYPE_CNT]; }; struct ice_pkg_es { diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 2297ee7dba26..1d74466ce77f 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -486,7 +486,6 @@ static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) struct ice_hw *hw = &pf->hw; dev_dbg(dev, "reset_type 0x%x requested\n", reset_type); - WARN_ON(in_interrupt()); ice_prepare_for_reset(pf); @@ -2873,6 +2872,7 @@ static void ice_set_ops(struct net_device *netdev) } netdev->netdev_ops = &ice_netdev_ops; + netdev->udp_tunnel_nic_info = &pf->hw.udp_tunnel_nic; ice_set_ethtool_ops(netdev); } @@ -3978,7 +3978,7 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) struct device *dev = &pdev->dev; struct ice_pf *pf; struct ice_hw *hw; - int err; + int i, err; /* this driver uses devres, see * Documentation/driver-api/driver-model/devres.rst @@ -4073,11 +4073,37 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) ice_devlink_init_regions(pf); + pf->hw.udp_tunnel_nic.set_port = ice_udp_tunnel_set_port; + pf->hw.udp_tunnel_nic.unset_port = ice_udp_tunnel_unset_port; + pf->hw.udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP; + pf->hw.udp_tunnel_nic.shared = &pf->hw.udp_tunnel_shared; + i = 0; + if (pf->hw.tnl.valid_count[TNL_VXLAN]) { + pf->hw.udp_tunnel_nic.tables[i].n_entries = + pf->hw.tnl.valid_count[TNL_VXLAN]; + pf->hw.udp_tunnel_nic.tables[i].tunnel_types = + UDP_TUNNEL_TYPE_VXLAN; + i++; + } + if (pf->hw.tnl.valid_count[TNL_GENEVE]) { + pf->hw.udp_tunnel_nic.tables[i].n_entries = + pf->hw.tnl.valid_count[TNL_GENEVE]; + pf->hw.udp_tunnel_nic.tables[i].tunnel_types = + UDP_TUNNEL_TYPE_GENEVE; + i++; + } + pf->num_alloc_vsi = hw->func_caps.guar_num_vsi; if (!pf->num_alloc_vsi) { err = -EIO; goto err_init_pf_unroll; } + if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { + dev_warn(&pf->pdev->dev, + "limiting the VSI count due to UDP tunnel limitation %d > %d\n", + pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); + pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; + } pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi), GFP_KERNEL); @@ -6575,70 +6601,6 @@ static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) } /** - * ice_udp_tunnel_add - Get notifications about UDP tunnel ports that come up - * @netdev: This physical port's netdev - * @ti: Tunnel endpoint information - */ -static void -ice_udp_tunnel_add(struct net_device *netdev, struct udp_tunnel_info *ti) -{ - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_vsi *vsi = np->vsi; - struct ice_pf *pf = vsi->back; - enum ice_tunnel_type tnl_type; - u16 port = ntohs(ti->port); - enum ice_status status; - - switch (ti->type) { - case UDP_TUNNEL_TYPE_VXLAN: - tnl_type = TNL_VXLAN; - break; - case UDP_TUNNEL_TYPE_GENEVE: - tnl_type = TNL_GENEVE; - break; - default: - netdev_err(netdev, "Unknown tunnel type\n"); - return; - } - - status = ice_create_tunnel(&pf->hw, tnl_type, port); - if (status == ICE_ERR_OUT_OF_RANGE) - netdev_info(netdev, "Max tunneled UDP ports reached, port %d not added\n", - port); - else if (status) - netdev_err(netdev, "Error adding UDP tunnel - %s\n", - ice_stat_str(status)); -} - -/** - * ice_udp_tunnel_del - Get notifications about UDP tunnel ports that go away - * @netdev: This physical port's netdev - * @ti: Tunnel endpoint information - */ -static void -ice_udp_tunnel_del(struct net_device *netdev, struct udp_tunnel_info *ti) -{ - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_vsi *vsi = np->vsi; - struct ice_pf *pf = vsi->back; - u16 port = ntohs(ti->port); - enum ice_status status; - bool retval; - - retval = ice_tunnel_port_in_use(&pf->hw, port, NULL); - if (!retval) { - netdev_info(netdev, "port %d not found in UDP tunnels list\n", - port); - return; - } - - status = ice_destroy_tunnel(&pf->hw, port, false); - if (status) - netdev_err(netdev, "error deleting port %d from UDP tunnels list\n", - port); -} - -/** * ice_open - Called when a network interface becomes active * @netdev: network interface device structure * @@ -6830,6 +6792,6 @@ static const struct net_device_ops ice_netdev_ops = { .ndo_bpf = ice_xdp, .ndo_xdp_xmit = ice_xdp_xmit, .ndo_xsk_wakeup = ice_xsk_wakeup, - .ndo_udp_tunnel_add = ice_udp_tunnel_add, - .ndo_udp_tunnel_del = ice_udp_tunnel_del, + .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, + .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, }; diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index 4cdccfadf274..2226a291a394 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -676,6 +676,9 @@ struct ice_hw { struct mutex tnl_lock; struct ice_tunnel_table tnl; + struct udp_tunnel_nic_shared udp_tunnel_shared; + struct udp_tunnel_nic_info udp_tunnel_nic; + /* HW block tables */ struct ice_blk_info blk[ICE_BLK_COUNT]; struct mutex fl_profs_locks[ICE_BLK_COUNT]; /* lock fltr profiles */ diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h index 2f015b60a995..0286d2fceee4 100644 --- a/drivers/net/ethernet/intel/igb/igb.h +++ b/drivers/net/ethernet/intel/igb/igb.h @@ -19,6 +19,8 @@ #include <linux/pci.h> #include <linux/mdio.h> +#include <net/xdp.h> + struct igb_adapter; #define E1000_PCS_CFG_IGN_SD 1 @@ -79,6 +81,12 @@ struct igb_adapter; #define IGB_I210_RX_LATENCY_100 2213 #define IGB_I210_RX_LATENCY_1000 448 +/* XDP */ +#define IGB_XDP_PASS 0 +#define IGB_XDP_CONSUMED BIT(0) +#define IGB_XDP_TX BIT(1) +#define IGB_XDP_REDIR BIT(2) + struct vf_data_storage { unsigned char vf_mac_addresses[ETH_ALEN]; u16 vf_mc_hashes[IGB_MAX_VF_MC_ENTRIES]; @@ -132,17 +140,62 @@ struct vf_mac_filter { /* Supported Rx Buffer Sizes */ #define IGB_RXBUFFER_256 256 +#define IGB_RXBUFFER_1536 1536 #define IGB_RXBUFFER_2048 2048 #define IGB_RXBUFFER_3072 3072 #define IGB_RX_HDR_LEN IGB_RXBUFFER_256 #define IGB_TS_HDR_LEN 16 -#define IGB_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) +/* Attempt to maximize the headroom available for incoming frames. We + * use a 2K buffer for receives and need 1536/1534 to store the data for + * the frame. This leaves us with 512 bytes of room. From that we need + * to deduct the space needed for the shared info and the padding needed + * to IP align the frame. + * + * Note: For cache line sizes 256 or larger this value is going to end + * up negative. In these cases we should fall back to the 3K + * buffers. + */ #if (PAGE_SIZE < 8192) -#define IGB_MAX_FRAME_BUILD_SKB \ - (SKB_WITH_OVERHEAD(IGB_RXBUFFER_2048) - IGB_SKB_PAD - IGB_TS_HDR_LEN) +#define IGB_MAX_FRAME_BUILD_SKB (IGB_RXBUFFER_1536 - NET_IP_ALIGN) +#define IGB_2K_TOO_SMALL_WITH_PADDING \ +((NET_SKB_PAD + IGB_TS_HDR_LEN + IGB_RXBUFFER_1536) > SKB_WITH_OVERHEAD(IGB_RXBUFFER_2048)) + +static inline int igb_compute_pad(int rx_buf_len) +{ + int page_size, pad_size; + + page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2); + pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len; + + return pad_size; +} + +static inline int igb_skb_pad(void) +{ + int rx_buf_len; + + /* If a 2K buffer cannot handle a standard Ethernet frame then + * optimize padding for a 3K buffer instead of a 1.5K buffer. + * + * For a 3K buffer we need to add enough padding to allow for + * tailroom due to NET_IP_ALIGN possibly shifting us out of + * cache-line alignment. + */ + if (IGB_2K_TOO_SMALL_WITH_PADDING) + rx_buf_len = IGB_RXBUFFER_3072 + SKB_DATA_ALIGN(NET_IP_ALIGN); + else + rx_buf_len = IGB_RXBUFFER_1536; + + /* if needed make room for NET_IP_ALIGN */ + rx_buf_len -= NET_IP_ALIGN; + + return igb_compute_pad(rx_buf_len); +} + +#define IGB_SKB_PAD igb_skb_pad() #else -#define IGB_MAX_FRAME_BUILD_SKB (IGB_RXBUFFER_2048 - IGB_TS_HDR_LEN) +#define IGB_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) #endif /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -194,13 +247,22 @@ enum igb_tx_flags { #define IGB_SFF_ADDRESSING_MODE 0x4 #define IGB_SFF_8472_UNSUP 0x00 +enum igb_tx_buf_type { + IGB_TYPE_SKB = 0, + IGB_TYPE_XDP, +}; + /* wrapper around a pointer to a socket buffer, * so a DMA handle can be stored along with the buffer */ struct igb_tx_buffer { union e1000_adv_tx_desc *next_to_watch; unsigned long time_stamp; - struct sk_buff *skb; + enum igb_tx_buf_type type; + union { + struct sk_buff *skb; + struct xdp_frame *xdpf; + }; unsigned int bytecount; u16 gso_segs; __be16 protocol; @@ -248,6 +310,7 @@ struct igb_ring_container { struct igb_ring { struct igb_q_vector *q_vector; /* backlink to q_vector */ struct net_device *netdev; /* back pointer to net_device */ + struct bpf_prog *xdp_prog; struct device *dev; /* device pointer for dma mapping */ union { /* array of buffer info structs */ struct igb_tx_buffer *tx_buffer_info; @@ -288,6 +351,7 @@ struct igb_ring { struct u64_stats_sync rx_syncp; }; }; + struct xdp_rxq_info xdp_rxq; } ____cacheline_internodealigned_in_smp; struct igb_q_vector { @@ -339,7 +403,7 @@ static inline unsigned int igb_rx_bufsz(struct igb_ring *ring) return IGB_RXBUFFER_3072; if (ring_uses_build_skb(ring)) - return IGB_MAX_FRAME_BUILD_SKB + IGB_TS_HDR_LEN; + return IGB_MAX_FRAME_BUILD_SKB; #endif return IGB_RXBUFFER_2048; } @@ -467,6 +531,7 @@ struct igb_adapter { unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; struct net_device *netdev; + struct bpf_prog *xdp_prog; unsigned long state; unsigned int flags; @@ -643,6 +708,9 @@ enum igb_boards { extern char igb_driver_name[]; +int igb_xmit_xdp_ring(struct igb_adapter *adapter, + struct igb_ring *ring, + struct xdp_frame *xdpf); int igb_open(struct net_device *netdev); int igb_close(struct net_device *netdev); int igb_up(struct igb_adapter *); diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 6e8231c1ddf0..28baf203459a 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -961,6 +961,10 @@ static int igb_set_ringparam(struct net_device *netdev, memcpy(&temp_ring[i], adapter->rx_ring[i], sizeof(struct igb_ring)); + /* Clear copied XDP RX-queue info */ + memset(&temp_ring[i].xdp_rxq, 0, + sizeof(temp_ring[i].xdp_rxq)); + temp_ring[i].count = new_rx_count; err = igb_setup_rx_resources(&temp_ring[i]); if (err) { diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 44157fcd3cf7..5fc2c381da55 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -30,6 +30,8 @@ #include <linux/if_ether.h> #include <linux/aer.h> #include <linux/prefetch.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> #include <linux/pm_runtime.h> #include <linux/etherdevice.h> #ifdef CONFIG_IGB_DCA @@ -2219,7 +2221,6 @@ void igb_down(struct igb_adapter *adapter) void igb_reinit_locked(struct igb_adapter *adapter) { - WARN_ON(in_interrupt()); while (test_and_set_bit(__IGB_RESETTING, &adapter->state)) usleep_range(1000, 2000); igb_down(adapter); @@ -2823,6 +2824,147 @@ static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type, } } +static int igb_xdp_setup(struct net_device *dev, struct bpf_prog *prog) +{ + int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; + struct igb_adapter *adapter = netdev_priv(dev); + bool running = netif_running(dev); + struct bpf_prog *old_prog; + bool need_reset; + + /* verify igb ring attributes are sufficient for XDP */ + for (i = 0; i < adapter->num_rx_queues; i++) { + struct igb_ring *ring = adapter->rx_ring[i]; + + if (frame_size > igb_rx_bufsz(ring)) + return -EINVAL; + } + + old_prog = xchg(&adapter->xdp_prog, prog); + need_reset = (!!prog != !!old_prog); + + /* device is up and bpf is added/removed, must setup the RX queues */ + if (need_reset && running) { + igb_close(dev); + } else { + for (i = 0; i < adapter->num_rx_queues; i++) + (void)xchg(&adapter->rx_ring[i]->xdp_prog, + adapter->xdp_prog); + } + + if (old_prog) + bpf_prog_put(old_prog); + + /* bpf is just replaced, RXQ and MTU are already setup */ + if (!need_reset) + return 0; + + if (running) + igb_open(dev); + + return 0; +} + +static int igb_xdp(struct net_device *dev, struct netdev_bpf *xdp) +{ + switch (xdp->command) { + case XDP_SETUP_PROG: + return igb_xdp_setup(dev, xdp->prog); + default: + return -EINVAL; + } +} + +static void igb_xdp_ring_update_tail(struct igb_ring *ring) +{ + /* Force memory writes to complete before letting h/w know there + * are new descriptors to fetch. + */ + wmb(); + writel(ring->next_to_use, ring->tail); +} + +static struct igb_ring *igb_xdp_tx_queue_mapping(struct igb_adapter *adapter) +{ + unsigned int r_idx = smp_processor_id(); + + if (r_idx >= adapter->num_tx_queues) + r_idx = r_idx % adapter->num_tx_queues; + + return adapter->tx_ring[r_idx]; +} + +static int igb_xdp_xmit_back(struct igb_adapter *adapter, struct xdp_buff *xdp) +{ + struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); + int cpu = smp_processor_id(); + struct igb_ring *tx_ring; + struct netdev_queue *nq; + u32 ret; + + if (unlikely(!xdpf)) + return IGB_XDP_CONSUMED; + + /* During program transitions its possible adapter->xdp_prog is assigned + * but ring has not been configured yet. In this case simply abort xmit. + */ + tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL; + if (unlikely(!tx_ring)) + return -ENXIO; + + nq = txring_txq(tx_ring); + __netif_tx_lock(nq, cpu); + ret = igb_xmit_xdp_ring(adapter, tx_ring, xdpf); + __netif_tx_unlock(nq); + + return ret; +} + +static int igb_xdp_xmit(struct net_device *dev, int n, + struct xdp_frame **frames, u32 flags) +{ + struct igb_adapter *adapter = netdev_priv(dev); + int cpu = smp_processor_id(); + struct igb_ring *tx_ring; + struct netdev_queue *nq; + int drops = 0; + int i; + + if (unlikely(test_bit(__IGB_DOWN, &adapter->state))) + return -ENETDOWN; + + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) + return -EINVAL; + + /* During program transitions its possible adapter->xdp_prog is assigned + * but ring has not been configured yet. In this case simply abort xmit. + */ + tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL; + if (unlikely(!tx_ring)) + return -ENXIO; + + nq = txring_txq(tx_ring); + __netif_tx_lock(nq, cpu); + + for (i = 0; i < n; i++) { + struct xdp_frame *xdpf = frames[i]; + int err; + + err = igb_xmit_xdp_ring(adapter, tx_ring, xdpf); + if (err != IGB_XDP_TX) { + xdp_return_frame_rx_napi(xdpf); + drops++; + } + } + + __netif_tx_unlock(nq); + + if (unlikely(flags & XDP_XMIT_FLUSH)) + igb_xdp_ring_update_tail(tx_ring); + + return n - drops; +} + static const struct net_device_ops igb_netdev_ops = { .ndo_open = igb_open, .ndo_stop = igb_close, @@ -2847,6 +2989,8 @@ static const struct net_device_ops igb_netdev_ops = { .ndo_fdb_add = igb_ndo_fdb_add, .ndo_features_check = igb_features_check, .ndo_setup_tc = igb_setup_tc, + .ndo_bpf = igb_xdp, + .ndo_xdp_xmit = igb_xdp_xmit, }; /** @@ -3387,7 +3531,9 @@ static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent) "Width x1" : "unknown"), netdev->dev_addr); } - if ((hw->mac.type >= e1000_i210 || + if ((hw->mac.type == e1000_82576 && + rd32(E1000_EECD) & E1000_EECD_PRES) || + (hw->mac.type >= e1000_i210 || igb_get_flash_presence_i210(hw))) { ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH); @@ -4179,6 +4325,7 @@ static void igb_configure_tx(struct igb_adapter *adapter) **/ int igb_setup_rx_resources(struct igb_ring *rx_ring) { + struct igb_adapter *adapter = netdev_priv(rx_ring->netdev); struct device *dev = rx_ring->dev; int size; @@ -4201,6 +4348,13 @@ int igb_setup_rx_resources(struct igb_ring *rx_ring) rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; + rx_ring->xdp_prog = adapter->xdp_prog; + + /* XDP RX-queue info */ + if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index) < 0) + goto err; + return 0; err: @@ -4505,6 +4659,10 @@ void igb_configure_rx_ring(struct igb_adapter *adapter, int reg_idx = ring->reg_idx; u32 rxdctl = 0; + xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); + WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, + MEM_TYPE_PAGE_SHARED, NULL)); + /* disable the queue */ wr32(E1000_RXDCTL(reg_idx), 0); @@ -4709,6 +4867,8 @@ void igb_free_rx_resources(struct igb_ring *rx_ring) { igb_clean_rx_ring(rx_ring); + rx_ring->xdp_prog = NULL; + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); vfree(rx_ring->rx_buffer_info); rx_ring->rx_buffer_info = NULL; @@ -6078,6 +6238,80 @@ dma_error: return -1; } +int igb_xmit_xdp_ring(struct igb_adapter *adapter, + struct igb_ring *tx_ring, + struct xdp_frame *xdpf) +{ + union e1000_adv_tx_desc *tx_desc; + u32 len, cmd_type, olinfo_status; + struct igb_tx_buffer *tx_buffer; + dma_addr_t dma; + u16 i; + + len = xdpf->len; + + if (unlikely(!igb_desc_unused(tx_ring))) + return IGB_XDP_CONSUMED; + + dma = dma_map_single(tx_ring->dev, xdpf->data, len, DMA_TO_DEVICE); + if (dma_mapping_error(tx_ring->dev, dma)) + return IGB_XDP_CONSUMED; + + /* record the location of the first descriptor for this packet */ + tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; + tx_buffer->bytecount = len; + tx_buffer->gso_segs = 1; + tx_buffer->protocol = 0; + + i = tx_ring->next_to_use; + tx_desc = IGB_TX_DESC(tx_ring, i); + + dma_unmap_len_set(tx_buffer, len, len); + dma_unmap_addr_set(tx_buffer, dma, dma); + tx_buffer->type = IGB_TYPE_XDP; + tx_buffer->xdpf = xdpf; + + tx_desc->read.buffer_addr = cpu_to_le64(dma); + + /* put descriptor type bits */ + cmd_type = E1000_ADVTXD_DTYP_DATA | + E1000_ADVTXD_DCMD_DEXT | + E1000_ADVTXD_DCMD_IFCS; + cmd_type |= len | IGB_TXD_DCMD; + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + + olinfo_status = cpu_to_le32(len << E1000_ADVTXD_PAYLEN_SHIFT); + /* 82575 requires a unique index per ring */ + if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) + olinfo_status |= tx_ring->reg_idx << 4; + + tx_desc->read.olinfo_status = olinfo_status; + + netdev_tx_sent_queue(txring_txq(tx_ring), tx_buffer->bytecount); + + /* set the timestamp */ + tx_buffer->time_stamp = jiffies; + + /* Avoid any potential race with xdp_xmit and cleanup */ + smp_wmb(); + + /* set next_to_watch value indicating a packet is present */ + i++; + if (i == tx_ring->count) + i = 0; + + tx_buffer->next_to_watch = tx_desc; + tx_ring->next_to_use = i; + + /* Make sure there is space in the ring for the next send. */ + igb_maybe_stop_tx(tx_ring, DESC_NEEDED); + + if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) + writel(i, tx_ring->tail); + + return IGB_XDP_TX; +} + netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb, struct igb_ring *tx_ring) { @@ -6106,6 +6340,7 @@ netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb, /* record the location of the first descriptor for this packet */ first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; + first->type = IGB_TYPE_SKB; first->skb = skb; first->bytecount = skb->len; first->gso_segs = 1; @@ -6258,6 +6493,19 @@ static int igb_change_mtu(struct net_device *netdev, int new_mtu) struct igb_adapter *adapter = netdev_priv(netdev); int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; + if (adapter->xdp_prog) { + int i; + + for (i = 0; i < adapter->num_rx_queues; i++) { + struct igb_ring *ring = adapter->rx_ring[i]; + + if (max_frame > igb_rx_bufsz(ring)) { + netdev_warn(adapter->netdev, "Requested MTU size is not supported with XDP\n"); + return -EINVAL; + } + } + } + /* adjust max frame to be at least the size of a standard frame */ if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; @@ -7811,7 +8059,10 @@ static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget) total_packets += tx_buffer->gso_segs; /* free the skb */ - napi_consume_skb(tx_buffer->skb, napi_budget); + if (tx_buffer->type == IGB_TYPE_SKB) + napi_consume_skb(tx_buffer->skb, napi_budget); + else + xdp_return_frame(tx_buffer->xdpf); /* unmap skb header data */ dma_unmap_single(tx_ring->dev, @@ -7995,8 +8246,8 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer) * the pagecnt_bias and page count so that we fully restock the * number of references the driver holds. */ - if (unlikely(!pagecnt_bias)) { - page_ref_add(page, USHRT_MAX); + if (unlikely(pagecnt_bias == 1)) { + page_ref_add(page, USHRT_MAX - 1); rx_buffer->pagecnt_bias = USHRT_MAX; } @@ -8035,20 +8286,21 @@ static void igb_add_rx_frag(struct igb_ring *rx_ring, static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring, struct igb_rx_buffer *rx_buffer, - union e1000_adv_rx_desc *rx_desc, - unsigned int size) + struct xdp_buff *xdp, + union e1000_adv_rx_desc *rx_desc) { - void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; #if (PAGE_SIZE < 8192) unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; #else - unsigned int truesize = SKB_DATA_ALIGN(size); + unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - + xdp->data_hard_start); #endif + unsigned int size = xdp->data_end - xdp->data; unsigned int headlen; struct sk_buff *skb; /* prefetch first cache line of first page */ - net_prefetch(va); + net_prefetch(xdp->data); /* allocate a skb to store the frags */ skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN); @@ -8056,24 +8308,24 @@ static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring, return NULL; if (unlikely(igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))) { - igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); - va += IGB_TS_HDR_LEN; + igb_ptp_rx_pktstamp(rx_ring->q_vector, xdp->data, skb); + xdp->data += IGB_TS_HDR_LEN; size -= IGB_TS_HDR_LEN; } /* Determine available headroom for copy */ headlen = size; if (headlen > IGB_RX_HDR_LEN) - headlen = eth_get_headlen(skb->dev, va, IGB_RX_HDR_LEN); + headlen = eth_get_headlen(skb->dev, xdp->data, IGB_RX_HDR_LEN); /* align pull length to size of long to optimize memcpy performance */ - memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); + memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen, sizeof(long))); /* update all of the pointers */ size -= headlen; if (size) { skb_add_rx_frag(skb, 0, rx_buffer->page, - (va + headlen) - page_address(rx_buffer->page), + (xdp->data + headlen) - page_address(rx_buffer->page), size, truesize); #if (PAGE_SIZE < 8192) rx_buffer->page_offset ^= truesize; @@ -8089,29 +8341,29 @@ static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring, static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring, struct igb_rx_buffer *rx_buffer, - union e1000_adv_rx_desc *rx_desc, - unsigned int size) + struct xdp_buff *xdp, + union e1000_adv_rx_desc *rx_desc) { - void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; #if (PAGE_SIZE < 8192) unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; #else unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + - SKB_DATA_ALIGN(IGB_SKB_PAD + size); + SKB_DATA_ALIGN(xdp->data_end - + xdp->data_hard_start); #endif struct sk_buff *skb; /* prefetch first cache line of first page */ - net_prefetch(va); + net_prefetch(xdp->data_meta); /* build an skb around the page buffer */ - skb = build_skb(va - IGB_SKB_PAD, truesize); + skb = build_skb(xdp->data_hard_start, truesize); if (unlikely(!skb)) return NULL; /* update pointers within the skb to store the data */ - skb_reserve(skb, IGB_SKB_PAD); - __skb_put(skb, size); + skb_reserve(skb, xdp->data - xdp->data_hard_start); + __skb_put(skb, xdp->data_end - xdp->data); /* pull timestamp out of packet data */ if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) { @@ -8129,6 +8381,79 @@ static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring, return skb; } +static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter, + struct igb_ring *rx_ring, + struct xdp_buff *xdp) +{ + int err, result = IGB_XDP_PASS; + struct bpf_prog *xdp_prog; + u32 act; + + rcu_read_lock(); + xdp_prog = READ_ONCE(rx_ring->xdp_prog); + + if (!xdp_prog) + goto xdp_out; + + prefetchw(xdp->data_hard_start); /* xdp_frame write */ + + act = bpf_prog_run_xdp(xdp_prog, xdp); + switch (act) { + case XDP_PASS: + break; + case XDP_TX: + result = igb_xdp_xmit_back(adapter, xdp); + break; + case XDP_REDIRECT: + err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog); + if (!err) + result = IGB_XDP_REDIR; + else + result = IGB_XDP_CONSUMED; + break; + default: + bpf_warn_invalid_xdp_action(act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(rx_ring->netdev, xdp_prog, act); + fallthrough; + case XDP_DROP: + result = IGB_XDP_CONSUMED; + break; + } +xdp_out: + rcu_read_unlock(); + return ERR_PTR(-result); +} + +static unsigned int igb_rx_frame_truesize(struct igb_ring *rx_ring, + unsigned int size) +{ + unsigned int truesize; + +#if (PAGE_SIZE < 8192) + truesize = igb_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ +#else + truesize = ring_uses_build_skb(rx_ring) ? + SKB_DATA_ALIGN(IGB_SKB_PAD + size) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : + SKB_DATA_ALIGN(size); +#endif + return truesize; +} + +static void igb_rx_buffer_flip(struct igb_ring *rx_ring, + struct igb_rx_buffer *rx_buffer, + unsigned int size) +{ + unsigned int truesize = igb_rx_frame_truesize(rx_ring, size); +#if (PAGE_SIZE < 8192) + rx_buffer->page_offset ^= truesize; +#else + rx_buffer->page_offset += truesize; +#endif +} + static inline void igb_rx_checksum(struct igb_ring *ring, union e1000_adv_rx_desc *rx_desc, struct sk_buff *skb) @@ -8224,6 +8549,10 @@ static bool igb_cleanup_headers(struct igb_ring *rx_ring, union e1000_adv_rx_desc *rx_desc, struct sk_buff *skb) { + /* XDP packets use error pointer so abort at this point */ + if (IS_ERR(skb)) + return true; + if (unlikely((igb_test_staterr(rx_desc, E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) { struct net_device *netdev = rx_ring->netdev; @@ -8282,6 +8611,11 @@ static void igb_process_skb_fields(struct igb_ring *rx_ring, skb->protocol = eth_type_trans(skb, rx_ring->netdev); } +static unsigned int igb_rx_offset(struct igb_ring *rx_ring) +{ + return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0; +} + static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring, const unsigned int size) { @@ -8325,10 +8659,20 @@ static void igb_put_rx_buffer(struct igb_ring *rx_ring, static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) { + struct igb_adapter *adapter = q_vector->adapter; struct igb_ring *rx_ring = q_vector->rx.ring; struct sk_buff *skb = rx_ring->skb; unsigned int total_bytes = 0, total_packets = 0; u16 cleaned_count = igb_desc_unused(rx_ring); + unsigned int xdp_xmit = 0; + struct xdp_buff xdp; + + xdp.rxq = &rx_ring->xdp_rxq; + + /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ +#if (PAGE_SIZE < 8192) + xdp.frame_sz = igb_rx_frame_truesize(rx_ring, 0); +#endif while (likely(total_packets < budget)) { union e1000_adv_rx_desc *rx_desc; @@ -8355,13 +8699,38 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) rx_buffer = igb_get_rx_buffer(rx_ring, size); /* retrieve a buffer from the ring */ - if (skb) + if (!skb) { + xdp.data = page_address(rx_buffer->page) + + rx_buffer->page_offset; + xdp.data_meta = xdp.data; + xdp.data_hard_start = xdp.data - + igb_rx_offset(rx_ring); + xdp.data_end = xdp.data + size; +#if (PAGE_SIZE > 4096) + /* At larger PAGE_SIZE, frame_sz depend on len size */ + xdp.frame_sz = igb_rx_frame_truesize(rx_ring, size); +#endif + skb = igb_run_xdp(adapter, rx_ring, &xdp); + } + + if (IS_ERR(skb)) { + unsigned int xdp_res = -PTR_ERR(skb); + + if (xdp_res & (IGB_XDP_TX | IGB_XDP_REDIR)) { + xdp_xmit |= xdp_res; + igb_rx_buffer_flip(rx_ring, rx_buffer, size); + } else { + rx_buffer->pagecnt_bias++; + } + total_packets++; + total_bytes += size; + } else if (skb) igb_add_rx_frag(rx_ring, rx_buffer, skb, size); else if (ring_uses_build_skb(rx_ring)) - skb = igb_build_skb(rx_ring, rx_buffer, rx_desc, size); + skb = igb_build_skb(rx_ring, rx_buffer, &xdp, rx_desc); else skb = igb_construct_skb(rx_ring, rx_buffer, - rx_desc, size); + &xdp, rx_desc); /* exit if we failed to retrieve a buffer */ if (!skb) { @@ -8401,6 +8770,15 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) /* place incomplete frames back on ring for completion */ rx_ring->skb = skb; + if (xdp_xmit & IGB_XDP_REDIR) + xdp_do_flush_map(); + + if (xdp_xmit & IGB_XDP_TX) { + struct igb_ring *tx_ring = igb_xdp_tx_queue_mapping(adapter); + + igb_xdp_ring_update_tail(tx_ring); + } + u64_stats_update_begin(&rx_ring->rx_syncp); rx_ring->rx_stats.packets += total_packets; rx_ring->rx_stats.bytes += total_bytes; @@ -8414,11 +8792,6 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) return total_packets; } -static inline unsigned int igb_rx_offset(struct igb_ring *rx_ring) -{ - return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0; -} - static bool igb_alloc_mapped_page(struct igb_ring *rx_ring, struct igb_rx_buffer *bi) { @@ -8455,7 +8828,8 @@ static bool igb_alloc_mapped_page(struct igb_ring *rx_ring, bi->dma = dma; bi->page = page; bi->page_offset = igb_rx_offset(rx_ring); - bi->pagecnt_bias = 1; + page_ref_add(page, USHRT_MAX - 1); + bi->pagecnt_bias = USHRT_MAX; return true; } diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index 2d566f3c827b..35baae900c1f 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -215,6 +215,8 @@ struct igc_adapter { spinlock_t tmreg_lock; struct cyclecounter cc; struct timecounter tc; + struct timespec64 prev_ptp_time; /* Pre-reset PTP clock */ + ktime_t ptp_reset_start; /* Reset time in clock mono */ }; void igc_up(struct igc_adapter *adapter); @@ -548,6 +550,7 @@ void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va, int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr); int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr); void igc_ptp_tx_hang(struct igc_adapter *adapter); +void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts); #define igc_rx_pg_size(_ring) (PAGE_SIZE << igc_rx_pg_order(_ring)) diff --git a/drivers/net/ethernet/intel/igc/igc_base.c b/drivers/net/ethernet/intel/igc/igc_base.c index cc5a6cf531c7..fd37d2c203af 100644 --- a/drivers/net/ethernet/intel/igc/igc_base.c +++ b/drivers/net/ethernet/intel/igc/igc_base.c @@ -215,6 +215,11 @@ static s32 igc_get_invariants_base(struct igc_hw *hw) case IGC_DEV_ID_I225_K2: case IGC_DEV_ID_I225_LMVP: case IGC_DEV_ID_I225_IT: + case IGC_DEV_ID_I226_LM: + case IGC_DEV_ID_I226_V: + case IGC_DEV_ID_I226_IT: + case IGC_DEV_ID_I221_V: + case IGC_DEV_ID_I226_BLANK_NVM: case IGC_DEV_ID_I225_BLANK_NVM: mac->type = igc_i225; break; diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index f1f464967f87..32f5fd684139 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -324,22 +324,10 @@ /* Advanced Receive Descriptor bit definitions */ #define IGC_RXDADV_STAT_TSIP 0x08000 /* timestamp in packet */ -#define IGC_RXDEXT_STATERR_CE 0x01000000 -#define IGC_RXDEXT_STATERR_SE 0x02000000 -#define IGC_RXDEXT_STATERR_SEQ 0x04000000 -#define IGC_RXDEXT_STATERR_CXE 0x10000000 -#define IGC_RXDEXT_STATERR_TCPE 0x20000000 +#define IGC_RXDEXT_STATERR_L4E 0x20000000 #define IGC_RXDEXT_STATERR_IPE 0x40000000 #define IGC_RXDEXT_STATERR_RXE 0x80000000 -/* Same mask, but for extended and packet split descriptors */ -#define IGC_RXDEXT_ERR_FRAME_ERR_MASK ( \ - IGC_RXDEXT_STATERR_CE | \ - IGC_RXDEXT_STATERR_SE | \ - IGC_RXDEXT_STATERR_SEQ | \ - IGC_RXDEXT_STATERR_CXE | \ - IGC_RXDEXT_STATERR_RXE) - #define IGC_MRQC_RSS_FIELD_IPV4_TCP 0x00010000 #define IGC_MRQC_RSS_FIELD_IPV4 0x00020000 #define IGC_MRQC_RSS_FIELD_IPV6_TCP_EX 0x00040000 @@ -409,7 +397,7 @@ #define IGC_IMIREXT_SIZE_BP 0x00001000 /* Packet size bypass */ /* Time Sync Transmit Control bit definitions */ -#define IGC_TSYNCTXCTL_VALID 0x00000001 /* Tx timestamp valid */ +#define IGC_TSYNCTXCTL_TXTT_0 0x00000001 /* Tx timestamp reg 0 valid */ #define IGC_TSYNCTXCTL_ENABLED 0x00000010 /* enable Tx timestamping */ #define IGC_TSYNCTXCTL_MAX_ALLOWED_DLY_MASK 0x0000F000 /* max delay */ #define IGC_TSYNCTXCTL_SYNC_COMP_ERR 0x20000000 /* sync err */ diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index 44410c2265d6..61d331ce38cd 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -321,6 +321,9 @@ static void igc_ethtool_get_regs(struct net_device *netdev, for (i = 0; i < 8; i++) regs_buff[205 + i] = rd32(IGC_ETQF(i)); + + regs_buff[213] = adapter->stats.tlpic; + regs_buff[214] = adapter->stats.rlpic; } static void igc_ethtool_get_wol(struct net_device *netdev, diff --git a/drivers/net/ethernet/intel/igc/igc_hw.h b/drivers/net/ethernet/intel/igc/igc_hw.h index b9fe51b91c47..55dae7c4703f 100644 --- a/drivers/net/ethernet/intel/igc/igc_hw.h +++ b/drivers/net/ethernet/intel/igc/igc_hw.h @@ -24,6 +24,11 @@ #define IGC_DEV_ID_I225_K2 0x3101 #define IGC_DEV_ID_I225_LMVP 0x5502 #define IGC_DEV_ID_I225_IT 0x0D9F +#define IGC_DEV_ID_I226_LM 0x125B +#define IGC_DEV_ID_I226_V 0x125C +#define IGC_DEV_ID_I226_IT 0x125D +#define IGC_DEV_ID_I221_V 0x125E +#define IGC_DEV_ID_I226_BLANK_NVM 0x125F #define IGC_DEV_ID_I225_BLANK_NVM 0x15FD /* Function pointers for the MAC. */ @@ -125,9 +130,6 @@ struct igc_nvm_info { struct igc_nvm_operations ops; enum igc_nvm_type type; - u32 flash_bank_size; - u32 flash_base_addr; - u16 word_size; u16 delay_usec; u16 address_bits; @@ -153,7 +155,6 @@ struct igc_phy_info { u8 mdix; bool is_mdix; - bool reset_disable; bool speed_downgraded; bool autoneg_wait_to_complete; }; @@ -239,6 +240,8 @@ struct igc_hw_stats { u64 prc511; u64 prc1023; u64 prc1522; + u64 tlpic; + u64 rlpic; u64 gprc; u64 bprc; u64 mprc; diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 3183150c7995..9112dff075cf 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -47,6 +47,11 @@ static const struct pci_device_id igc_pci_tbl[] = { { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, /* required last entry */ {0, } @@ -1428,7 +1433,7 @@ static void igc_rx_checksum(struct igc_ring *ring, /* TCP/UDP checksum error bit is set */ if (igc_test_staterr(rx_desc, - IGC_RXDEXT_STATERR_TCPE | + IGC_RXDEXT_STATERR_L4E | IGC_RXDEXT_STATERR_IPE)) { /* work around errata with sctp packets where the TCPE aka * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) @@ -1737,8 +1742,7 @@ static bool igc_cleanup_headers(struct igc_ring *rx_ring, union igc_adv_rx_desc *rx_desc, struct sk_buff *skb) { - if (unlikely((igc_test_staterr(rx_desc, - IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { + if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) { struct net_device *netdev = rx_ring->netdev; if (!(netdev->features & NETIF_F_RXALL)) { @@ -3679,6 +3683,8 @@ void igc_update_stats(struct igc_adapter *adapter) adapter->stats.prc511 += rd32(IGC_PRC511); adapter->stats.prc1023 += rd32(IGC_PRC1023); adapter->stats.prc1522 += rd32(IGC_PRC1522); + adapter->stats.tlpic += rd32(IGC_TLPIC); + adapter->stats.rlpic += rd32(IGC_RLPIC); mpc = rd32(IGC_MPC); adapter->stats.mpc += mpc; @@ -3772,6 +3778,8 @@ void igc_down(struct igc_adapter *adapter) set_bit(__IGC_DOWN, &adapter->state); + igc_ptp_suspend(adapter); + /* disable receives in the hardware */ rctl = rd32(IGC_RCTL); wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); @@ -3825,7 +3833,6 @@ void igc_down(struct igc_adapter *adapter) void igc_reinit_locked(struct igc_adapter *adapter) { - WARN_ON(in_interrupt()); while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) usleep_range(1000, 2000); igc_down(adapter); @@ -4694,14 +4701,35 @@ static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, return 0; } -static bool validate_schedule(const struct tc_taprio_qopt_offload *qopt) +static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now) +{ + struct timespec64 b; + + b = ktime_to_timespec64(base_time); + + return timespec64_compare(now, &b) > 0; +} + +static bool validate_schedule(struct igc_adapter *adapter, + const struct tc_taprio_qopt_offload *qopt) { int queue_uses[IGC_MAX_TX_QUEUES] = { }; + struct timespec64 now; size_t n; if (qopt->cycle_time_extension) return false; + igc_ptp_read(adapter, &now); + + /* If we program the controller's BASET registers with a time + * in the future, it will hold all the packets until that + * time, causing a lot of TX Hangs, so to avoid that, we + * reject schedules that would start in the future. + */ + if (!is_base_time_past(qopt->base_time, &now)) + return false; + for (n = 0; n < qopt->num_entries; n++) { const struct tc_taprio_sched_entry *e; int i; @@ -4756,7 +4784,7 @@ static int igc_save_qbv_schedule(struct igc_adapter *adapter, if (adapter->base_time) return -EALREADY; - if (!validate_schedule(qopt)) + if (!validate_schedule(adapter, qopt)) return -EINVAL; adapter->cycle_time = qopt->cycle_time; diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c index e4b8f312f97c..ac0b9c85da7c 100644 --- a/drivers/net/ethernet/intel/igc/igc_ptp.c +++ b/drivers/net/ethernet/intel/igc/igc_ptp.c @@ -8,6 +8,7 @@ #include <linux/pci.h> #include <linux/ptp_classify.h> #include <linux/clocksource.h> +#include <linux/ktime.h> #define INCVALUE_MASK 0x7fffffff #define ISGN 0x80000000 @@ -16,17 +17,12 @@ #define IGC_PTP_TX_TIMEOUT (HZ * 15) /* SYSTIM read access for I225 */ -static void igc_ptp_read_i225(struct igc_adapter *adapter, - struct timespec64 *ts) +void igc_ptp_read(struct igc_adapter *adapter, struct timespec64 *ts) { struct igc_hw *hw = &adapter->hw; u32 sec, nsec; - /* The timestamp latches on lowest register read. For I210/I211, the - * lowest register is SYSTIMR. Since we only need to provide nanosecond - * resolution, we can ignore it. - */ - rd32(IGC_SYSTIMR); + /* The timestamp is latched when SYSTIML is read. */ nsec = rd32(IGC_SYSTIML); sec = rd32(IGC_SYSTIMH); @@ -39,9 +35,6 @@ static void igc_ptp_write_i225(struct igc_adapter *adapter, { struct igc_hw *hw = &adapter->hw; - /* Writing the SYSTIMR register is not necessary as it only - * provides sub-nanosecond resolution. - */ wr32(IGC_SYSTIML, ts->tv_nsec); wr32(IGC_SYSTIMH, ts->tv_sec); } @@ -81,7 +74,7 @@ static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta) spin_lock_irqsave(&igc->tmreg_lock, flags); - igc_ptp_read_i225(igc, &now); + igc_ptp_read(igc, &now); now = timespec64_add(now, then); igc_ptp_write_i225(igc, (const struct timespec64 *)&now); @@ -102,10 +95,9 @@ static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp, spin_lock_irqsave(&igc->tmreg_lock, flags); ptp_read_system_prets(sts); - rd32(IGC_SYSTIMR); - ptp_read_system_postts(sts); ts->tv_nsec = rd32(IGC_SYSTIML); ts->tv_sec = rd32(IGC_SYSTIMH); + ptp_read_system_postts(sts); spin_unlock_irqrestore(&igc->tmreg_lock, flags); @@ -422,18 +414,11 @@ static void igc_ptp_tx_work(struct work_struct *work) if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state)) return; - if (time_is_before_jiffies(adapter->ptp_tx_start + - IGC_PTP_TX_TIMEOUT)) { - igc_ptp_tx_timeout(adapter); + tsynctxctl = rd32(IGC_TSYNCTXCTL); + if (WARN_ON_ONCE(!(tsynctxctl & IGC_TSYNCTXCTL_TXTT_0))) return; - } - tsynctxctl = rd32(IGC_TSYNCTXCTL); - if (tsynctxctl & IGC_TSYNCTXCTL_VALID) - igc_ptp_tx_hwtstamp(adapter); - else - /* reschedule to check later */ - schedule_work(&adapter->ptp_tx_work); + igc_ptp_tx_hwtstamp(adapter); } /** @@ -515,6 +500,9 @@ void igc_ptp_init(struct igc_adapter *adapter) adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF; + adapter->prev_ptp_time = ktime_to_timespec64(ktime_get_real()); + adapter->ptp_reset_start = ktime_get(); + adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps, &adapter->pdev->dev); if (IS_ERR(adapter->ptp_clock)) { @@ -526,6 +514,24 @@ void igc_ptp_init(struct igc_adapter *adapter) } } +static void igc_ptp_time_save(struct igc_adapter *adapter) +{ + igc_ptp_read(adapter, &adapter->prev_ptp_time); + adapter->ptp_reset_start = ktime_get(); +} + +static void igc_ptp_time_restore(struct igc_adapter *adapter) +{ + struct timespec64 ts = adapter->prev_ptp_time; + ktime_t delta; + + delta = ktime_sub(ktime_get(), adapter->ptp_reset_start); + + timespec64_add_ns(&ts, ktime_to_ns(delta)); + + igc_ptp_write_i225(adapter, &ts); +} + /** * igc_ptp_suspend - Disable PTP work items and prepare for suspend * @adapter: Board private structure @@ -542,6 +548,8 @@ void igc_ptp_suspend(struct igc_adapter *adapter) dev_kfree_skb_any(adapter->ptp_tx_skb); adapter->ptp_tx_skb = NULL; clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state); + + igc_ptp_time_save(adapter); } /** @@ -591,9 +599,7 @@ void igc_ptp_reset(struct igc_adapter *adapter) /* Re-initialize the timer. */ if (hw->mac.type == igc_i225) { - struct timespec64 ts64 = ktime_to_timespec64(ktime_get_real()); - - igc_ptp_write_i225(adapter, &ts64); + igc_ptp_time_restore(adapter); } else { timecounter_init(&adapter->tc, &adapter->cc, ktime_to_ns(ktime_get_real())); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 02899f79f0ff..a190d5c616fc 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -5672,7 +5672,6 @@ static void ixgbe_up_complete(struct ixgbe_adapter *adapter) void ixgbe_reinit_locked(struct ixgbe_adapter *adapter) { - WARN_ON(in_interrupt()); /* put off any impending NetWatchDogTimeout */ netif_trans_update(adapter->netdev); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index ba54c728aef2..82fce27f682b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -2522,8 +2522,6 @@ void ixgbevf_down(struct ixgbevf_adapter *adapter) void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) { - WARN_ON(in_interrupt()); - while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) msleep(1); diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 14df3aec285d..d095718355d3 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -1831,7 +1831,7 @@ static struct mvneta_tx_queue *mvneta_tx_done_policy(struct mvneta_port *pp, /* Free tx queue skbuffs */ static void mvneta_txq_bufs_free(struct mvneta_port *pp, struct mvneta_tx_queue *txq, int num, - struct netdev_queue *nq) + struct netdev_queue *nq, bool napi) { unsigned int bytes_compl = 0, pkts_compl = 0; int i; @@ -1854,7 +1854,10 @@ static void mvneta_txq_bufs_free(struct mvneta_port *pp, dev_kfree_skb_any(buf->skb); } else if (buf->type == MVNETA_TYPE_XDP_TX || buf->type == MVNETA_TYPE_XDP_NDO) { - xdp_return_frame(buf->xdpf); + if (napi && buf->type == MVNETA_TYPE_XDP_TX) + xdp_return_frame_rx_napi(buf->xdpf); + else + xdp_return_frame(buf->xdpf); } } @@ -1872,7 +1875,7 @@ static void mvneta_txq_done(struct mvneta_port *pp, if (!tx_done) return; - mvneta_txq_bufs_free(pp, txq, tx_done, nq); + mvneta_txq_bufs_free(pp, txq, tx_done, nq, true); txq->count -= tx_done; @@ -2233,19 +2236,22 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, enum dma_data_direction dma_dir; struct skb_shared_info *sinfo; - if (rx_desc->data_size > MVNETA_MAX_RX_BUF_SIZE) { + if (*size > MVNETA_MAX_RX_BUF_SIZE) { len = MVNETA_MAX_RX_BUF_SIZE; data_len += len; } else { - len = rx_desc->data_size; + len = *size; data_len += len - ETH_FCS_LEN; } + *size = *size - len; dma_dir = page_pool_get_dma_dir(rxq->page_pool); dma_sync_single_for_cpu(dev->dev.parent, rx_desc->buf_phys_addr, len, dma_dir); + rx_desc->buf_phys_addr = 0; + /* Prefetch header */ prefetch(data); @@ -2256,9 +2262,6 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, sinfo = xdp_get_shared_info_from_buff(xdp); sinfo->nr_frags = 0; - - *size = rx_desc->data_size - len; - rx_desc->buf_phys_addr = 0; } static void @@ -2372,7 +2375,7 @@ static int mvneta_rx_swbm(struct napi_struct *napi, size = rx_desc->data_size; frame_sz = size - ETH_FCS_LEN; - desc_status = rx_desc->status; + desc_status = rx_status; mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf, &size, page); @@ -2859,7 +2862,7 @@ static void mvneta_txq_done_force(struct mvneta_port *pp, struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id); int tx_done = txq->count; - mvneta_txq_bufs_free(pp, txq, tx_done, nq); + mvneta_txq_bufs_free(pp, txq, tx_done, nq, false); /* reset txq */ txq->count = 0; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/npc.h index 95c646ae7e23..91a9d00e4fb5 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc.h @@ -49,6 +49,7 @@ enum npc_kpu_lb_ltype { NPC_LT_LB_EDSA_VLAN, NPC_LT_LB_EXDSA, NPC_LT_LB_EXDSA_VLAN, + NPC_LT_LB_FDSA, NPC_LT_LB_CUSTOM0 = 0xE, NPC_LT_LB_CUSTOM1 = 0xF, }; @@ -77,21 +78,21 @@ enum npc_kpu_ld_ltype { NPC_LT_LD_ICMP, NPC_LT_LD_SCTP, NPC_LT_LD_ICMP6, + NPC_LT_LD_CUSTOM0, + NPC_LT_LD_CUSTOM1, NPC_LT_LD_IGMP = 8, - NPC_LT_LD_ESP, NPC_LT_LD_AH, NPC_LT_LD_GRE, NPC_LT_LD_NVGRE, NPC_LT_LD_NSH, NPC_LT_LD_TU_MPLS_IN_NSH, NPC_LT_LD_TU_MPLS_IN_IP, - NPC_LT_LD_CUSTOM0 = 0xE, - NPC_LT_LD_CUSTOM1 = 0xF, }; enum npc_kpu_le_ltype { NPC_LT_LE_VXLAN = 1, NPC_LT_LE_GENEVE, + NPC_LT_LE_ESP, NPC_LT_LE_GTPU = 4, NPC_LT_LE_VXLANGPE, NPC_LT_LE_GTPC, @@ -296,6 +297,9 @@ struct nix_rx_action { #endif }; +/* NPC_AF_INTFX_KEX_CFG field masks */ +#define NPC_PARSE_NIBBLE GENMASK_ULL(30, 0) + /* NIX Receive Vtag Action Structure */ #define VTAG0_VALID_BIT BIT_ULL(15) #define VTAG0_TYPE_MASK GENMASK_ULL(14, 12) @@ -320,4 +324,37 @@ struct npc_mcam_kex { u64 intf_ld_flags[NPC_MAX_INTF][NPC_MAX_LD][NPC_MAX_LFL]; } __packed; +struct npc_lt_def { + u8 ltype_mask; + u8 ltype_match; + u8 lid; +}; + +struct npc_lt_def_ipsec { + u8 ltype_mask; + u8 ltype_match; + u8 lid; + u8 spi_offset; + u8 spi_nz; +}; + +struct npc_lt_def_cfg { + struct npc_lt_def rx_ol2; + struct npc_lt_def rx_oip4; + struct npc_lt_def rx_iip4; + struct npc_lt_def rx_oip6; + struct npc_lt_def rx_iip6; + struct npc_lt_def rx_otcp; + struct npc_lt_def rx_itcp; + struct npc_lt_def rx_oudp; + struct npc_lt_def rx_iudp; + struct npc_lt_def rx_osctp; + struct npc_lt_def rx_isctp; + struct npc_lt_def_ipsec rx_ipsec[2]; + struct npc_lt_def pck_ol2; + struct npc_lt_def pck_oip4; + struct npc_lt_def pck_oip6; + struct npc_lt_def pck_iip4; +}; + #endif /* NPC_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h index b29c6689ace2..77bb4ed32600 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h @@ -63,6 +63,7 @@ #define NPC_UDP_PORT_VXLANGPE 4790 #define NPC_UDP_PORT_GENEVE 6081 #define NPC_UDP_PORT_MPLS 6635 +#define NPC_UDP_PORT_ESP 4500 #define NPC_VXLANGPE_NP_IP 0x1 #define NPC_VXLANGPE_NP_IP6 0x2 @@ -139,6 +140,13 @@ #define NPC_DSA_EXTEND 0x1000 #define NPC_DSA_EDSA 0x8000 +#define NPC_DSA_FDSA 0xc000 + +#define NPC_KEXOF_DMAC 8 +#define MKEX_SIGN 0x19bbfdbd15f /* strtoull of "mkexprof" with base:36 */ +#define KEX_LD_CFG(bytesm1, hdr_ofs, ena, flags_ena, key_ofs) \ + (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ + ((flags_ena) << 6) | ((key_ofs) & 0x3F)) enum npc_kpu_parser_state { NPC_S_NA = 0, @@ -166,6 +174,7 @@ enum npc_kpu_parser_state { NPC_S_KPU3_DSA, NPC_S_KPU4_MPLS, NPC_S_KPU4_NSH, + NPC_S_KPU4_FDSA, NPC_S_KPU5_IP, NPC_S_KPU5_IP6, NPC_S_KPU5_ARP, @@ -189,7 +198,6 @@ enum npc_kpu_parser_state { NPC_S_KPU8_IGMP, NPC_S_KPU8_ICMP6, NPC_S_KPU8_GRE, - NPC_S_KPU8_ESP, NPC_S_KPU8_AH, NPC_S_KPU9_TU_MPLS_IN_GRE, NPC_S_KPU9_TU_MPLS_IN_NSH, @@ -201,6 +209,7 @@ enum npc_kpu_parser_state { NPC_S_KPU9_GENEVE, NPC_S_KPU9_GTPC, NPC_S_KPU9_GTPU, + NPC_S_KPU9_ESP, NPC_S_KPU10_TU_MPLS_IN_VXLANGPE, NPC_S_KPU10_TU_MPLS_PL, NPC_S_KPU10_TU_MPLS, @@ -271,6 +280,7 @@ enum npc_kpu_lb_lflag { NPC_F_LB_L_EDSA_VLAN, NPC_F_LB_L_EXDSA, NPC_F_LB_L_EXDSA_VLAN, + NPC_F_LB_L_FDSA, }; enum npc_kpu_lc_uflag { @@ -979,7 +989,7 @@ static const struct npc_kpu_profile_action ikpu_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 12, 16, 20, 0, 0, + 12, 14, 20, 0, 0, NPC_S_KPU1_EXDSA, 0, 0, NPC_LID_LA, NPC_LT_NA, 0, @@ -1351,10 +1361,19 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_EXDSA, 0xff, + 0x0000, + 0x0000, NPC_DSA_EXTEND, NPC_DSA_EXTEND, 0x0000, 0x0000, + }, + { + NPC_S_KPU1_EXDSA, 0xff, + NPC_DSA_FDSA, + NPC_DSA_FDSA, + 0x0000, + 0x0000, 0x0000, 0x0000, }, @@ -3996,6 +4015,69 @@ static const struct npc_kpu_profile_cam kpu4_cam_entries[] = { 0x0000, }, { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_IP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_IP6, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_ARP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_RARP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_PTP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_FCOE, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + 0x0000, + NPC_DSA_FDSA, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { NPC_S_NA, 0X00, 0x0000, 0x0000, @@ -5341,15 +5423,24 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { }, { NPC_S_KPU8_UDP, 0xff, + NPC_UDP_PORT_ESP, + 0xffff, 0x0000, 0x0000, 0x0000, 0x0000, + }, + { + NPC_S_KPU8_UDP, 0xff, + 0x0000, + 0x0000, + NPC_UDP_PORT_ESP, + 0xffff, 0x0000, 0x0000, }, { - NPC_S_KPU8_SCTP, 0xff, + NPC_S_KPU8_UDP, 0xff, 0x0000, 0x0000, 0x0000, @@ -5358,7 +5449,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { 0x0000, }, { - NPC_S_KPU8_ICMP, 0xff, + NPC_S_KPU8_SCTP, 0xff, 0x0000, 0x0000, 0x0000, @@ -5367,7 +5458,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { 0x0000, }, { - NPC_S_KPU8_IGMP, 0xff, + NPC_S_KPU8_ICMP, 0xff, 0x0000, 0x0000, 0x0000, @@ -5376,7 +5467,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { 0x0000, }, { - NPC_S_KPU8_ICMP6, 0xff, + NPC_S_KPU8_IGMP, 0xff, 0x0000, 0x0000, 0x0000, @@ -5385,7 +5476,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { 0x0000, }, { - NPC_S_KPU8_ESP, 0xff, + NPC_S_KPU8_ICMP6, 0xff, 0x0000, 0x0000, 0x0000, @@ -6324,6 +6415,15 @@ static const struct npc_kpu_profile_cam kpu9_cam_entries[] = { NPC_MPLS_S, }, { + NPC_S_KPU9_ESP, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { NPC_S_NA, 0X00, 0x0000, 0x0000, @@ -7673,6 +7773,14 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 16, 2, 0, + NPC_S_KPU4_FDSA, 12, 1, + NPC_LID_LA, NPC_LT_LA_ETHER, + 0, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LA, NPC_EC_EDSA_UNK, 0, 0, 0, 0, 1, NPC_S_NA, 0, 1, @@ -10034,6 +10142,62 @@ static const struct npc_kpu_profile_action kpu4_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 0, 0, + NPC_S_KPU5_IP, 6, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 0, 0, + NPC_S_KPU5_IP6, 6, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU5_ARP, 6, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 0, 0, + NPC_S_KPU5_RARP, 6, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 0, 0, + NPC_S_KPU5_PTP, 6, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU5_FCOE, 6, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_U_UNK_ETYPE | NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LB, NPC_EC_L2_K4, 0, 0, 0, 0, 1, NPC_S_NA, 0, 0, @@ -10102,8 +10266,8 @@ static const struct npc_kpu_profile_action kpu5_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 2, 0, - NPC_S_KPU8_ESP, 20, 1, + 0, 0, 0, 3, 0, + NPC_S_KPU9_ESP, 20, 1, NPC_LID_LC, NPC_LT_LC_IP, 0, 0, 0, 0, 0, @@ -10206,8 +10370,8 @@ static const struct npc_kpu_profile_action kpu5_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 2, 0, - NPC_S_KPU8_ESP, 0, 1, + 0, 0, 0, 3, 0, + NPC_S_KPU9_ESP, 0, 1, NPC_LID_LC, NPC_LT_LC_IP_OPT, 0, 0, 0xf, 0, 2, @@ -10414,8 +10578,8 @@ static const struct npc_kpu_profile_action kpu5_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 2, 0, - NPC_S_KPU8_ESP, 40, 1, + 0, 0, 0, 3, 0, + NPC_S_KPU9_ESP, 40, 1, NPC_LID_LC, NPC_LT_LC_IP6_EXT, 0, 0, 0, 0, 0, @@ -10561,80 +10725,80 @@ static const struct npc_kpu_profile_action kpu6_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 2, 12, 0, 1, 0, - NPC_S_KPU8_TCP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 2, 8, 10, 1, 0, - NPC_S_KPU8_UDP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU8_SCTP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU8_ICMP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU8_ICMP6, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU8_ESP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU8_AH, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU8_GRE, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 6, 0, 0, 5, 0, - NPC_S_KPU12_TU_IP6, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 2, 6, 10, 2, 0, - NPC_S_KPU9_TU_MPLS_IN_IP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -10689,8 +10853,8 @@ static const struct npc_kpu_profile_action kpu6_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU8_ESP, 8, 0, + 0, 0, 0, 2, 0, + NPC_S_KPU9_ESP, 8, 0, NPC_LID_LC, NPC_LT_NA, 0, 1, 0xff, 0, 3, @@ -10793,8 +10957,8 @@ static const struct npc_kpu_profile_action kpu6_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU8_ESP, 8, 0, + 0, 0, 0, 2, 0, + NPC_S_KPU9_ESP, 8, 0, NPC_LID_LC, NPC_LT_NA, 0, 1, 0xff, 0, 3, @@ -10908,8 +11072,8 @@ static const struct npc_kpu_profile_action kpu7_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 0, 0, - NPC_S_KPU8_ESP, 8, 0, + 0, 0, 0, 1, 0, + NPC_S_KPU9_ESP, 8, 0, NPC_LID_LC, NPC_LT_NA, 0, 1, 0xff, 0, 3, @@ -10956,80 +11120,80 @@ static const struct npc_kpu_profile_action kpu7_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 2, 12, 0, 0, 0, - NPC_S_KPU8_TCP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 2, 8, 10, 0, 0, - NPC_S_KPU8_UDP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 0, 0, - NPC_S_KPU8_SCTP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 0, 0, - NPC_S_KPU8_ICMP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 0, 0, - NPC_S_KPU8_ICMP6, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 0, 0, - NPC_S_KPU8_ESP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 0, 0, - NPC_S_KPU8_AH, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 0, 0, - NPC_S_KPU8_GRE, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 6, 0, 0, 4, 0, - NPC_S_KPU12_TU_IP6, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 2, 6, 10, 1, 0, - NPC_S_KPU9_TU_MPLS_IN_IP, 8, 0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, NPC_LID_LC, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -11231,6 +11395,22 @@ static const struct npc_kpu_profile_action kpu8_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU9_ESP, 8, 1, + NPC_LID_LD, NPC_LT_LD_UDP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU9_ESP, 8, 1, + NPC_LID_LD, NPC_LT_LD_UDP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 7, 0, NPC_S_KPU16_UDP_DATA, 8, 1, NPC_LID_LD, NPC_LT_LD_UDP, @@ -11273,14 +11453,6 @@ static const struct npc_kpu_profile_action kpu8_action_entries[] = { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 1, NPC_S_NA, 0, 1, - NPC_LID_LD, NPC_LT_LD_ESP, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 1, NPC_LID_LD, NPC_LT_LD_AH, 0, 0, 0, 0, 0, @@ -12105,6 +12277,14 @@ static const struct npc_kpu_profile_action kpu9_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LE, NPC_LT_LE_ESP, + 0, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LE, NPC_EC_UNK, 0, 0, 0, 0, 1, NPC_S_NA, 0, 0, @@ -13114,4 +13294,163 @@ static const struct npc_kpu_profile npc_kpu_profiles[] = { }, }; +static const struct npc_lt_def_cfg npc_lt_defaults = { + .rx_ol2 = { + .lid = NPC_LID_LA, + .ltype_match = NPC_LT_LA_ETHER, + .ltype_mask = 0x0F, + }, + .rx_oip4 = { + .lid = NPC_LID_LC, + .ltype_match = NPC_LT_LC_IP, + .ltype_mask = 0x0E, + }, + .rx_iip4 = { + .lid = NPC_LID_LG, + .ltype_match = NPC_LT_LG_TU_IP, + .ltype_mask = 0x0F, + }, + .rx_oip6 = { + .lid = NPC_LID_LC, + .ltype_match = NPC_LT_LC_IP6, + .ltype_mask = 0x0E, + }, + .rx_iip6 = { + .lid = NPC_LID_LG, + .ltype_match = NPC_LT_LG_TU_IP6, + .ltype_mask = 0x0F, + }, + .rx_otcp = { + .lid = NPC_LID_LD, + .ltype_match = NPC_LT_LD_TCP, + .ltype_mask = 0x0F, + }, + .rx_itcp = { + .lid = NPC_LID_LH, + .ltype_match = NPC_LT_LH_TU_TCP, + .ltype_mask = 0x0F, + }, + .rx_oudp = { + .lid = NPC_LID_LD, + .ltype_match = NPC_LT_LD_UDP, + .ltype_mask = 0x0F, + }, + .rx_iudp = { + .lid = NPC_LID_LH, + .ltype_match = NPC_LT_LH_TU_UDP, + .ltype_mask = 0x0F, + }, + .rx_osctp = { + .lid = NPC_LID_LD, + .ltype_match = NPC_LT_LD_SCTP, + .ltype_mask = 0x0F, + }, + .rx_isctp = { + .lid = NPC_LID_LH, + .ltype_match = NPC_LT_LH_TU_SCTP, + .ltype_mask = 0x0F, + }, + .rx_ipsec = { + { + .lid = NPC_LID_LE, + .ltype_match = NPC_LT_LE_ESP, + .ltype_mask = 0x0F, + }, + { + .spi_offset = 8, + .lid = NPC_LID_LH, + .ltype_match = NPC_LT_LH_TU_ESP, + .ltype_mask = 0x0F, + }, + }, + .pck_ol2 = { + .lid = NPC_LID_LA, + .ltype_match = NPC_LT_LA_ETHER, + .ltype_mask = 0x0F, + }, + .pck_oip4 = { + .lid = NPC_LID_LC, + .ltype_match = NPC_LT_LC_IP, + .ltype_mask = 0x0E, + }, + .pck_iip4 = { + .lid = NPC_LID_LG, + .ltype_match = NPC_LT_LG_TU_IP, + .ltype_mask = 0x0F, + }, +}; + +static const struct npc_mcam_kex npc_mkex_default = { + .mkex_sign = MKEX_SIGN, + .name = "default", + .kpu_version = NPC_KPU_PROFILE_VER, + .keyx_cfg = { + /* nibble: LA..LE (ltype only) + Channel */ + [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_X2 << 32) | 0x49247, + [NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) | ((1ULL << 19) - 1), + }, + .intf_lid_lt_ld = { + /* Default RX MCAM KEX profile */ + [NIX_INTF_RX] = { + [NPC_LID_LA] = { + /* Layer A: Ethernet: */ + [NPC_LT_LA_ETHER] = { + /* DMAC: 6 bytes, KW1[47:0] */ + KEX_LD_CFG(0x05, 0x0, 0x1, 0x0, NPC_KEXOF_DMAC), + /* Ethertype: 2 bytes, KW0[47:32] */ + KEX_LD_CFG(0x01, 0xc, 0x1, 0x0, 0x4), + }, + }, + [NPC_LID_LB] = { + /* Layer B: Single VLAN (CTAG) */ + /* CTAG VLAN[2..3] + Ethertype, 4 bytes, KW0[63:32] */ + [NPC_LT_LB_CTAG] = { + KEX_LD_CFG(0x03, 0x0, 0x1, 0x0, 0x4), + }, + /* Layer B: Stacked VLAN (STAG|QinQ) */ + [NPC_LT_LB_STAG_QINQ] = { + /* CTAG VLAN[2..3] + Ethertype, 4 bytes, KW0[63:32] */ + KEX_LD_CFG(0x03, 0x4, 0x1, 0x0, 0x4), + }, + [NPC_LT_LB_FDSA] = { + /* SWITCH PORT: 1 byte, KW0[63:48] */ + KEX_LD_CFG(0x0, 0x1, 0x1, 0x0, 0x6), + /* Ethertype: 2 bytes, KW0[47:32] */ + KEX_LD_CFG(0x01, 0x4, 0x1, 0x0, 0x4), + }, + }, + [NPC_LID_LC] = { + /* Layer C: IPv4 */ + [NPC_LT_LC_IP] = { + /* SIP+DIP: 8 bytes, KW2[63:0] */ + KEX_LD_CFG(0x07, 0xc, 0x1, 0x0, 0x10), + /* TOS: 1 byte, KW1[63:56] */ + KEX_LD_CFG(0x0, 0x1, 0x1, 0x0, 0xf), + }, + /* Layer C: IPv6 */ + [NPC_LT_LC_IP6] = { + /* Everything up to SADDR: 8 bytes, KW2[63:0] */ + KEX_LD_CFG(0x07, 0x0, 0x1, 0x0, 0x10), + }, + }, + [NPC_LID_LD] = { + /* Layer D:UDP */ + [NPC_LT_LD_UDP] = { + /* SPORT: 2 bytes, KW3[15:0] */ + KEX_LD_CFG(0x1, 0x0, 0x1, 0x0, 0x18), + /* DPORT: 2 bytes, KW3[31:16] */ + KEX_LD_CFG(0x1, 0x2, 0x1, 0x0, 0x1a), + }, + /* Layer D:TCP */ + [NPC_LT_LD_TCP] = { + /* SPORT: 2 bytes, KW3[15:0] */ + KEX_LD_CFG(0x1, 0x0, 0x1, 0x0, 0x18), + /* DPORT: 2 bytes, KW3[31:16] */ + KEX_LD_CFG(0x1, 0x2, 0x1, 0x0, 0x1a), + }, + }, + }, + }, +}; + #endif /* NPC_PROFILE_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index 05da7a91944a..f1b03f8ab0ee 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -291,6 +291,20 @@ struct rvu_fwdata { struct ptp; +/* KPU profile adapter structure gathering all KPU configuration data and abstracting out the + * source where it came from. + */ +struct npc_kpu_profile_adapter { + const char *name; + u64 version; + const struct npc_lt_def_cfg *lt_def; + const struct npc_kpu_profile_action *ikpu; /* array[pkinds] */ + const struct npc_kpu_profile *kpu; /* array[kpus] */ + const struct npc_mcam_kex *mkex; + size_t pkinds; + size_t kpus; +}; + struct rvu { void __iomem *afreg_base; void __iomem *pfreg_base; @@ -339,6 +353,9 @@ struct rvu { /* Firmware data */ struct rvu_fwdata *fwdata; + /* NPC KPU data */ + struct npc_kpu_profile_adapter kpu; + struct ptp *ptp; #ifdef CONFIG_DEBUG_FS diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 4bdc4baa3c59..b403c16caa2f 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -3112,6 +3112,7 @@ static int nix_aq_init(struct rvu *rvu, struct rvu_block *block) int rvu_nix_init(struct rvu *rvu) { + const struct npc_lt_def_cfg *ltdefs; struct rvu_hwinfo *hw = rvu->hw; struct rvu_block *block; int blkaddr, err; @@ -3142,6 +3143,7 @@ int rvu_nix_init(struct rvu *rvu) rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg); } + ltdefs = rvu->kpu.lt_def; /* Calibrate X2P bus to check if CGX/LBK links are fine */ err = nix_calibrate_x2p(rvu, blkaddr); if (err) @@ -3189,28 +3191,38 @@ int rvu_nix_init(struct rvu *rvu) * and validate length and checksums. */ rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OL2, - (NPC_LID_LA << 8) | (NPC_LT_LA_ETHER << 4) | 0x0F); + (ltdefs->rx_ol2.lid << 8) | (ltdefs->rx_ol2.ltype_match << 4) | + ltdefs->rx_ol2.ltype_mask); rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4, - (NPC_LID_LC << 8) | (NPC_LT_LC_IP << 4) | 0x0F); + (ltdefs->rx_oip4.lid << 8) | (ltdefs->rx_oip4.ltype_match << 4) | + ltdefs->rx_oip4.ltype_mask); rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4, - (NPC_LID_LG << 8) | (NPC_LT_LG_TU_IP << 4) | 0x0F); + (ltdefs->rx_iip4.lid << 8) | (ltdefs->rx_iip4.ltype_match << 4) | + ltdefs->rx_iip4.ltype_mask); rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6, - (NPC_LID_LC << 8) | (NPC_LT_LC_IP6 << 4) | 0x0F); + (ltdefs->rx_oip6.lid << 8) | (ltdefs->rx_oip6.ltype_match << 4) | + ltdefs->rx_oip6.ltype_mask); rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6, - (NPC_LID_LG << 8) | (NPC_LT_LG_TU_IP6 << 4) | 0x0F); + (ltdefs->rx_iip6.lid << 8) | (ltdefs->rx_iip6.ltype_match << 4) | + ltdefs->rx_iip6.ltype_mask); rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OTCP, - (NPC_LID_LD << 8) | (NPC_LT_LD_TCP << 4) | 0x0F); + (ltdefs->rx_otcp.lid << 8) | (ltdefs->rx_otcp.ltype_match << 4) | + ltdefs->rx_otcp.ltype_mask); rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ITCP, - (NPC_LID_LH << 8) | (NPC_LT_LH_TU_TCP << 4) | 0x0F); + (ltdefs->rx_itcp.lid << 8) | (ltdefs->rx_itcp.ltype_match << 4) | + ltdefs->rx_itcp.ltype_mask); rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OUDP, - (NPC_LID_LD << 8) | (NPC_LT_LD_UDP << 4) | 0x0F); + (ltdefs->rx_oudp.lid << 8) | (ltdefs->rx_oudp.ltype_match << 4) | + ltdefs->rx_oudp.ltype_mask); rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IUDP, - (NPC_LID_LH << 8) | (NPC_LT_LH_TU_UDP << 4) | 0x0F); + (ltdefs->rx_iudp.lid << 8) | (ltdefs->rx_iudp.ltype_match << 4) | + ltdefs->rx_iudp.ltype_mask); rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OSCTP, - (NPC_LID_LD << 8) | (NPC_LT_LD_SCTP << 4) | 0x0F); + (ltdefs->rx_osctp.lid << 8) | (ltdefs->rx_osctp.ltype_match << 4) | + ltdefs->rx_osctp.ltype_mask); rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ISCTP, - (NPC_LID_LH << 8) | (NPC_LT_LH_TU_SCTP << 4) | - 0x0F); + (ltdefs->rx_isctp.lid << 8) | (ltdefs->rx_isctp.ltype_match << 4) | + ltdefs->rx_isctp.ltype_mask); err = nix_rx_flowkey_alg_cfg(rvu, blkaddr); if (err) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index 6a1ddb2da1a5..0abe5fd12131 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -29,6 +29,8 @@ #define NPC_PARSE_RESULT_DMAC_OFFSET 8 #define NPC_HW_TSTAMP_OFFSET 8 +static const char def_pfl_name[] = "default"; + static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, u16 pcifunc); static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, @@ -448,7 +450,7 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, entry.kw_mask[0] = 0xFFFULL; if (allmulti) { - kwi = NPC_PARSE_RESULT_DMAC_OFFSET / sizeof(u64); + kwi = NPC_KEXOF_DMAC / sizeof(u64); entry.kw[kwi] = BIT_ULL(40); /* LSB bit of 1st byte in DMAC */ entry.kw_mask[kwi] = BIT_ULL(40); } @@ -718,88 +720,8 @@ void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) rvu_write64(rvu, blkaddr, \ NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) -#define KEX_LD_CFG(bytesm1, hdr_ofs, ena, flags_ena, key_ofs) \ - (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ - ((flags_ena) << 6) | ((key_ofs) & 0x3F)) - -static void npc_config_ldata_extract(struct rvu *rvu, int blkaddr) -{ - struct npc_mcam *mcam = &rvu->hw->mcam; - int lid, ltype; - int lid_count; - u64 cfg; - - cfg = rvu_read64(rvu, blkaddr, NPC_AF_CONST); - lid_count = (cfg >> 4) & 0xF; - - /* First clear any existing config i.e - * disable LDATA and FLAGS extraction. - */ - for (lid = 0; lid < lid_count; lid++) { - for (ltype = 0; ltype < 16; ltype++) { - SET_KEX_LD(NIX_INTF_RX, lid, ltype, 0, 0ULL); - SET_KEX_LD(NIX_INTF_RX, lid, ltype, 1, 0ULL); - SET_KEX_LD(NIX_INTF_TX, lid, ltype, 0, 0ULL); - SET_KEX_LD(NIX_INTF_TX, lid, ltype, 1, 0ULL); - - SET_KEX_LDFLAGS(NIX_INTF_RX, 0, ltype, 0ULL); - SET_KEX_LDFLAGS(NIX_INTF_RX, 1, ltype, 0ULL); - SET_KEX_LDFLAGS(NIX_INTF_TX, 0, ltype, 0ULL); - SET_KEX_LDFLAGS(NIX_INTF_TX, 1, ltype, 0ULL); - } - } - - if (mcam->keysize != NPC_MCAM_KEY_X2) - return; - - /* Default MCAM KEX profile */ - /* Layer A: Ethernet: */ - - /* DMAC: 6 bytes, KW1[47:0] */ - cfg = KEX_LD_CFG(0x05, 0x0, 0x1, 0x0, NPC_PARSE_RESULT_DMAC_OFFSET); - SET_KEX_LD(NIX_INTF_RX, NPC_LID_LA, NPC_LT_LA_ETHER, 0, cfg); - - /* Ethertype: 2 bytes, KW0[47:32] */ - cfg = KEX_LD_CFG(0x01, 0xc, 0x1, 0x0, 0x4); - SET_KEX_LD(NIX_INTF_RX, NPC_LID_LA, NPC_LT_LA_ETHER, 1, cfg); - - /* Layer B: Single VLAN (CTAG) */ - /* CTAG VLAN[2..3] + Ethertype, 4 bytes, KW0[63:32] */ - cfg = KEX_LD_CFG(0x03, 0x0, 0x1, 0x0, 0x4); - SET_KEX_LD(NIX_INTF_RX, NPC_LID_LB, NPC_LT_LB_CTAG, 0, cfg); - - /* Layer B: Stacked VLAN (STAG|QinQ) */ - /* CTAG VLAN[2..3] + Ethertype, 4 bytes, KW0[63:32] */ - cfg = KEX_LD_CFG(0x03, 0x4, 0x1, 0x0, 0x4); - SET_KEX_LD(NIX_INTF_RX, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 0, cfg); - - /* Layer C: IPv4 */ - /* SIP+DIP: 8 bytes, KW2[63:0] */ - cfg = KEX_LD_CFG(0x07, 0xc, 0x1, 0x0, 0x10); - SET_KEX_LD(NIX_INTF_RX, NPC_LID_LC, NPC_LT_LC_IP, 0, cfg); - /* TOS: 1 byte, KW1[63:56] */ - cfg = KEX_LD_CFG(0x0, 0x1, 0x1, 0x0, 0xf); - SET_KEX_LD(NIX_INTF_RX, NPC_LID_LC, NPC_LT_LC_IP, 1, cfg); - - /* Layer D:UDP */ - /* SPORT: 2 bytes, KW3[15:0] */ - cfg = KEX_LD_CFG(0x1, 0x0, 0x1, 0x0, 0x18); - SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_UDP, 0, cfg); - /* DPORT: 2 bytes, KW3[31:16] */ - cfg = KEX_LD_CFG(0x1, 0x2, 0x1, 0x0, 0x1a); - SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_UDP, 1, cfg); - - /* Layer D:TCP */ - /* SPORT: 2 bytes, KW3[15:0] */ - cfg = KEX_LD_CFG(0x1, 0x0, 0x1, 0x0, 0x18); - SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_TCP, 0, cfg); - /* DPORT: 2 bytes, KW3[31:16] */ - cfg = KEX_LD_CFG(0x1, 0x2, 0x1, 0x0, 0x1a); - SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_TCP, 1, cfg); -} - static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, - struct npc_mcam_kex *mkex) + const struct npc_mcam_kex *mkex) { int lid, lt, ld, fl; @@ -839,34 +761,31 @@ static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, } } -/* strtoull of "mkexprof" with base:36 */ -#define MKEX_SIGN 0x19bbfdbd15f #define MKEX_END_SIGN 0xdeadbeef -static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr) +static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, + const char *mkex_profile) { - const char *mkex_profile = rvu->mkex_pfl_name; struct device *dev = &rvu->pdev->dev; - void __iomem *mkex_prfl_addr = NULL; struct npc_mcam_kex *mcam_kex; - u64 prfl_addr; - u64 prfl_sz; + void *mkex_prfl_addr = NULL; + u64 prfl_addr, prfl_sz; /* If user not selected mkex profile */ - if (!strncmp(mkex_profile, "default", MKEX_NAME_LEN)) - goto load_default; + if (!strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) + goto program_mkex; if (!rvu->fwdata) - goto load_default; + goto program_mkex; prfl_addr = rvu->fwdata->mcam_addr; prfl_sz = rvu->fwdata->mcam_sz; if (!prfl_addr || !prfl_sz) - goto load_default; + goto program_mkex; - mkex_prfl_addr = ioremap_wc(prfl_addr, prfl_sz); + mkex_prfl_addr = memremap(prfl_addr, prfl_sz, MEMREMAP_WC); if (!mkex_prfl_addr) - goto load_default; + goto program_mkex; mcam_kex = (struct npc_mcam_kex *)mkex_prfl_addr; @@ -878,31 +797,23 @@ static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr) * parse nibble enable configuration has to be * identical for both Rx and Tx interfaces. */ - if (is_rvu_96xx_B0(rvu) && - mcam_kex->keyx_cfg[NIX_INTF_RX] != - mcam_kex->keyx_cfg[NIX_INTF_TX]) - goto load_default; - - /* Program selected mkex profile */ - npc_program_mkex_profile(rvu, blkaddr, mcam_kex); - - goto unmap; + if (!is_rvu_96xx_B0(rvu) || + mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) + rvu->kpu.mkex = mcam_kex; + goto program_mkex; } mcam_kex++; prfl_sz -= sizeof(struct npc_mcam_kex); } - dev_warn(dev, "Failed to load requested profile: %s\n", - rvu->mkex_pfl_name); + dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); -load_default: - dev_info(rvu->dev, "Using default mkex profile\n"); - /* Config packet data and flags extraction into PARSE result */ - npc_config_ldata_extract(rvu, blkaddr); - -unmap: +program_mkex: + dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); + /* Program selected mkex profile */ + npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); if (mkex_prfl_addr) - iounmap(mkex_prfl_addr); + memunmap(mkex_prfl_addr); } static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, @@ -1014,6 +925,27 @@ static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); } +static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) +{ + profile->name = def_pfl_name; + profile->version = NPC_KPU_PROFILE_VER; + profile->ikpu = ikpu_action_entries; + profile->pkinds = ARRAY_SIZE(ikpu_action_entries); + profile->kpu = npc_kpu_profiles; + profile->kpus = ARRAY_SIZE(npc_kpu_profiles); + profile->lt_def = &npc_lt_defaults; + profile->mkex = &npc_mkex_default; + + return 0; +} + +static void npc_load_kpu_profile(struct rvu *rvu) +{ + struct npc_kpu_profile_adapter *profile = &rvu->kpu; + + npc_prepare_default_kpu(profile); +} + static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) { struct rvu_hwinfo *hw = rvu->hw; @@ -1032,25 +964,26 @@ static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); } + /* Load and customize KPU profile. */ + npc_load_kpu_profile(rvu); + /* First program IKPU profile i.e PKIND configs. * Check HW max count to avoid configuring junk or * writing to unsupported CSR addresses. */ pkind = &hw->pkind; - num_pkinds = ARRAY_SIZE(ikpu_action_entries); + num_pkinds = rvu->kpu.pkinds; num_pkinds = min_t(int, pkind->rsrc.max, num_pkinds); for (idx = 0; idx < num_pkinds; idx++) - npc_config_kpuaction(rvu, blkaddr, - &ikpu_action_entries[idx], 0, idx, true); + npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true); /* Program KPU CAM and Action profiles */ - num_kpus = ARRAY_SIZE(npc_kpu_profiles); + num_kpus = rvu->kpu.kpus; num_kpus = min_t(int, hw->npc_kpus, num_kpus); for (idx = 0; idx < num_kpus; idx++) - npc_program_kpu_profile(rvu, blkaddr, - idx, &npc_kpu_profiles[idx]); + npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); } static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) @@ -1175,11 +1108,11 @@ free_mem: int rvu_npc_init(struct rvu *rvu) { + struct npc_kpu_profile_adapter *kpu = &rvu->kpu; struct npc_pkind *pkind = &rvu->hw->pkind; struct npc_mcam *mcam = &rvu->hw->mcam; - u64 keyz = NPC_MCAM_KEY_X2; + u64 cfg, nibble_ena, rx_kex, tx_kex; int blkaddr, entry, bank, err; - u64 cfg, nibble_ena; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) { @@ -1213,13 +1146,16 @@ int rvu_npc_init(struct rvu *rvu) /* Config Outer L2, IPv4's NPC layer info */ rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, - (NPC_LID_LA << 8) | (NPC_LT_LA_ETHER << 4) | 0x0F); + (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | + kpu->lt_def->pck_ol2.ltype_mask); rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, - (NPC_LID_LC << 8) | (NPC_LT_LC_IP << 4) | 0x0F); + (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | + kpu->lt_def->pck_oip4.ltype_mask); /* Config Inner IPV4 NPC layer info */ rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, - (NPC_LID_LG << 8) | (NPC_LT_LG_TU_IP << 4) | 0x0F); + (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | + kpu->lt_def->pck_iip4.ltype_mask); /* Enable below for Rx pkts. * - Outer IPv4 header checksum validation. @@ -1235,23 +1171,25 @@ int rvu_npc_init(struct rvu *rvu) /* Set RX and TX side MCAM search key size. * LA..LD (ltype only) + Channel */ - nibble_ena = 0x49247; - rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX), - ((keyz & 0x3) << 32) | nibble_ena); + rx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_RX]; + tx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_TX]; + nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX), rx_kex); /* Due to an errata (35786) in A0 pass silicon, parse nibble enable * configuration has to be identical for both Rx and Tx interfaces. */ - if (!is_rvu_96xx_B0(rvu)) - nibble_ena = (1ULL << 19) - 1; - rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX), - ((keyz & 0x3) << 32) | nibble_ena); + if (is_rvu_96xx_B0(rvu)) { + tx_kex &= ~NPC_PARSE_NIBBLE; + tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); + } + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX), tx_kex); err = npc_mcam_rsrcs_init(rvu, blkaddr); if (err) return err; /* Configure MKEX profile */ - npc_load_mkex_profile(rvu, blkaddr); + npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); /* Set TX miss action to UCAST_DEFAULT i.e * transmit the packet on NIX LF SQ's default channel. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 9826a041e407..b24aeee1db8c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -49,7 +49,8 @@ mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o eswitch_offlo ecpf.o rdma.o mlx5_core-$(CONFIG_MLX5_ESWITCH) += esw/acl/helper.o \ esw/acl/egress_lgcy.o esw/acl/egress_ofld.o \ - esw/acl/ingress_lgcy.o esw/acl/ingress_ofld.o + esw/acl/ingress_lgcy.o esw/acl/ingress_ofld.o \ + esw/devlink_port.o mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o mlx5_core-$(CONFIG_VXLAN) += lib/vxlan.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c index 8db4b5f0f963..291e427e9e4f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c @@ -56,8 +56,8 @@ static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev, size_t size, dma_addr_t *dma_handle, int node) { + struct device *device = mlx5_core_dma_dev(dev); struct mlx5_priv *priv = &dev->priv; - struct device *device = dev->device; int original_node; void *cpu_handle; @@ -111,7 +111,7 @@ EXPORT_SYMBOL(mlx5_buf_alloc); void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf) { - dma_free_coherent(dev->device, buf->size, buf->frags->buf, + dma_free_coherent(mlx5_core_dma_dev(dev), buf->size, buf->frags->buf, buf->frags->map); kfree(buf->frags); @@ -140,7 +140,7 @@ int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, if (!frag->buf) goto err_free_buf; if (frag->map & ((1 << buf->page_shift) - 1)) { - dma_free_coherent(dev->device, frag_sz, + dma_free_coherent(mlx5_core_dma_dev(dev), frag_sz, buf->frags[i].buf, buf->frags[i].map); mlx5_core_warn(dev, "unexpected map alignment: %pad, page_shift=%d\n", &frag->map, buf->page_shift); @@ -153,7 +153,7 @@ int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, err_free_buf: while (i--) - dma_free_coherent(dev->device, PAGE_SIZE, buf->frags[i].buf, + dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE, buf->frags[i].buf, buf->frags[i].map); kfree(buf->frags); err_out: @@ -169,7 +169,7 @@ void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf) for (i = 0; i < buf->npages; i++) { int frag_sz = min_t(int, size, PAGE_SIZE); - dma_free_coherent(dev->device, frag_sz, buf->frags[i].buf, + dma_free_coherent(mlx5_core_dma_dev(dev), frag_sz, buf->frags[i].buf, buf->frags[i].map); size -= frag_sz; } @@ -275,7 +275,7 @@ void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db) __set_bit(db->index, db->u.pgdir->bitmap); if (bitmap_full(db->u.pgdir->bitmap, db_per_page)) { - dma_free_coherent(dev->device, PAGE_SIZE, + dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE, db->u.pgdir->db_page, db->u.pgdir->db_dma); list_del(&db->u.pgdir->list); bitmap_free(db->u.pgdir->bitmap); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 1d91a0d0ab1d..1ccae653319f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -1899,9 +1899,7 @@ static void create_msg_cache(struct mlx5_core_dev *dev) static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) { - struct device *ddev = dev->device; - - cmd->cmd_alloc_buf = dma_alloc_coherent(ddev, MLX5_ADAPTER_PAGE_SIZE, + cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, &cmd->alloc_dma, GFP_KERNEL); if (!cmd->cmd_alloc_buf) return -ENOMEM; @@ -1914,9 +1912,9 @@ static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) return 0; } - dma_free_coherent(ddev, MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf, + dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf, cmd->alloc_dma); - cmd->cmd_alloc_buf = dma_alloc_coherent(ddev, + cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), 2 * MLX5_ADAPTER_PAGE_SIZE - 1, &cmd->alloc_dma, GFP_KERNEL); if (!cmd->cmd_alloc_buf) @@ -1930,9 +1928,7 @@ static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) { - struct device *ddev = dev->device; - - dma_free_coherent(ddev, cmd->alloc_size, cmd->cmd_alloc_buf, + dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf, cmd->alloc_dma); } @@ -1964,7 +1960,7 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) if (!cmd->stats) return -ENOMEM; - cmd->pool = dma_pool_create("mlx5_cmd", dev->device, size, align, 0); + cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0); if (!cmd->pool) { err = -ENOMEM; goto dma_pool_err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c index ad3594c4afcb..ede4640b8428 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c @@ -124,7 +124,7 @@ static void mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer *tracer) static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer *tracer) { struct mlx5_core_dev *dev = tracer->dev; - struct device *ddev = &dev->pdev->dev; + struct device *ddev; dma_addr_t dma; void *buff; gfp_t gfp; @@ -142,6 +142,7 @@ static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer *tracer) } tracer->buff.log_buf = buff; + ddev = mlx5_core_dma_dev(dev); dma = dma_map_single(ddev, buff, tracer->buff.size, DMA_FROM_DEVICE); if (dma_mapping_error(ddev, dma)) { mlx5_core_warn(dev, "FWTracer: Unable to map DMA: %d\n", @@ -162,11 +163,12 @@ free_pages: static void mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer *tracer) { struct mlx5_core_dev *dev = tracer->dev; - struct device *ddev = &dev->pdev->dev; + struct device *ddev; if (!tracer->buff.log_buf) return; + ddev = mlx5_core_dma_dev(dev); dma_unmap_single(ddev, tracer->buff.dma, tracer->buff.size, DMA_FROM_DEVICE); free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size)); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c index 4924a5658853..ed4fb79b4db7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c @@ -78,7 +78,7 @@ static int mlx5_rsc_dump_trigger(struct mlx5_core_dev *dev, struct mlx5_rsc_dump struct page *page) { struct mlx5_rsc_dump *rsc_dump = dev->rsc_dump; - struct device *ddev = &dev->pdev->dev; + struct device *ddev = mlx5_core_dma_dev(dev); u32 out_seq_num; u32 in_seq_num; dma_addr_t dma; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index b5f8ed30047b..cea2070af9af 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -739,6 +739,7 @@ mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_core_dev *dev = ct_priv->dev; struct mlx5_ct_entry *rev_entry; __be16 tmp_port; + int ret; /* get the reversed tuple */ tmp_port = rev_tuple.port.src; @@ -778,8 +779,9 @@ mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv, shared_counter->counter = mlx5_fc_create(dev, true); if (IS_ERR(shared_counter->counter)) { ct_dbg("Failed to create counter for ct entry"); + ret = PTR_ERR(shared_counter->counter); kfree(shared_counter); - return ERR_PTR(PTR_ERR(shared_counter->counter)); + return ERR_PTR(ret); } refcount_set(&shared_counter->refcount, 1); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c index 3503e7711178..71e8d66fa150 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c @@ -9,7 +9,7 @@ static int mlx5e_xsk_map_pool(struct mlx5e_priv *priv, struct xsk_buff_pool *pool) { - struct device *dev = priv->mdev->device; + struct device *dev = mlx5_core_dma_dev(priv->mdev); return xsk_pool_dma_map(pool, dev, 0); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c index 6bbfcf18107d..ccaccb9fc2f7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -253,7 +253,7 @@ resync_post_get_progress_params(struct mlx5e_icosq *sq, goto err_out; } - pdev = sq->channel->priv->mdev->device; + pdev = mlx5_core_dma_dev(sq->channel->priv->mdev); buf->dma_addr = dma_map_single(pdev, &buf->progress, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) { @@ -390,7 +390,7 @@ void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi, priv_rx = buf->priv_rx; resync = &priv_rx->resync; - dev = resync->priv->mdev->device; + dev = mlx5_core_dma_dev(resync->priv->mdev); if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) goto out; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 961cdce37cc4..2c3c594a8621 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -1943,7 +1943,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, c->tstamp = &priv->tstamp; c->ix = ix; c->cpu = cpu; - c->pdev = priv->mdev->device; + c->pdev = mlx5_core_dma_dev(priv->mdev); c->netdev = priv->netdev; c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key); c->num_tc = params->num_tc; @@ -2131,7 +2131,7 @@ void mlx5e_build_rq_param(struct mlx5e_priv *priv, MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); - param->wq.buf_numa_node = dev_to_node(mdev->device); + param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); mlx5e_build_rx_cq_param(priv, params, xsk, ¶m->cqp); } @@ -2147,7 +2147,7 @@ static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv, mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1)); MLX5_SET(rqc, rqc, counter_set_id, priv->drop_rq_q_counter); - param->wq.buf_numa_node = dev_to_node(mdev->device); + param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); } void mlx5e_build_sq_param_common(struct mlx5e_priv *priv, @@ -2159,7 +2159,7 @@ void mlx5e_build_sq_param_common(struct mlx5e_priv *priv, MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn); - param->wq.buf_numa_node = dev_to_node(priv->mdev->device); + param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(priv->mdev)); } static void mlx5e_build_sq_param(struct mlx5e_priv *priv, @@ -3197,8 +3197,8 @@ static int mlx5e_alloc_drop_cq(struct mlx5_core_dev *mdev, struct mlx5e_cq *cq, struct mlx5e_cq_param *param) { - param->wq.buf_numa_node = dev_to_node(mdev->device); - param->wq.db_numa_node = dev_to_node(mdev->device); + param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); + param->wq.db_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); return mlx5e_alloc_cq_common(mdev, param, cq); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 9f5c97d22af4..67247c33b9fd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -374,19 +374,6 @@ static const struct ethtool_ops mlx5e_uplink_rep_ethtool_ops = { .set_pauseparam = mlx5e_uplink_rep_set_pauseparam, }; -static void mlx5e_rep_get_port_parent_id(struct net_device *dev, - struct netdev_phys_item_id *ppid) -{ - struct mlx5e_priv *priv; - u64 parent_id; - - priv = netdev_priv(dev); - - parent_id = mlx5_query_nic_system_image_guid(priv->mdev); - ppid->id_len = sizeof(parent_id); - memcpy(ppid->id, &parent_id, sizeof(parent_id)); -} - static void mlx5e_sqs2vport_stop(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep) { @@ -611,12 +598,13 @@ static int mlx5e_uplink_rep_set_vf_vlan(struct net_device *dev, int vf, u16 vlan return 0; } -static struct devlink_port *mlx5e_rep_get_devlink_port(struct net_device *dev) +static struct devlink_port *mlx5e_rep_get_devlink_port(struct net_device *netdev) { - struct mlx5e_priv *priv = netdev_priv(dev); + struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5e_rep_priv *rpriv = priv->ppriv; + struct mlx5_core_dev *dev = priv->mdev; - return &rpriv->dl_port; + return mlx5_esw_offloads_devlink_port(dev->priv.eswitch, rpriv->rep->vport); } static int mlx5e_rep_change_carrier(struct net_device *dev, bool new_carrier) @@ -1206,70 +1194,13 @@ static const struct mlx5e_profile mlx5e_uplink_rep_profile = { .stats_grps_num = mlx5e_ul_rep_stats_grps_num, }; -static bool -is_devlink_port_supported(const struct mlx5_core_dev *dev, - const struct mlx5e_rep_priv *rpriv) -{ - return rpriv->rep->vport == MLX5_VPORT_UPLINK || - rpriv->rep->vport == MLX5_VPORT_PF || - mlx5_eswitch_is_vf_vport(dev->priv.eswitch, rpriv->rep->vport); -} - -static int register_devlink_port(struct mlx5_core_dev *dev, - struct mlx5e_rep_priv *rpriv) -{ - struct mlx5_esw_offload *offloads = &dev->priv.eswitch->offloads; - struct devlink *devlink = priv_to_devlink(dev); - struct mlx5_eswitch_rep *rep = rpriv->rep; - struct devlink_port_attrs attrs = {}; - struct netdev_phys_item_id ppid = {}; - unsigned int dl_port_index = 0; - u32 controller_num = 0; - bool external; - u16 pfnum; - - if (!is_devlink_port_supported(dev, rpriv)) - return 0; - - external = mlx5_core_is_ecpf_esw_manager(dev); - if (external) - controller_num = offloads->host_number + 1; - mlx5e_rep_get_port_parent_id(rpriv->netdev, &ppid); - dl_port_index = mlx5_esw_vport_to_devlink_port_index(dev, rep->vport); - pfnum = PCI_FUNC(dev->pdev->devfn); - if (rep->vport == MLX5_VPORT_UPLINK) { - attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; - attrs.phys.port_number = pfnum; - memcpy(attrs.switch_id.id, &ppid.id[0], ppid.id_len); - attrs.switch_id.id_len = ppid.id_len; - devlink_port_attrs_set(&rpriv->dl_port, &attrs); - } else if (rep->vport == MLX5_VPORT_PF) { - memcpy(rpriv->dl_port.attrs.switch_id.id, &ppid.id[0], ppid.id_len); - rpriv->dl_port.attrs.switch_id.id_len = ppid.id_len; - devlink_port_attrs_pci_pf_set(&rpriv->dl_port, controller_num, - pfnum, external); - } else if (mlx5_eswitch_is_vf_vport(dev->priv.eswitch, rpriv->rep->vport)) { - memcpy(rpriv->dl_port.attrs.switch_id.id, &ppid.id[0], ppid.id_len); - rpriv->dl_port.attrs.switch_id.id_len = ppid.id_len; - devlink_port_attrs_pci_vf_set(&rpriv->dl_port, controller_num, - pfnum, rep->vport - 1, external); - } - return devlink_port_register(devlink, &rpriv->dl_port, dl_port_index); -} - -static void unregister_devlink_port(struct mlx5_core_dev *dev, - struct mlx5e_rep_priv *rpriv) -{ - if (is_devlink_port_supported(dev, rpriv)) - devlink_port_unregister(&rpriv->dl_port); -} - /* e-Switch vport representors */ static int mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) { const struct mlx5e_profile *profile; struct mlx5e_rep_priv *rpriv; + struct devlink_port *dl_port; struct net_device *netdev; int nch, err; @@ -1319,28 +1250,19 @@ mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) goto err_detach_netdev; } - err = register_devlink_port(dev, rpriv); - if (err) { - netdev_warn(netdev, "Failed to register devlink port %d\n", - rep->vport); - goto err_neigh_cleanup; - } - err = register_netdev(netdev); if (err) { netdev_warn(netdev, "Failed to register representor netdev for vport %d\n", rep->vport); - goto err_devlink_cleanup; + goto err_neigh_cleanup; } - if (is_devlink_port_supported(dev, rpriv)) - devlink_port_type_eth_set(&rpriv->dl_port, netdev); + dl_port = mlx5_esw_offloads_devlink_port(dev->priv.eswitch, rpriv->rep->vport); + if (dl_port) + devlink_port_type_eth_set(dl_port, netdev); return 0; -err_devlink_cleanup: - unregister_devlink_port(dev, rpriv); - err_neigh_cleanup: mlx5e_rep_neigh_cleanup(rpriv); @@ -1364,12 +1286,13 @@ mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep) struct net_device *netdev = rpriv->netdev; struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5_core_dev *dev = priv->mdev; + struct devlink_port *dl_port; void *ppriv = priv->ppriv; - if (is_devlink_port_supported(dev, rpriv)) - devlink_port_type_clear(&rpriv->dl_port); + dl_port = mlx5_esw_offloads_devlink_port(dev->priv.eswitch, rpriv->rep->vport); + if (dl_port) + devlink_port_type_clear(dl_port); unregister_netdev(netdev); - unregister_devlink_port(dev, rpriv); mlx5e_rep_neigh_cleanup(rpriv); mlx5e_detach_netdev(priv); if (rep->vport == MLX5_VPORT_UPLINK) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h index 622c27ae4ac7..8054d92ae37e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h @@ -101,7 +101,6 @@ struct mlx5e_rep_priv { struct list_head vport_sqs_list; struct mlx5_rep_uplink_priv uplink_priv; /* valid for uplink rep */ struct rtnl_link_stats64 prev_vf_vport_stats; - struct devlink_port dl_port; }; static inline diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index f815b0c60a6c..a0c356987e1a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -1238,8 +1238,10 @@ mlx5e_tc_unoffload_from_slow_path(struct mlx5_eswitch *esw, struct mlx5_flow_attr *slow_attr; slow_attr = mlx5_alloc_flow_attr(MLX5_FLOW_NAMESPACE_FDB); - if (!slow_attr) - mlx5_core_warn(flow->priv->mdev, "Unable to unoffload slow path rule\n"); + if (!slow_attr) { + mlx5_core_warn(flow->priv->mdev, "Unable to alloc attr to unoffload slow path rule\n"); + return; + } memcpy(slow_attr, flow->attr, ESW_FLOW_ATTR_SZ); slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; @@ -4534,20 +4536,22 @@ mlx5e_alloc_flow(struct mlx5e_priv *priv, int attr_size, struct mlx5e_tc_flow_parse_attr *parse_attr; struct mlx5_flow_attr *attr; struct mlx5e_tc_flow *flow; - int out_index, err; + int err = -ENOMEM; + int out_index; flow = kzalloc(sizeof(*flow), GFP_KERNEL); parse_attr = kvzalloc(sizeof(*parse_attr), GFP_KERNEL); + if (!parse_attr || !flow) + goto err_free; flow->flags = flow_flags; flow->cookie = f->cookie; flow->priv = priv; attr = mlx5_alloc_flow_attr(get_flow_name_space(flow)); - if (!parse_attr || !flow || !attr) { - err = -ENOMEM; + if (!attr) goto err_free; - } + flow->attr = attr; for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/egress_ofld.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/egress_ofld.c index 07b2acd7e6b3..c3faae67e4d6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/egress_ofld.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/egress_ofld.c @@ -148,6 +148,11 @@ static void esw_acl_egress_ofld_groups_destroy(struct mlx5_vport *vport) esw_acl_egress_vlan_grp_destroy(vport); } +static bool esw_acl_egress_needed(const struct mlx5_eswitch *esw, u16 vport_num) +{ + return mlx5_eswitch_is_vf_vport(esw, vport_num); +} + int esw_acl_egress_ofld_setup(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { int table_size = 0; @@ -157,6 +162,9 @@ int esw_acl_egress_ofld_setup(struct mlx5_eswitch *esw, struct mlx5_vport *vport !MLX5_CAP_GEN(esw->dev, prio_tag_required)) return 0; + if (!esw_acl_egress_needed(esw, vport->vport)) + return 0; + esw_acl_egress_ofld_rules_destroy(vport); if (mlx5_esw_acl_egress_fwd2vport_supported(esw)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c new file mode 100644 index 000000000000..ffff11baa3d0 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020 Mellanox Technologies Ltd. */ + +#include <linux/mlx5/driver.h> +#include "eswitch.h" + +static void +mlx5_esw_get_port_parent_id(struct mlx5_core_dev *dev, struct netdev_phys_item_id *ppid) +{ + u64 parent_id; + + parent_id = mlx5_query_nic_system_image_guid(dev); + ppid->id_len = sizeof(parent_id); + memcpy(ppid->id, &parent_id, sizeof(parent_id)); +} + +static bool +mlx5_esw_devlink_port_supported(const struct mlx5_eswitch *esw, u16 vport_num) +{ + return vport_num == MLX5_VPORT_UPLINK || + (mlx5_core_is_ecpf(esw->dev) && vport_num == MLX5_VPORT_PF) || + mlx5_eswitch_is_vf_vport(esw, vport_num); +} + +static struct devlink_port *mlx5_esw_dl_port_alloc(struct mlx5_eswitch *esw, u16 vport_num) +{ + struct mlx5_core_dev *dev = esw->dev; + struct devlink_port_attrs attrs = {}; + struct netdev_phys_item_id ppid = {}; + struct devlink_port *dl_port; + u32 controller_num = 0; + bool external; + u16 pfnum; + + dl_port = kzalloc(sizeof(*dl_port), GFP_KERNEL); + if (!dl_port) + return NULL; + + mlx5_esw_get_port_parent_id(dev, &ppid); + pfnum = PCI_FUNC(dev->pdev->devfn); + external = mlx5_core_is_ecpf_esw_manager(dev); + if (external) + controller_num = dev->priv.eswitch->offloads.host_number + 1; + + if (vport_num == MLX5_VPORT_UPLINK) { + attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; + attrs.phys.port_number = pfnum; + memcpy(attrs.switch_id.id, ppid.id, ppid.id_len); + attrs.switch_id.id_len = ppid.id_len; + devlink_port_attrs_set(dl_port, &attrs); + } else if (vport_num == MLX5_VPORT_PF) { + memcpy(dl_port->attrs.switch_id.id, ppid.id, ppid.id_len); + dl_port->attrs.switch_id.id_len = ppid.id_len; + devlink_port_attrs_pci_pf_set(dl_port, controller_num, pfnum, external); + } else if (mlx5_eswitch_is_vf_vport(esw, vport_num)) { + memcpy(dl_port->attrs.switch_id.id, ppid.id, ppid.id_len); + dl_port->attrs.switch_id.id_len = ppid.id_len; + devlink_port_attrs_pci_vf_set(dl_port, controller_num, pfnum, + vport_num - 1, external); + } + return dl_port; +} + +static void mlx5_esw_dl_port_free(struct devlink_port *dl_port) +{ + kfree(dl_port); +} + +int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, u16 vport_num) +{ + struct mlx5_core_dev *dev = esw->dev; + struct devlink_port *dl_port; + unsigned int dl_port_index; + struct mlx5_vport *vport; + struct devlink *devlink; + int err; + + if (!mlx5_esw_devlink_port_supported(esw, vport_num)) + return 0; + + vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) + return PTR_ERR(vport); + + dl_port = mlx5_esw_dl_port_alloc(esw, vport_num); + if (!dl_port) + return -ENOMEM; + + devlink = priv_to_devlink(dev); + dl_port_index = mlx5_esw_vport_to_devlink_port_index(dev, vport_num); + err = devlink_port_register(devlink, dl_port, dl_port_index); + if (err) + goto reg_err; + + vport->dl_port = dl_port; + return 0; + +reg_err: + mlx5_esw_dl_port_free(dl_port); + return err; +} + +void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_eswitch *esw, u16 vport_num) +{ + struct mlx5_vport *vport; + + if (!mlx5_esw_devlink_port_supported(esw, vport_num)) + return; + + vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) + return; + devlink_port_unregister(vport->dl_port); + mlx5_esw_dl_port_free(vport->dl_port); + vport->dl_port = NULL; +} + +struct devlink_port *mlx5_esw_offloads_devlink_port(struct mlx5_eswitch *esw, u16 vport_num) +{ + struct mlx5_vport *vport; + + vport = mlx5_eswitch_get_vport(esw, vport_num); + return vport->dl_port; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 66393fbdcd94..cf87de94418f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -156,6 +156,7 @@ struct mlx5_vport { bool enabled; enum mlx5_eswitch_vport_event enabled_events; + struct devlink_port *dl_port; }; struct mlx5_eswitch_fdb { @@ -663,6 +664,9 @@ int mlx5_eswitch_load_vf_vports(struct mlx5_eswitch *esw, u16 num_vfs, enum mlx5_eswitch_vport_event enabled_events); void mlx5_eswitch_unload_vf_vports(struct mlx5_eswitch *esw, u16 num_vfs); +int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, u16 vport_num); +void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_eswitch *esw, u16 vport_num); +struct devlink_port *mlx5_esw_offloads_devlink_port(struct mlx5_eswitch *esw, u16 vport_num); #else /* CONFIG_MLX5_ESWITCH */ /* eswitch API stubs */ static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index ffd5d540a19e..c9c2962ad49f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -247,8 +247,11 @@ mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *attr) { if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) && - attr && attr->in_rep && attr->in_rep->vport == MLX5_VPORT_UPLINK) - spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; + attr && attr->in_rep) + spec->flow_context.flow_source = + attr->in_rep->vport == MLX5_VPORT_UPLINK ? + MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : + MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; } static void @@ -1820,15 +1823,12 @@ static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) __esw_offloads_unload_rep(esw, rep, rep_type); } -int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num) +static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) { struct mlx5_eswitch_rep *rep; int rep_type; int err; - if (esw->mode != MLX5_ESWITCH_OFFLOADS) - return 0; - rep = mlx5_eswitch_get_rep(esw, vport_num); for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) if (atomic_cmpxchg(&rep->rep_data[rep_type].state, @@ -1847,19 +1847,46 @@ err_reps: return err; } -void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num) +static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) { struct mlx5_eswitch_rep *rep; int rep_type; - if (esw->mode != MLX5_ESWITCH_OFFLOADS) - return; - rep = mlx5_eswitch_get_rep(esw, vport_num); for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) __esw_offloads_unload_rep(esw, rep, rep_type); } +int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num) +{ + int err; + + if (esw->mode != MLX5_ESWITCH_OFFLOADS) + return 0; + + err = mlx5_esw_offloads_devlink_port_register(esw, vport_num); + if (err) + return err; + + err = mlx5_esw_offloads_rep_load(esw, vport_num); + if (err) + goto load_err; + return err; + +load_err: + mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); + return err; +} + +void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num) +{ + if (esw->mode != MLX5_ESWITCH_OFFLOADS) + return; + + mlx5_esw_offloads_rep_unload(esw, vport_num); + mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); +} + #define ESW_OFFLOADS_DEVCOM_PAIR (0) #define ESW_OFFLOADS_DEVCOM_UNPAIR (1) @@ -2019,31 +2046,31 @@ esw_check_vport_match_metadata_supported(const struct mlx5_eswitch *esw) u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) { - u32 num_vports = GENMASK(ESW_VPORT_BITS - 1, 0) - 1; - u32 vhca_id_mask = GENMASK(ESW_VHCA_ID_BITS - 1, 0); - u32 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); - u32 start; - u32 end; + u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; + u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 1; + u32 pf_num; int id; - /* Make sure the vhca_id fits the ESW_VHCA_ID_BITS */ - WARN_ON_ONCE(vhca_id >= BIT(ESW_VHCA_ID_BITS)); - - /* Trim vhca_id to ESW_VHCA_ID_BITS */ - vhca_id &= vhca_id_mask; - - start = (vhca_id << ESW_VPORT_BITS); - end = start + num_vports; - if (!vhca_id) - start += 1; /* zero is reserved/invalid metadata */ - id = ida_alloc_range(&esw->offloads.vport_metadata_ida, start, end, GFP_KERNEL); + /* Only 4 bits of pf_num */ + pf_num = PCI_FUNC(esw->dev->pdev->devfn); + if (pf_num > max_pf_num) + return 0; - return (id < 0) ? 0 : id; + /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ + /* Use only non-zero vport_id (1-4095) for all PF's */ + id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 1, vport_end_ida, GFP_KERNEL); + if (id < 0) + return 0; + id = (pf_num << ESW_VPORT_BITS) | id; + return id; } void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) { - ida_free(&esw->offloads.vport_metadata_ida, metadata); + u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; + + /* Metadata contains only 12 bits of actual ida id */ + ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); } static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, @@ -2108,11 +2135,9 @@ esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, if (err) return err; - if (mlx5_eswitch_is_vf_vport(esw, vport->vport)) { - err = esw_acl_egress_ofld_setup(esw, vport); - if (err) - goto egress_err; - } + err = esw_acl_egress_ofld_setup(esw, vport); + if (err) + goto egress_err; return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c index 9f6d97eae0ae..80da50e12915 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c @@ -54,7 +54,7 @@ static int mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn *conn, if (unlikely(!buf->sg[0].data)) goto out; - dma_device = &conn->fdev->mdev->pdev->dev; + dma_device = mlx5_core_dma_dev(conn->fdev->mdev); buf->sg[0].dma_addr = dma_map_single(dma_device, buf->sg[0].data, buf->sg[0].size, buf->dma_dir); err = dma_mapping_error(dma_device, buf->sg[0].dma_addr); @@ -86,7 +86,7 @@ static void mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn *conn, { struct device *dma_device; - dma_device = &conn->fdev->mdev->pdev->dev; + dma_device = mlx5_core_dma_dev(conn->fdev->mdev); if (buf->sg[1].data) dma_unmap_single(dma_device, buf->sg[1].dma_addr, buf->sg[1].size, buf->dma_dir); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index ce43e3feccd9..3bd32c05ae71 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -739,7 +739,7 @@ static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev, pci_set_drvdata(dev->pdev, dev); dev->bar_addr = pci_resource_start(pdev, 0); - priv->numa_node = dev_to_node(&dev->pdev->dev); + priv->numa_node = dev_to_node(mlx5_core_dma_dev(dev)); err = mlx5_pci_enable_device(dev); if (err) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index fc1649dac11b..e994f84f50b6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -100,6 +100,11 @@ do { \ __func__, __LINE__, current->pid, \ ##__VA_ARGS__) +static inline struct device *mlx5_core_dma_dev(struct mlx5_core_dev *dev) +{ + return &dev->pdev->dev; +} + enum { MLX5_CMD_DATA, /* print command payload only */ MLX5_CMD_TIME, /* print command execution time */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index f9b798af6335..0809b2a14319 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -238,7 +238,7 @@ static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp, rb_erase(&fwp->rb_node, root); if (in_free_list) list_del(&fwp->list); - dma_unmap_page(dev->device, fwp->addr & MLX5_U64_4K_PAGE_MASK, + dma_unmap_page(mlx5_core_dma_dev(dev), fwp->addr & MLX5_U64_4K_PAGE_MASK, PAGE_SIZE, DMA_BIDIRECTIONAL); __free_page(fwp->page); kfree(fwp); @@ -265,7 +265,7 @@ static void free_4k(struct mlx5_core_dev *dev, u64 addr, u32 func_id) static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) { - struct device *device = dev->device; + struct device *device = mlx5_core_dma_dev(dev); int nid = dev_to_node(device); struct page *page; u64 zero_addr = 1; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c index c63f727273d8..7df883686d46 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c @@ -203,7 +203,6 @@ static int dr_matcher_set_ste_builders(struct mlx5dr_matcher *matcher, struct mlx5dr_domain_rx_tx *nic_dmn = nic_matcher->nic_tbl->nic_dmn; struct mlx5dr_domain *dmn = matcher->tbl->dmn; struct mlx5dr_match_param mask = {}; - struct mlx5dr_match_misc3 *misc3; struct mlx5dr_ste_build *sb; bool inner, rx; int idx = 0; @@ -252,18 +251,14 @@ static int dr_matcher_set_ste_builders(struct mlx5dr_matcher *matcher, if (dr_mask_is_gvmi_or_qpn_set(&mask.misc) && (dmn->type == MLX5DR_DOMAIN_TYPE_FDB || dmn->type == MLX5DR_DOMAIN_TYPE_NIC_RX)) { - ret = mlx5dr_ste_build_src_gvmi_qpn(&sb[idx++], &mask, - dmn, inner, rx); - if (ret) - return ret; + mlx5dr_ste_build_src_gvmi_qpn(&sb[idx++], &mask, + dmn, inner, rx); } if (dr_mask_is_smac_set(&mask.outer) && dr_mask_is_dmac_set(&mask.outer)) { - ret = mlx5dr_ste_build_eth_l2_src_des(&sb[idx++], &mask, - inner, rx); - if (ret) - return ret; + mlx5dr_ste_build_eth_l2_src_des(&sb[idx++], &mask, + inner, rx); } if (dr_mask_is_smac_set(&mask.outer)) @@ -313,8 +308,7 @@ static int dr_matcher_set_ste_builders(struct mlx5dr_matcher *matcher, mlx5dr_ste_build_flex_parser_0(&sb[idx++], &mask, inner, rx); - misc3 = &mask.misc3; - if ((DR_MASK_IS_FLEX_PARSER_ICMPV4_SET(misc3) && + if ((DR_MASK_IS_FLEX_PARSER_ICMPV4_SET(&mask.misc3) && mlx5dr_matcher_supp_flex_parser_icmp_v4(&dmn->info.caps)) || (dr_mask_is_flex_parser_icmpv6_set(&mask.misc3) && mlx5dr_matcher_supp_flex_parser_icmp_v6(&dmn->info.caps))) { @@ -340,10 +334,8 @@ static int dr_matcher_set_ste_builders(struct mlx5dr_matcher *matcher, if (dr_mask_is_smac_set(&mask.inner) && dr_mask_is_dmac_set(&mask.inner)) { - ret = mlx5dr_ste_build_eth_l2_src_des(&sb[idx++], - &mask, inner, rx); - if (ret) - return ret; + mlx5dr_ste_build_eth_l2_src_des(&sb[idx++], + &mask, inner, rx); } if (dr_mask_is_smac_set(&mask.inner)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c index 6ec5106bc472..b3c9dc032026 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c @@ -242,7 +242,7 @@ dr_rule_rehash_copy_ste(struct mlx5dr_matcher *matcher, new_idx = mlx5dr_ste_calc_hash_index(hw_ste, new_htbl); new_ste = &new_htbl->ste_arr[new_idx]; - if (mlx5dr_ste_not_used_ste(new_ste)) { + if (mlx5dr_ste_is_not_used(new_ste)) { mlx5dr_htbl_get(new_htbl); list_add_tail(&new_ste->miss_list_node, mlx5dr_ste_get_miss_list(new_ste)); @@ -335,7 +335,7 @@ static int dr_rule_rehash_copy_htbl(struct mlx5dr_matcher *matcher, for (i = 0; i < cur_entries; i++) { cur_ste = &cur_htbl->ste_arr[i]; - if (mlx5dr_ste_not_used_ste(cur_ste)) /* Empty, nothing to copy */ + if (mlx5dr_ste_is_not_used(cur_ste)) /* Empty, nothing to copy */ continue; err = dr_rule_rehash_copy_miss_list(matcher, @@ -791,7 +791,7 @@ again: miss_list = &cur_htbl->chunk->miss_list[index]; ste = &cur_htbl->ste_arr[index]; - if (mlx5dr_ste_not_used_ste(ste)) { + if (mlx5dr_ste_is_not_used(ste)) { if (dr_rule_handle_empty_entry(matcher, nic_matcher, cur_htbl, ste, ste_location, hw_ste, miss_list, @@ -985,31 +985,28 @@ static enum mlx5dr_ipv dr_rule_get_ipv(struct mlx5dr_match_spec *spec) static bool dr_rule_skip(enum mlx5dr_domain_type domain, enum mlx5dr_ste_entry_type ste_type, struct mlx5dr_match_param *mask, - struct mlx5dr_match_param *value) + struct mlx5dr_match_param *value, + u32 flow_source) { + bool rx = ste_type == MLX5DR_STE_TYPE_RX; + if (domain != MLX5DR_DOMAIN_TYPE_FDB) return false; if (mask->misc.source_port) { - if (ste_type == MLX5DR_STE_TYPE_RX) - if (value->misc.source_port != WIRE_PORT) - return true; + if (rx && value->misc.source_port != WIRE_PORT) + return true; - if (ste_type == MLX5DR_STE_TYPE_TX) - if (value->misc.source_port == WIRE_PORT) - return true; + if (!rx && value->misc.source_port == WIRE_PORT) + return true; } - /* Metadata C can be used to describe the source vport */ - if (mask->misc2.metadata_reg_c_0) { - if (ste_type == MLX5DR_STE_TYPE_RX) - if ((value->misc2.metadata_reg_c_0 & WIRE_PORT) != WIRE_PORT) - return true; + if (rx && flow_source == MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT) + return true; + + if (!rx && flow_source == MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK) + return true; - if (ste_type == MLX5DR_STE_TYPE_TX) - if ((value->misc2.metadata_reg_c_0 & WIRE_PORT) == WIRE_PORT) - return true; - } return false; } @@ -1038,7 +1035,8 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule, INIT_LIST_HEAD(&nic_rule->rule_members_list); - if (dr_rule_skip(dmn->type, nic_dmn->ste_type, &matcher->mask, param)) + if (dr_rule_skip(dmn->type, nic_dmn->ste_type, &matcher->mask, param, + rule->flow_source)) return 0; hw_ste_arr = kzalloc(DR_RULE_MAX_STE_CHAIN * DR_STE_SIZE, GFP_KERNEL); @@ -1173,7 +1171,8 @@ static struct mlx5dr_rule * dr_rule_create_rule(struct mlx5dr_matcher *matcher, struct mlx5dr_match_parameters *value, size_t num_actions, - struct mlx5dr_action *actions[]) + struct mlx5dr_action *actions[], + u32 flow_source) { struct mlx5dr_domain *dmn = matcher->tbl->dmn; struct mlx5dr_match_param param = {}; @@ -1188,6 +1187,7 @@ dr_rule_create_rule(struct mlx5dr_matcher *matcher, return NULL; rule->matcher = matcher; + rule->flow_source = flow_source; INIT_LIST_HEAD(&rule->rule_actions_list); ret = dr_rule_add_action_members(rule, num_actions, actions); @@ -1232,13 +1232,14 @@ free_rule: struct mlx5dr_rule *mlx5dr_rule_create(struct mlx5dr_matcher *matcher, struct mlx5dr_match_parameters *value, size_t num_actions, - struct mlx5dr_action *actions[]) + struct mlx5dr_action *actions[], + u32 flow_source) { struct mlx5dr_rule *rule; refcount_inc(&matcher->refcount); - rule = dr_rule_create_rule(matcher, value, num_actions, actions); + rule = dr_rule_create_rule(matcher, value, num_actions, actions, flow_source); if (!rule) refcount_dec(&matcher->refcount); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c index 2ca79b9bde1f..24dede1b0a20 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c @@ -466,10 +466,10 @@ int mlx5dr_send_postsend_htbl(struct mlx5dr_domain *dmn, * need to add the bit_mask */ for (j = 0; j < num_stes_per_iter; j++) { - u8 *hw_ste = htbl->ste_arr[ste_index + j].hw_ste; + struct mlx5dr_ste *ste = &htbl->ste_arr[ste_index + j]; u32 ste_off = j * DR_STE_SIZE; - if (mlx5dr_ste_is_not_valid_entry(hw_ste)) { + if (mlx5dr_ste_is_not_used(ste)) { memcpy(data + ste_off, formatted_ste, DR_STE_SIZE); } else { @@ -831,7 +831,7 @@ static struct mlx5dr_mr *dr_reg_mr(struct mlx5_core_dev *mdev, if (!mr) return NULL; - dma_device = &mdev->pdev->dev; + dma_device = mlx5_core_dma_dev(mdev); dma_addr = dma_map_single(dma_device, buf, size, DMA_BIDIRECTIONAL); err = dma_mapping_error(dma_device, dma_addr); @@ -860,7 +860,7 @@ static struct mlx5dr_mr *dr_reg_mr(struct mlx5_core_dev *mdev, static void dr_dereg_mr(struct mlx5_core_dev *mdev, struct mlx5dr_mr *mr) { mlx5_core_destroy_mkey(mdev, &mr->mkey); - dma_unmap_single(&mdev->pdev->dev, mr->dma_addr, mr->size, + dma_unmap_single(mlx5_core_dma_dev(mdev), mr->dma_addr, mr->size, DMA_BIDIRECTIONAL); kfree(mr); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c index 00c2f598f034..b01aaec75622 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c @@ -155,6 +155,13 @@ static u16 dr_ste_conv_bit_to_byte_mask(u8 *bit_mask) return byte_mask; } +static u8 *mlx5dr_ste_get_tag(u8 *hw_ste_p) +{ + struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; + + return hw_ste->tag; +} + void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask) { struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; @@ -549,25 +556,6 @@ void mlx5dr_ste_always_miss_addr(struct mlx5dr_ste *ste, u64 miss_addr) dr_ste_set_always_miss((struct dr_hw_ste_format *)ste->hw_ste); } -/* The assumption here is that we don't update the ste->hw_ste if it is not - * used ste, so it will be all zero, checking the next_lu_type. - */ -bool mlx5dr_ste_is_not_valid_entry(u8 *p_hw_ste) -{ - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)p_hw_ste; - - if (MLX5_GET(ste_general, hw_ste, next_lu_type) == - MLX5DR_STE_LU_TYPE_NOP) - return true; - - return false; -} - -bool mlx5dr_ste_not_used_ste(struct mlx5dr_ste *ste) -{ - return !ste->refcount; -} - /* Init one ste as a pattern for ste data array */ void mlx5dr_ste_set_formatted_ste(u16 gvmi, struct mlx5dr_domain_rx_tx *nic_dmn, @@ -728,7 +716,14 @@ int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn, { if (!value && (match_criteria & DR_MATCHER_CRITERIA_MISC)) { if (mask->misc.source_port && mask->misc.source_port != 0xffff) { - mlx5dr_err(dmn, "Partial mask source_port is not supported\n"); + mlx5dr_err(dmn, + "Partial mask source_port is not supported\n"); + return -EINVAL; + } + if (mask->misc.source_eswitch_owner_vhca_id && + mask->misc.source_eswitch_owner_vhca_id != 0xffff) { + mlx5dr_err(dmn, + "Partial mask source_eswitch_owner_vhca_id is not supported\n"); return -EINVAL; } } @@ -760,7 +755,7 @@ int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher, mlx5dr_ste_set_bit_mask(ste_arr, sb->bit_mask); - ret = sb->ste_build_tag_func(value, sb, ste_arr); + ret = sb->ste_build_tag_func(value, sb, mlx5dr_ste_get_tag(ste_arr)); if (ret) return ret; @@ -778,8 +773,8 @@ int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher, return 0; } -static int dr_ste_build_eth_l2_src_des_bit_mask(struct mlx5dr_match_param *value, - bool inner, u8 *bit_mask) +static void dr_ste_build_eth_l2_src_des_bit_mask(struct mlx5dr_match_param *value, + bool inner, u8 *bit_mask) { struct mlx5dr_match_spec *mask = inner ? &value->inner : &value->outer; @@ -807,13 +802,6 @@ static int dr_ste_build_eth_l2_src_des_bit_mask(struct mlx5dr_match_param *value MLX5_SET(ste_eth_l2_src_dst, bit_mask, first_vlan_qualifier, -1); mask->svlan_tag = 0; } - - if (mask->cvlan_tag || mask->svlan_tag) { - pr_info("Invalid c/svlan mask configuration\n"); - return -EINVAL; - } - - return 0; } static void dr_ste_copy_mask_misc(char *mask, struct mlx5dr_match_misc *spec) @@ -1059,11 +1047,9 @@ void mlx5dr_ste_copy_param(u8 match_criteria, static int dr_ste_build_eth_l2_src_des_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { struct mlx5dr_match_spec *spec = sb->inner ? &value->inner : &value->outer; - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(eth_l2_src_dst, tag, dmac_47_16, spec, dmac_47_16); DR_STE_SET_TAG(eth_l2_src_dst, tag, dmac_15_0, spec, dmac_15_0); @@ -1104,23 +1090,17 @@ static int dr_ste_build_eth_l2_src_des_tag(struct mlx5dr_match_param *value, return 0; } -int mlx5dr_ste_build_eth_l2_src_des(struct mlx5dr_ste_build *sb, - struct mlx5dr_match_param *mask, - bool inner, bool rx) +void mlx5dr_ste_build_eth_l2_src_des(struct mlx5dr_ste_build *sb, + struct mlx5dr_match_param *mask, + bool inner, bool rx) { - int ret; - - ret = dr_ste_build_eth_l2_src_des_bit_mask(mask, inner, sb->bit_mask); - if (ret) - return ret; + dr_ste_build_eth_l2_src_des_bit_mask(mask, inner, sb->bit_mask); sb->rx = rx; sb->inner = inner; sb->lu_type = DR_STE_CALC_LU_TYPE(ETHL2_SRC_DST, rx, inner); sb->byte_mask = dr_ste_conv_bit_to_byte_mask(sb->bit_mask); sb->ste_build_tag_func = &dr_ste_build_eth_l2_src_des_tag; - - return 0; } static void dr_ste_build_eth_l3_ipv6_dst_bit_mask(struct mlx5dr_match_param *value, @@ -1136,11 +1116,9 @@ static void dr_ste_build_eth_l3_ipv6_dst_bit_mask(struct mlx5dr_match_param *val static int dr_ste_build_eth_l3_ipv6_dst_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_spec *spec = sb->inner ? &value->inner : &value->outer; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(eth_l3_ipv6_dst, tag, dst_ip_127_96, spec, dst_ip_127_96); DR_STE_SET_TAG(eth_l3_ipv6_dst, tag, dst_ip_95_64, spec, dst_ip_95_64); @@ -1176,11 +1154,9 @@ static void dr_ste_build_eth_l3_ipv6_src_bit_mask(struct mlx5dr_match_param *val static int dr_ste_build_eth_l3_ipv6_src_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { struct mlx5dr_match_spec *spec = sb->inner ? &value->inner : &value->outer; - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(eth_l3_ipv6_src, tag, src_ip_127_96, spec, src_ip_127_96); DR_STE_SET_TAG(eth_l3_ipv6_src, tag, src_ip_95_64, spec, src_ip_95_64); @@ -1238,11 +1214,9 @@ static void dr_ste_build_eth_l3_ipv4_5_tuple_bit_mask(struct mlx5dr_match_param static int dr_ste_build_eth_l3_ipv4_5_tuple_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_spec *spec = sb->inner ? &value->inner : &value->outer; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(eth_l3_ipv4_5_tuple, tag, destination_address, spec, dst_ip_31_0); DR_STE_SET_TAG(eth_l3_ipv4_5_tuple, tag, source_address, spec, src_ip_31_0); @@ -1328,12 +1302,10 @@ dr_ste_build_eth_l2_src_or_dst_bit_mask(struct mlx5dr_match_param *value, } static int dr_ste_build_eth_l2_src_or_dst_tag(struct mlx5dr_match_param *value, - bool inner, u8 *hw_ste_p) + bool inner, u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_spec *spec = inner ? &value->inner : &value->outer; struct mlx5dr_match_misc *misc_spec = &value->misc; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(eth_l2_src, tag, first_vlan_id, spec, first_vid); DR_STE_SET_TAG(eth_l2_src, tag, first_cfi, spec, first_cfi); @@ -1403,16 +1375,14 @@ static void dr_ste_build_eth_l2_src_bit_mask(struct mlx5dr_match_param *value, static int dr_ste_build_eth_l2_src_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_spec *spec = sb->inner ? &value->inner : &value->outer; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(eth_l2_src, tag, smac_47_16, spec, smac_47_16); DR_STE_SET_TAG(eth_l2_src, tag, smac_15_0, spec, smac_15_0); - return dr_ste_build_eth_l2_src_or_dst_tag(value, sb->inner, hw_ste_p); + return dr_ste_build_eth_l2_src_or_dst_tag(value, sb->inner, tag); } void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_build *sb, @@ -1440,16 +1410,14 @@ static void dr_ste_build_eth_l2_dst_bit_mask(struct mlx5dr_match_param *value, static int dr_ste_build_eth_l2_dst_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_spec *spec = sb->inner ? &value->inner : &value->outer; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(eth_l2_dst, tag, dmac_47_16, spec, dmac_47_16); DR_STE_SET_TAG(eth_l2_dst, tag, dmac_15_0, spec, dmac_15_0); - return dr_ste_build_eth_l2_src_or_dst_tag(value, sb->inner, hw_ste_p); + return dr_ste_build_eth_l2_src_or_dst_tag(value, sb->inner, tag); } void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_build *sb, @@ -1495,12 +1463,10 @@ static void dr_ste_build_eth_l2_tnl_bit_mask(struct mlx5dr_match_param *value, static int dr_ste_build_eth_l2_tnl_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { struct mlx5dr_match_spec *spec = sb->inner ? &value->inner : &value->outer; - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc *misc = &value->misc; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(eth_l2_tnl, tag, dmac_47_16, spec, dmac_47_16); DR_STE_SET_TAG(eth_l2_tnl, tag, dmac_15_0, spec, dmac_15_0); @@ -1561,11 +1527,9 @@ static void dr_ste_build_eth_l3_ipv4_misc_bit_mask(struct mlx5dr_match_param *va static int dr_ste_build_eth_l3_ipv4_misc_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_spec *spec = sb->inner ? &value->inner : &value->outer; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(eth_l3_ipv4_misc, tag, time_to_live, spec, ttl_hoplimit); @@ -1608,11 +1572,9 @@ static void dr_ste_build_ipv6_l3_l4_bit_mask(struct mlx5dr_match_param *value, static int dr_ste_build_ipv6_l3_l4_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { struct mlx5dr_match_spec *spec = sb->inner ? &value->inner : &value->outer; - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(eth_l4, tag, dst_port, spec, tcp_dport); DR_STE_SET_TAG(eth_l4, tag, src_port, spec, tcp_sport); @@ -1647,7 +1609,7 @@ void mlx5dr_ste_build_ipv6_l3_l4(struct mlx5dr_ste_build *sb, static int dr_ste_build_empty_always_hit_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { return 0; } @@ -1673,11 +1635,9 @@ static void dr_ste_build_mpls_bit_mask(struct mlx5dr_match_param *value, static int dr_ste_build_mpls_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc2 *misc2_mask = &value->misc2; - u8 *tag = hw_ste->tag; if (sb->inner) DR_STE_SET_MPLS_TAG(mpls, misc2_mask, inner, tag); @@ -1716,11 +1676,9 @@ static void dr_ste_build_gre_bit_mask(struct mlx5dr_match_param *value, static int dr_ste_build_gre_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc *misc = &value->misc; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(gre, tag, gre_protocol, misc, gre_protocol); @@ -1781,11 +1739,9 @@ static void dr_ste_build_flex_parser_0_bit_mask(struct mlx5dr_match_param *value static int dr_ste_build_flex_parser_0_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc2 *misc_2_mask = &value->misc2; - u8 *tag = hw_ste->tag; if (DR_STE_IS_OUTER_MPLS_OVER_GRE_SET(misc_2_mask)) { DR_STE_SET_TAG(flex_parser_0, tag, parser_3_label, @@ -1903,11 +1859,9 @@ static int dr_ste_build_flex_parser_1_bit_mask(struct mlx5dr_match_param *mask, static int dr_ste_build_flex_parser_1_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc3 *misc_3 = &value->misc3; - u8 *tag = hw_ste->tag; u32 icmp_header_data; int dw0_location; int dw1_location; @@ -2007,11 +1961,9 @@ static void dr_ste_build_general_purpose_bit_mask(struct mlx5dr_match_param *val static int dr_ste_build_general_purpose_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc2 *misc_2_mask = &value->misc2; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(general_purpose, tag, general_purpose_lookup_field, misc_2_mask, metadata_reg_a); @@ -2052,11 +2004,9 @@ static void dr_ste_build_eth_l4_misc_bit_mask(struct mlx5dr_match_param *value, static int dr_ste_build_eth_l4_misc_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc3 *misc3 = &value->misc3; - u8 *tag = hw_ste->tag; if (sb->inner) { DR_STE_SET_TAG(eth_l4_misc, tag, seq_num, misc3, inner_tcp_seq_num); @@ -2102,11 +2052,9 @@ dr_ste_build_flex_parser_tnl_vxlan_gpe_bit_mask(struct mlx5dr_match_param *value static int dr_ste_build_flex_parser_tnl_vxlan_gpe_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc3 *misc3 = &value->misc3; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(flex_parser_tnl_vxlan_gpe, tag, outer_vxlan_gpe_flags, misc3, @@ -2158,11 +2106,9 @@ dr_ste_build_flex_parser_tnl_geneve_bit_mask(struct mlx5dr_match_param *value, static int dr_ste_build_flex_parser_tnl_geneve_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc *misc = &value->misc; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(flex_parser_tnl_geneve, tag, geneve_protocol_type, misc, geneve_protocol_type); @@ -2205,11 +2151,9 @@ static void dr_ste_build_register_0_bit_mask(struct mlx5dr_match_param *value, static int dr_ste_build_register_0_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc2 *misc2 = &value->misc2; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(register_0, tag, register_0_h, misc2, metadata_reg_c_0); DR_STE_SET_TAG(register_0, tag, register_0_l, misc2, metadata_reg_c_1); @@ -2249,11 +2193,9 @@ static void dr_ste_build_register_1_bit_mask(struct mlx5dr_match_param *value, static int dr_ste_build_register_1_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc2 *misc2 = &value->misc2; - u8 *tag = hw_ste->tag; DR_STE_SET_TAG(register_1, tag, register_2_h, misc2, metadata_reg_c_4); DR_STE_SET_TAG(register_1, tag, register_2_l, misc2, metadata_reg_c_5); @@ -2276,38 +2218,25 @@ void mlx5dr_ste_build_register_1(struct mlx5dr_ste_build *sb, sb->ste_build_tag_func = &dr_ste_build_register_1_tag; } -static int dr_ste_build_src_gvmi_qpn_bit_mask(struct mlx5dr_match_param *value, - u8 *bit_mask) +static void dr_ste_build_src_gvmi_qpn_bit_mask(struct mlx5dr_match_param *value, + u8 *bit_mask) { struct mlx5dr_match_misc *misc_mask = &value->misc; - /* Partial misc source_port is not supported */ - if (misc_mask->source_port && misc_mask->source_port != 0xffff) - return -EINVAL; - - /* Partial misc source_eswitch_owner_vhca_id is not supported */ - if (misc_mask->source_eswitch_owner_vhca_id && - misc_mask->source_eswitch_owner_vhca_id != 0xffff) - return -EINVAL; - DR_STE_SET_MASK(src_gvmi_qp, bit_mask, source_gvmi, misc_mask, source_port); DR_STE_SET_MASK(src_gvmi_qp, bit_mask, source_qp, misc_mask, source_sqn); misc_mask->source_eswitch_owner_vhca_id = 0; - - return 0; } static int dr_ste_build_src_gvmi_qpn_tag(struct mlx5dr_match_param *value, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p) + u8 *tag) { - struct dr_hw_ste_format *hw_ste = (struct dr_hw_ste_format *)hw_ste_p; struct mlx5dr_match_misc *misc = &value->misc; struct mlx5dr_cmd_vport_cap *vport_cap; struct mlx5dr_domain *dmn = sb->dmn; struct mlx5dr_cmd_caps *caps; u8 *bit_mask = sb->bit_mask; - u8 *tag = hw_ste->tag; bool source_gvmi_set; DR_STE_SET_TAG(src_gvmi_qp, tag, source_qp, misc, source_sqn); @@ -2339,19 +2268,15 @@ static int dr_ste_build_src_gvmi_qpn_tag(struct mlx5dr_match_param *value, return 0; } -int mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_build *sb, - struct mlx5dr_match_param *mask, - struct mlx5dr_domain *dmn, - bool inner, bool rx) +void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_build *sb, + struct mlx5dr_match_param *mask, + struct mlx5dr_domain *dmn, + bool inner, bool rx) { - int ret; - /* Set vhca_id_valid before we reset source_eswitch_owner_vhca_id */ sb->vhca_id_valid = mask->misc.source_eswitch_owner_vhca_id; - ret = dr_ste_build_src_gvmi_qpn_bit_mask(mask, sb->bit_mask); - if (ret) - return ret; + dr_ste_build_src_gvmi_qpn_bit_mask(mask, sb->bit_mask); sb->rx = rx; sb->dmn = dmn; @@ -2359,6 +2284,4 @@ int mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_build *sb, sb->lu_type = MLX5DR_STE_LU_TYPE_SRC_GVMI_AND_QP; sb->byte_mask = dr_ste_conv_bit_to_byte_mask(sb->bit_mask); sb->ste_build_tag_func = &dr_ste_build_src_gvmi_qpn_tag; - - return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h index 0883956c58c0..f50f3b107aa3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h @@ -194,7 +194,7 @@ struct mlx5dr_ste_build { u8 bit_mask[DR_STE_SIZE_MASK]; int (*ste_build_tag_func)(struct mlx5dr_match_param *spec, struct mlx5dr_ste_build *sb, - u8 *hw_ste_p); + u8 *tag); }; struct mlx5dr_ste_htbl * @@ -227,7 +227,6 @@ void mlx5dr_ste_set_hit_gvmi(u8 *hw_ste_p, u16 gvmi); void mlx5dr_ste_set_hit_addr(u8 *hw_ste, u64 icm_addr, u32 ht_size); void mlx5dr_ste_always_miss_addr(struct mlx5dr_ste *ste, u64 miss_addr); void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask); -bool mlx5dr_ste_not_used_ste(struct mlx5dr_ste *ste); bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher, u8 ste_location); void mlx5dr_ste_rx_set_flow_tag(u8 *hw_ste_p, u32 flow_tag); @@ -266,6 +265,11 @@ static inline void mlx5dr_ste_get(struct mlx5dr_ste *ste) ste->refcount++; } +static inline bool mlx5dr_ste_is_not_used(struct mlx5dr_ste *ste) +{ + return !ste->refcount; +} + void mlx5dr_ste_set_hit_addr_by_next_htbl(u8 *hw_ste, struct mlx5dr_ste_htbl *next_htbl); bool mlx5dr_ste_equal_tag(void *src, void *dst); @@ -284,9 +288,9 @@ int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher, struct mlx5dr_matcher_rx_tx *nic_matcher, struct mlx5dr_match_param *value, u8 *ste_arr); -int mlx5dr_ste_build_eth_l2_src_des(struct mlx5dr_ste_build *builder, - struct mlx5dr_match_param *mask, - bool inner, bool rx); +void mlx5dr_ste_build_eth_l2_src_des(struct mlx5dr_ste_build *builder, + struct mlx5dr_match_param *mask, + bool inner, bool rx); void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_build *sb, struct mlx5dr_match_param *mask, bool inner, bool rx); @@ -342,10 +346,10 @@ void mlx5dr_ste_build_register_0(struct mlx5dr_ste_build *sb, void mlx5dr_ste_build_register_1(struct mlx5dr_ste_build *sb, struct mlx5dr_match_param *mask, bool inner, bool rx); -int mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_build *sb, - struct mlx5dr_match_param *mask, - struct mlx5dr_domain *dmn, - bool inner, bool rx); +void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_build *sb, + struct mlx5dr_match_param *mask, + struct mlx5dr_domain *dmn, + bool inner, bool rx); void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx); /* Actions utils */ @@ -793,6 +797,7 @@ struct mlx5dr_rule { struct mlx5dr_rule_rx_tx rx; struct mlx5dr_rule_rx_tx tx; struct list_head rule_actions_list; + u32 flow_source; }; void mlx5dr_rule_update_rule_member(struct mlx5dr_ste *new_ste, @@ -991,7 +996,6 @@ struct mlx5dr_icm_chunk * mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool, enum mlx5dr_icm_chunk_size chunk_size); void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk); -bool mlx5dr_ste_is_not_valid_entry(u8 *p_hw_ste); int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn, struct mlx5dr_domain_rx_tx *nic_dmn, struct mlx5dr_ste_htbl *htbl, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c index 9b08eb557a31..96c39a17d026 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c @@ -487,7 +487,8 @@ static int mlx5_cmd_dr_create_fte(struct mlx5_flow_root_namespace *ns, rule = mlx5dr_rule_create(group->fs_dr_matcher.dr_matcher, ¶ms, num_actions, - actions); + actions, + fte->flow_context.flow_source); if (!rule) { err = -EINVAL; goto free_actions; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h index 7deaca9ade3b..7914fe3fc68d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h @@ -67,7 +67,8 @@ struct mlx5dr_rule * mlx5dr_rule_create(struct mlx5dr_matcher *matcher, struct mlx5dr_match_parameters *value, size_t num_actions, - struct mlx5dr_action *actions[]); + struct mlx5dr_action *actions[], + u32 flow_source); int mlx5dr_rule_destroy(struct mlx5dr_rule *rule); diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index b9375d96cdbc..e026617d6133 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -5,6 +5,7 @@ * Copyright (c) 2017 Microsemi Corporation */ #include <linux/if_bridge.h> +#include <soc/mscc/ocelot_vcap.h> #include "ocelot.h" #include "ocelot_vcap.h" @@ -107,6 +108,13 @@ static void ocelot_vcap_enable(struct ocelot *ocelot, int port) ocelot_write_gix(ocelot, ANA_PORT_VCAP_S2_CFG_S2_ENA | ANA_PORT_VCAP_S2_CFG_S2_IP6_CFG(0xa), ANA_PORT_VCAP_S2_CFG, port); + + ocelot_write_gix(ocelot, ANA_PORT_VCAP_CFG_S1_ENA, + ANA_PORT_VCAP_CFG, port); + + ocelot_rmw_gix(ocelot, REW_PORT_CFG_ES0_EN, + REW_PORT_CFG_ES0_EN, + REW_PORT_CFG, port); } static inline u32 ocelot_vlant_read_vlanaccess(struct ocelot *ocelot) diff --git a/drivers/net/ethernet/mscc/ocelot.h b/drivers/net/ethernet/mscc/ocelot.h index dc29e05103a1..abb407dff93c 100644 --- a/drivers/net/ethernet/mscc/ocelot.h +++ b/drivers/net/ethernet/mscc/ocelot.h @@ -98,6 +98,8 @@ int ocelot_port_lag_join(struct ocelot *ocelot, int port, struct net_device *bond); void ocelot_port_lag_leave(struct ocelot *ocelot, int port, struct net_device *bond); +struct net_device *ocelot_port_to_netdev(struct ocelot *ocelot, int port); +int ocelot_netdev_to_port(struct net_device *dev); u32 ocelot_port_readl(struct ocelot_port *port, u32 reg); void ocelot_port_writel(struct ocelot_port *port, u32 val, u32 reg); diff --git a/drivers/net/ethernet/mscc/ocelot_flower.c b/drivers/net/ethernet/mscc/ocelot_flower.c index ec1b6e2572ba..0ea6d4f411cb 100644 --- a/drivers/net/ethernet/mscc/ocelot_flower.c +++ b/drivers/net/ethernet/mscc/ocelot_flower.c @@ -5,56 +5,410 @@ #include <net/pkt_cls.h> #include <net/tc_act/tc_gact.h> - +#include <soc/mscc/ocelot_vcap.h> #include "ocelot_vcap.h" -static int ocelot_flower_parse_action(struct flow_cls_offload *f, +/* Arbitrarily chosen constants for encoding the VCAP block and lookup number + * into the chain number. This is UAPI. + */ +#define VCAP_BLOCK 10000 +#define VCAP_LOOKUP 1000 +#define VCAP_IS1_NUM_LOOKUPS 3 +#define VCAP_IS2_NUM_LOOKUPS 2 +#define VCAP_IS2_NUM_PAG 256 +#define VCAP_IS1_CHAIN(lookup) \ + (1 * VCAP_BLOCK + (lookup) * VCAP_LOOKUP) +#define VCAP_IS2_CHAIN(lookup, pag) \ + (2 * VCAP_BLOCK + (lookup) * VCAP_LOOKUP + (pag)) + +static int ocelot_chain_to_block(int chain, bool ingress) +{ + int lookup, pag; + + if (!ingress) { + if (chain == 0) + return VCAP_ES0; + return -EOPNOTSUPP; + } + + /* Backwards compatibility with older, single-chain tc-flower + * offload support in Ocelot + */ + if (chain == 0) + return VCAP_IS2; + + for (lookup = 0; lookup < VCAP_IS1_NUM_LOOKUPS; lookup++) + if (chain == VCAP_IS1_CHAIN(lookup)) + return VCAP_IS1; + + for (lookup = 0; lookup < VCAP_IS2_NUM_LOOKUPS; lookup++) + for (pag = 0; pag < VCAP_IS2_NUM_PAG; pag++) + if (chain == VCAP_IS2_CHAIN(lookup, pag)) + return VCAP_IS2; + + return -EOPNOTSUPP; +} + +/* Caller must ensure this is a valid IS1 or IS2 chain first, + * by calling ocelot_chain_to_block. + */ +static int ocelot_chain_to_lookup(int chain) +{ + return (chain / VCAP_LOOKUP) % 10; +} + +/* Caller must ensure this is a valid IS2 chain first, + * by calling ocelot_chain_to_block. + */ +static int ocelot_chain_to_pag(int chain) +{ + int lookup = ocelot_chain_to_lookup(chain); + + /* calculate PAG value as chain index relative to the first PAG */ + return chain - VCAP_IS2_CHAIN(lookup, 0); +} + +static bool ocelot_is_goto_target_valid(int goto_target, int chain, + bool ingress) +{ + int pag; + + /* Can't offload GOTO in VCAP ES0 */ + if (!ingress) + return (goto_target < 0); + + /* Non-optional GOTOs */ + if (chain == 0) + /* VCAP IS1 can be skipped, either partially or completely */ + return (goto_target == VCAP_IS1_CHAIN(0) || + goto_target == VCAP_IS1_CHAIN(1) || + goto_target == VCAP_IS1_CHAIN(2) || + goto_target == VCAP_IS2_CHAIN(0, 0) || + goto_target == VCAP_IS2_CHAIN(1, 0)); + + if (chain == VCAP_IS1_CHAIN(0)) + return (goto_target == VCAP_IS1_CHAIN(1)); + + if (chain == VCAP_IS1_CHAIN(1)) + return (goto_target == VCAP_IS1_CHAIN(2)); + + /* Lookup 2 of VCAP IS1 can really support non-optional GOTOs, + * using a Policy Association Group (PAG) value, which is an 8-bit + * value encoding a VCAP IS2 target chain. + */ + if (chain == VCAP_IS1_CHAIN(2)) { + for (pag = 0; pag < VCAP_IS2_NUM_PAG; pag++) + if (goto_target == VCAP_IS2_CHAIN(0, pag)) + return true; + + return false; + } + + /* Non-optional GOTO from VCAP IS2 lookup 0 to lookup 1. + * We cannot change the PAG at this point. + */ + for (pag = 0; pag < VCAP_IS2_NUM_PAG; pag++) + if (chain == VCAP_IS2_CHAIN(0, pag)) + return (goto_target == VCAP_IS2_CHAIN(1, pag)); + + /* VCAP IS2 lookup 1 cannot jump anywhere */ + return false; +} + +static struct ocelot_vcap_filter * +ocelot_find_vcap_filter_that_points_at(struct ocelot *ocelot, int chain) +{ + struct ocelot_vcap_filter *filter; + struct ocelot_vcap_block *block; + int block_id; + + block_id = ocelot_chain_to_block(chain, true); + if (block_id < 0) + return NULL; + + if (block_id == VCAP_IS2) { + block = &ocelot->block[VCAP_IS1]; + + list_for_each_entry(filter, &block->rules, list) + if (filter->type == OCELOT_VCAP_FILTER_PAG && + filter->goto_target == chain) + return filter; + } + + list_for_each_entry(filter, &ocelot->dummy_rules, list) + if (filter->goto_target == chain) + return filter; + + return NULL; +} + +static int ocelot_flower_parse_action(struct ocelot *ocelot, bool ingress, + struct flow_cls_offload *f, struct ocelot_vcap_filter *filter) { + struct netlink_ext_ack *extack = f->common.extack; + bool allow_missing_goto_target = false; const struct flow_action_entry *a; + enum ocelot_tag_tpid_sel tpid; + int i, chain, egress_port; u64 rate; - int i; - - if (!flow_offload_has_one_action(&f->rule->action)) - return -EOPNOTSUPP; if (!flow_action_basic_hw_stats_check(&f->rule->action, f->common.extack)) return -EOPNOTSUPP; + chain = f->common.chain_index; + filter->block_id = ocelot_chain_to_block(chain, ingress); + if (filter->block_id < 0) { + NL_SET_ERR_MSG_MOD(extack, "Cannot offload to this chain"); + return -EOPNOTSUPP; + } + if (filter->block_id == VCAP_IS1 || filter->block_id == VCAP_IS2) + filter->lookup = ocelot_chain_to_lookup(chain); + if (filter->block_id == VCAP_IS2) + filter->pag = ocelot_chain_to_pag(chain); + + filter->goto_target = -1; + filter->type = OCELOT_VCAP_FILTER_DUMMY; + flow_action_for_each(i, a, &f->rule->action) { switch (a->id) { case FLOW_ACTION_DROP: - filter->action = OCELOT_VCAP_ACTION_DROP; + if (filter->block_id != VCAP_IS2) { + NL_SET_ERR_MSG_MOD(extack, + "Drop action can only be offloaded to VCAP IS2"); + return -EOPNOTSUPP; + } + if (filter->goto_target != -1) { + NL_SET_ERR_MSG_MOD(extack, + "Last action must be GOTO"); + return -EOPNOTSUPP; + } + filter->action.mask_mode = OCELOT_MASK_MODE_PERMIT_DENY; + filter->action.port_mask = 0; + filter->action.police_ena = true; + filter->action.pol_ix = OCELOT_POLICER_DISCARD; + filter->type = OCELOT_VCAP_FILTER_OFFLOAD; break; case FLOW_ACTION_TRAP: - filter->action = OCELOT_VCAP_ACTION_TRAP; + if (filter->block_id != VCAP_IS2) { + NL_SET_ERR_MSG_MOD(extack, + "Trap action can only be offloaded to VCAP IS2"); + return -EOPNOTSUPP; + } + if (filter->goto_target != -1) { + NL_SET_ERR_MSG_MOD(extack, + "Last action must be GOTO"); + return -EOPNOTSUPP; + } + filter->action.mask_mode = OCELOT_MASK_MODE_PERMIT_DENY; + filter->action.port_mask = 0; + filter->action.cpu_copy_ena = true; + filter->action.cpu_qu_num = 0; + filter->type = OCELOT_VCAP_FILTER_OFFLOAD; break; case FLOW_ACTION_POLICE: - filter->action = OCELOT_VCAP_ACTION_POLICE; + if (filter->block_id != VCAP_IS2 || + filter->lookup != 0) { + NL_SET_ERR_MSG_MOD(extack, + "Police action can only be offloaded to VCAP IS2 lookup 0"); + return -EOPNOTSUPP; + } + if (filter->goto_target != -1) { + NL_SET_ERR_MSG_MOD(extack, + "Last action must be GOTO"); + return -EOPNOTSUPP; + } + filter->action.police_ena = true; rate = a->police.rate_bytes_ps; - filter->pol.rate = div_u64(rate, 1000) * 8; - filter->pol.burst = a->police.burst; + filter->action.pol.rate = div_u64(rate, 1000) * 8; + filter->action.pol.burst = a->police.burst; + filter->type = OCELOT_VCAP_FILTER_OFFLOAD; + break; + case FLOW_ACTION_REDIRECT: + if (filter->block_id != VCAP_IS2) { + NL_SET_ERR_MSG_MOD(extack, + "Redirect action can only be offloaded to VCAP IS2"); + return -EOPNOTSUPP; + } + if (filter->goto_target != -1) { + NL_SET_ERR_MSG_MOD(extack, + "Last action must be GOTO"); + return -EOPNOTSUPP; + } + egress_port = ocelot->ops->netdev_to_port(a->dev); + if (egress_port < 0) { + NL_SET_ERR_MSG_MOD(extack, + "Destination not an ocelot port"); + return -EOPNOTSUPP; + } + filter->action.mask_mode = OCELOT_MASK_MODE_REDIRECT; + filter->action.port_mask = BIT(egress_port); + filter->type = OCELOT_VCAP_FILTER_OFFLOAD; + break; + case FLOW_ACTION_VLAN_POP: + if (filter->block_id != VCAP_IS1) { + NL_SET_ERR_MSG_MOD(extack, + "VLAN pop action can only be offloaded to VCAP IS1"); + return -EOPNOTSUPP; + } + if (filter->goto_target != -1) { + NL_SET_ERR_MSG_MOD(extack, + "Last action must be GOTO"); + return -EOPNOTSUPP; + } + filter->action.vlan_pop_cnt_ena = true; + filter->action.vlan_pop_cnt++; + if (filter->action.vlan_pop_cnt > 2) { + NL_SET_ERR_MSG_MOD(extack, + "Cannot pop more than 2 VLAN headers"); + return -EOPNOTSUPP; + } + filter->type = OCELOT_VCAP_FILTER_OFFLOAD; + break; + case FLOW_ACTION_PRIORITY: + if (filter->block_id != VCAP_IS1) { + NL_SET_ERR_MSG_MOD(extack, + "Priority action can only be offloaded to VCAP IS1"); + return -EOPNOTSUPP; + } + if (filter->goto_target != -1) { + NL_SET_ERR_MSG_MOD(extack, + "Last action must be GOTO"); + return -EOPNOTSUPP; + } + filter->action.qos_ena = true; + filter->action.qos_val = a->priority; + filter->type = OCELOT_VCAP_FILTER_OFFLOAD; + break; + case FLOW_ACTION_GOTO: + filter->goto_target = a->chain_index; + + if (filter->block_id == VCAP_IS1 && filter->lookup == 2) { + int pag = ocelot_chain_to_pag(filter->goto_target); + + filter->action.pag_override_mask = 0xff; + filter->action.pag_val = pag; + filter->type = OCELOT_VCAP_FILTER_PAG; + } + break; + case FLOW_ACTION_VLAN_PUSH: + if (filter->block_id != VCAP_ES0) { + NL_SET_ERR_MSG_MOD(extack, + "VLAN push action can only be offloaded to VCAP ES0"); + return -EOPNOTSUPP; + } + switch (ntohs(a->vlan.proto)) { + case ETH_P_8021Q: + tpid = OCELOT_TAG_TPID_SEL_8021Q; + break; + case ETH_P_8021AD: + tpid = OCELOT_TAG_TPID_SEL_8021AD; + break; + default: + NL_SET_ERR_MSG_MOD(extack, + "Cannot push custom TPID"); + return -EOPNOTSUPP; + } + filter->action.tag_a_tpid_sel = tpid; + filter->action.push_outer_tag = OCELOT_ES0_TAG; + filter->action.tag_a_vid_sel = 1; + filter->action.vid_a_val = a->vlan.vid; + filter->action.pcp_a_val = a->vlan.prio; + filter->type = OCELOT_VCAP_FILTER_OFFLOAD; break; default: + NL_SET_ERR_MSG_MOD(extack, "Cannot offload action"); return -EOPNOTSUPP; } } + if (filter->goto_target == -1) { + if ((filter->block_id == VCAP_IS2 && filter->lookup == 1) || + chain == 0) { + allow_missing_goto_target = true; + } else { + NL_SET_ERR_MSG_MOD(extack, "Missing GOTO action"); + return -EOPNOTSUPP; + } + } + + if (!ocelot_is_goto_target_valid(filter->goto_target, chain, ingress) && + !allow_missing_goto_target) { + NL_SET_ERR_MSG_MOD(extack, "Cannot offload this GOTO target"); + return -EOPNOTSUPP; + } + return 0; } -static int ocelot_flower_parse(struct flow_cls_offload *f, - struct ocelot_vcap_filter *filter) +static int ocelot_flower_parse_indev(struct ocelot *ocelot, int port, + struct flow_cls_offload *f, + struct ocelot_vcap_filter *filter) +{ + struct flow_rule *rule = flow_cls_offload_flow_rule(f); + const struct vcap_props *vcap = &ocelot->vcap[VCAP_ES0]; + int key_length = vcap->keys[VCAP_ES0_IGR_PORT].length; + struct netlink_ext_ack *extack = f->common.extack; + struct net_device *dev, *indev; + struct flow_match_meta match; + int ingress_port; + + flow_rule_match_meta(rule, &match); + + if (!match.mask->ingress_ifindex) + return 0; + + if (match.mask->ingress_ifindex != 0xFFFFFFFF) { + NL_SET_ERR_MSG_MOD(extack, "Unsupported ingress ifindex mask"); + return -EOPNOTSUPP; + } + + dev = ocelot->ops->port_to_netdev(ocelot, port); + if (!dev) + return -EINVAL; + + indev = __dev_get_by_index(dev_net(dev), match.key->ingress_ifindex); + if (!indev) { + NL_SET_ERR_MSG_MOD(extack, + "Can't find the ingress port to match on"); + return -ENOENT; + } + + ingress_port = ocelot->ops->netdev_to_port(indev); + if (ingress_port < 0) { + NL_SET_ERR_MSG_MOD(extack, + "Can only offload an ocelot ingress port"); + return -EOPNOTSUPP; + } + if (ingress_port == port) { + NL_SET_ERR_MSG_MOD(extack, + "Ingress port is equal to the egress port"); + return -EINVAL; + } + + filter->ingress_port.value = ingress_port; + filter->ingress_port.mask = GENMASK(key_length - 1, 0); + + return 0; +} + +static int +ocelot_flower_parse_key(struct ocelot *ocelot, int port, bool ingress, + struct flow_cls_offload *f, + struct ocelot_vcap_filter *filter) { struct flow_rule *rule = flow_cls_offload_flow_rule(f); struct flow_dissector *dissector = rule->match.dissector; + struct netlink_ext_ack *extack = f->common.extack; u16 proto = ntohs(f->common.protocol); bool match_protocol = true; + int ret; if (dissector->used_keys & ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | BIT(FLOW_DISSECTOR_KEY_BASIC) | + BIT(FLOW_DISSECTOR_KEY_META) | BIT(FLOW_DISSECTOR_KEY_PORTS) | BIT(FLOW_DISSECTOR_KEY_VLAN) | BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | @@ -63,6 +417,13 @@ static int ocelot_flower_parse(struct flow_cls_offload *f, return -EOPNOTSUPP; } + /* For VCAP ES0 (egress rewriter) we can match on the ingress port */ + if (!ingress) { + ret = ocelot_flower_parse_indev(ocelot, port, f, filter); + if (ret) + return ret; + } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { struct flow_match_control match; @@ -72,6 +433,19 @@ static int ocelot_flower_parse(struct flow_cls_offload *f, if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { struct flow_match_eth_addrs match; + if (filter->block_id == VCAP_ES0) { + NL_SET_ERR_MSG_MOD(extack, + "VCAP ES0 cannot match on MAC address"); + return -EOPNOTSUPP; + } + + if (filter->block_id == VCAP_IS1 && + !is_zero_ether_addr(match.mask->dst)) { + NL_SET_ERR_MSG_MOD(extack, + "Key type S1_NORMAL cannot match on destination MAC"); + return -EOPNOTSUPP; + } + /* The hw support mac matches only for MAC_ETYPE key, * therefore if other matches(port, tcp flags, etc) are added * then just bail out @@ -103,6 +477,12 @@ static int ocelot_flower_parse(struct flow_cls_offload *f, flow_rule_match_basic(rule, &match); if (ntohs(match.key->n_proto) == ETH_P_IP) { + if (filter->block_id == VCAP_ES0) { + NL_SET_ERR_MSG_MOD(extack, + "VCAP ES0 cannot match on IP protocol"); + return -EOPNOTSUPP; + } + filter->key_type = OCELOT_VCAP_KEY_IPV4; filter->key.ipv4.proto.value[0] = match.key->ip_proto; @@ -111,6 +491,12 @@ static int ocelot_flower_parse(struct flow_cls_offload *f, match_protocol = false; } if (ntohs(match.key->n_proto) == ETH_P_IPV6) { + if (filter->block_id == VCAP_ES0) { + NL_SET_ERR_MSG_MOD(extack, + "VCAP ES0 cannot match on IP protocol"); + return -EOPNOTSUPP; + } + filter->key_type = OCELOT_VCAP_KEY_IPV6; filter->key.ipv6.proto.value[0] = match.key->ip_proto; @@ -125,6 +511,18 @@ static int ocelot_flower_parse(struct flow_cls_offload *f, struct flow_match_ipv4_addrs match; u8 *tmp; + if (filter->block_id == VCAP_ES0) { + NL_SET_ERR_MSG_MOD(extack, + "VCAP ES0 cannot match on IP address"); + return -EOPNOTSUPP; + } + + if (filter->block_id == VCAP_IS1 && *(u32 *)&match.mask->dst) { + NL_SET_ERR_MSG_MOD(extack, + "Key type S1_NORMAL cannot match on destination IP"); + return -EOPNOTSUPP; + } + flow_rule_match_ipv4_addrs(rule, &match); tmp = &filter->key.ipv4.sip.value.addr[0]; memcpy(tmp, &match.key->src, 4); @@ -148,6 +546,12 @@ static int ocelot_flower_parse(struct flow_cls_offload *f, if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { struct flow_match_ports match; + if (filter->block_id == VCAP_ES0) { + NL_SET_ERR_MSG_MOD(extack, + "VCAP ES0 cannot match on L4 ports"); + return -EOPNOTSUPP; + } + flow_rule_match_ports(rule, &match); filter->key.ipv4.sport.value = ntohs(match.key->src); filter->key.ipv4.sport.mask = ntohs(match.mask->src); @@ -170,6 +574,12 @@ static int ocelot_flower_parse(struct flow_cls_offload *f, finished_key_parsing: if (match_protocol && proto != ETH_P_ALL) { + if (filter->block_id == VCAP_ES0) { + NL_SET_ERR_MSG_MOD(extack, + "VCAP ES0 cannot match on L2 proto"); + return -EOPNOTSUPP; + } + /* TODO: support SNAP, LLC etc */ if (proto < ETH_P_802_3_MIN) return -EOPNOTSUPP; @@ -179,14 +589,28 @@ finished_key_parsing: } /* else, a filter of type OCELOT_VCAP_KEY_ANY is implicitly added */ + return 0; +} + +static int ocelot_flower_parse(struct ocelot *ocelot, int port, bool ingress, + struct flow_cls_offload *f, + struct ocelot_vcap_filter *filter) +{ + int ret; + filter->prio = f->common.prio; filter->id = f->cookie; - return ocelot_flower_parse_action(f, filter); + + ret = ocelot_flower_parse_action(ocelot, ingress, f, filter); + if (ret) + return ret; + + return ocelot_flower_parse_key(ocelot, port, ingress, f, filter); } static struct ocelot_vcap_filter -*ocelot_vcap_filter_create(struct ocelot *ocelot, int port, - struct flow_cls_offload *f) +*ocelot_vcap_filter_create(struct ocelot *ocelot, int port, bool ingress, + struct flow_cls_offload *f) { struct ocelot_vcap_filter *filter; @@ -194,26 +618,65 @@ static struct ocelot_vcap_filter if (!filter) return NULL; - filter->ingress_port_mask = BIT(port); + if (ingress) { + filter->ingress_port_mask = BIT(port); + } else { + const struct vcap_props *vcap = &ocelot->vcap[VCAP_ES0]; + int key_length = vcap->keys[VCAP_ES0_EGR_PORT].length; + + filter->egress_port.value = port; + filter->egress_port.mask = GENMASK(key_length - 1, 0); + } + return filter; } +static int ocelot_vcap_dummy_filter_add(struct ocelot *ocelot, + struct ocelot_vcap_filter *filter) +{ + list_add(&filter->list, &ocelot->dummy_rules); + + return 0; +} + +static int ocelot_vcap_dummy_filter_del(struct ocelot *ocelot, + struct ocelot_vcap_filter *filter) +{ + list_del(&filter->list); + kfree(filter); + + return 0; +} + int ocelot_cls_flower_replace(struct ocelot *ocelot, int port, struct flow_cls_offload *f, bool ingress) { + struct netlink_ext_ack *extack = f->common.extack; struct ocelot_vcap_filter *filter; + int chain = f->common.chain_index; int ret; - filter = ocelot_vcap_filter_create(ocelot, port, f); + if (chain && !ocelot_find_vcap_filter_that_points_at(ocelot, chain)) { + NL_SET_ERR_MSG_MOD(extack, "No default GOTO action points to this chain"); + return -EOPNOTSUPP; + } + + filter = ocelot_vcap_filter_create(ocelot, port, ingress, f); if (!filter) return -ENOMEM; - ret = ocelot_flower_parse(f, filter); + ret = ocelot_flower_parse(ocelot, port, ingress, f, filter); if (ret) { kfree(filter); return ret; } + /* The non-optional GOTOs for the TCAM skeleton don't need + * to be actually offloaded. + */ + if (filter->type == OCELOT_VCAP_FILTER_DUMMY) + return ocelot_vcap_dummy_filter_add(ocelot, filter); + return ocelot_vcap_filter_add(ocelot, filter, f->common.extack); } EXPORT_SYMBOL_GPL(ocelot_cls_flower_replace); @@ -221,28 +684,49 @@ EXPORT_SYMBOL_GPL(ocelot_cls_flower_replace); int ocelot_cls_flower_destroy(struct ocelot *ocelot, int port, struct flow_cls_offload *f, bool ingress) { - struct ocelot_vcap_filter filter; + struct ocelot_vcap_filter *filter; + struct ocelot_vcap_block *block; + int block_id; + + block_id = ocelot_chain_to_block(f->common.chain_index, ingress); + if (block_id < 0) + return 0; - filter.prio = f->common.prio; - filter.id = f->cookie; + block = &ocelot->block[block_id]; - return ocelot_vcap_filter_del(ocelot, &filter); + filter = ocelot_vcap_block_find_filter_by_id(block, f->cookie); + if (!filter) + return 0; + + if (filter->type == OCELOT_VCAP_FILTER_DUMMY) + return ocelot_vcap_dummy_filter_del(ocelot, filter); + + return ocelot_vcap_filter_del(ocelot, filter); } EXPORT_SYMBOL_GPL(ocelot_cls_flower_destroy); int ocelot_cls_flower_stats(struct ocelot *ocelot, int port, struct flow_cls_offload *f, bool ingress) { - struct ocelot_vcap_filter filter; - int ret; + struct ocelot_vcap_filter *filter; + struct ocelot_vcap_block *block; + int block_id, ret; + + block_id = ocelot_chain_to_block(f->common.chain_index, ingress); + if (block_id < 0) + return 0; + + block = &ocelot->block[block_id]; + + filter = ocelot_vcap_block_find_filter_by_id(block, f->cookie); + if (!filter || filter->type == OCELOT_VCAP_FILTER_DUMMY) + return 0; - filter.prio = f->common.prio; - filter.id = f->cookie; - ret = ocelot_vcap_filter_stats_update(ocelot, &filter); + ret = ocelot_vcap_filter_stats_update(ocelot, filter); if (ret) return ret; - flow_stats_update(&f->stats, 0x0, filter.stats.pkts, 0, 0x0, + flow_stats_update(&f->stats, 0x0, filter->stats.pkts, 0, 0x0, FLOW_ACTION_HW_STATS_IMMEDIATE); return 0; } diff --git a/drivers/net/ethernet/mscc/ocelot_io.c b/drivers/net/ethernet/mscc/ocelot_io.c index d22711282183..0acb45948418 100644 --- a/drivers/net/ethernet/mscc/ocelot_io.c +++ b/drivers/net/ethernet/mscc/ocelot_io.c @@ -71,6 +71,23 @@ void ocelot_port_writel(struct ocelot_port *port, u32 val, u32 reg) } EXPORT_SYMBOL(ocelot_port_writel); +u32 __ocelot_target_read_ix(struct ocelot *ocelot, enum ocelot_target target, + u32 reg, u32 offset) +{ + u32 val; + + regmap_read(ocelot->targets[target], + ocelot->map[target][reg] + offset, &val); + return val; +} + +void __ocelot_target_write_ix(struct ocelot *ocelot, enum ocelot_target target, + u32 val, u32 reg, u32 offset) +{ + regmap_write(ocelot->targets[target], + ocelot->map[target][reg] + offset, val); +} + int ocelot_regfields_init(struct ocelot *ocelot, const struct reg_field *const regfields) { diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c index 028a0150f97d..64e619f0f5b2 100644 --- a/drivers/net/ethernet/mscc/ocelot_net.c +++ b/drivers/net/ethernet/mscc/ocelot_net.c @@ -656,6 +656,36 @@ static const struct net_device_ops ocelot_port_netdev_ops = { .ndo_do_ioctl = ocelot_ioctl, }; +struct net_device *ocelot_port_to_netdev(struct ocelot *ocelot, int port) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + struct ocelot_port_private *priv; + + if (!ocelot_port) + return NULL; + + priv = container_of(ocelot_port, struct ocelot_port_private, port); + + return priv->dev; +} + +static bool ocelot_port_dev_check(const struct net_device *dev) +{ + return dev->netdev_ops == &ocelot_port_netdev_ops; +} + +int ocelot_netdev_to_port(struct net_device *dev) +{ + struct ocelot_port_private *priv; + + if (!dev || !ocelot_port_dev_check(dev)) + return -EINVAL; + + priv = netdev_priv(dev); + + return priv->chip_port; +} + static void ocelot_port_get_strings(struct net_device *netdev, u32 sset, u8 *data) { diff --git a/drivers/net/ethernet/mscc/ocelot_s2.h b/drivers/net/ethernet/mscc/ocelot_s2.h deleted file mode 100644 index 80107bec2e45..000000000000 --- a/drivers/net/ethernet/mscc/ocelot_s2.h +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ -/* Microsemi Ocelot Switch driver - * Copyright (c) 2018 Microsemi Corporation - */ - -#ifndef _OCELOT_S2_CORE_H_ -#define _OCELOT_S2_CORE_H_ - -#define S2_CORE_UPDATE_CTRL_UPDATE_CMD(x) (((x) << 22) & GENMASK(24, 22)) -#define S2_CORE_UPDATE_CTRL_UPDATE_CMD_M GENMASK(24, 22) -#define S2_CORE_UPDATE_CTRL_UPDATE_CMD_X(x) (((x) & GENMASK(24, 22)) >> 22) -#define S2_CORE_UPDATE_CTRL_UPDATE_ENTRY_DIS BIT(21) -#define S2_CORE_UPDATE_CTRL_UPDATE_ACTION_DIS BIT(20) -#define S2_CORE_UPDATE_CTRL_UPDATE_CNT_DIS BIT(19) -#define S2_CORE_UPDATE_CTRL_UPDATE_ADDR(x) (((x) << 3) & GENMASK(18, 3)) -#define S2_CORE_UPDATE_CTRL_UPDATE_ADDR_M GENMASK(18, 3) -#define S2_CORE_UPDATE_CTRL_UPDATE_ADDR_X(x) (((x) & GENMASK(18, 3)) >> 3) -#define S2_CORE_UPDATE_CTRL_UPDATE_SHOT BIT(2) -#define S2_CORE_UPDATE_CTRL_CLEAR_CACHE BIT(1) -#define S2_CORE_UPDATE_CTRL_MV_TRAFFIC_IGN BIT(0) - -#define S2_CORE_MV_CFG_MV_NUM_POS(x) (((x) << 16) & GENMASK(31, 16)) -#define S2_CORE_MV_CFG_MV_NUM_POS_M GENMASK(31, 16) -#define S2_CORE_MV_CFG_MV_NUM_POS_X(x) (((x) & GENMASK(31, 16)) >> 16) -#define S2_CORE_MV_CFG_MV_SIZE(x) ((x) & GENMASK(15, 0)) -#define S2_CORE_MV_CFG_MV_SIZE_M GENMASK(15, 0) - -#define S2_CACHE_ENTRY_DAT_RSZ 0x4 - -#define S2_CACHE_MASK_DAT_RSZ 0x4 - -#define S2_CACHE_ACTION_DAT_RSZ 0x4 - -#define S2_CACHE_CNT_DAT_RSZ 0x4 - -#define S2_STICKY_VCAP_ROW_DELETED_STICKY BIT(0) - -#define S2_BIST_CTRL_TCAM_BIST BIT(1) -#define S2_BIST_CTRL_TCAM_INIT BIT(0) - -#define S2_BIST_CFG_TCAM_BIST_SOE_ENA BIT(8) -#define S2_BIST_CFG_TCAM_HCG_DIS BIT(7) -#define S2_BIST_CFG_TCAM_CG_DIS BIT(6) -#define S2_BIST_CFG_TCAM_BIAS(x) ((x) & GENMASK(5, 0)) -#define S2_BIST_CFG_TCAM_BIAS_M GENMASK(5, 0) - -#define S2_BIST_STAT_BIST_RT_ERR BIT(15) -#define S2_BIST_STAT_BIST_PENC_ERR BIT(14) -#define S2_BIST_STAT_BIST_COMP_ERR BIT(13) -#define S2_BIST_STAT_BIST_ADDR_ERR BIT(12) -#define S2_BIST_STAT_BIST_BL1E_ERR BIT(11) -#define S2_BIST_STAT_BIST_BL1_ERR BIT(10) -#define S2_BIST_STAT_BIST_BL0E_ERR BIT(9) -#define S2_BIST_STAT_BIST_BL0_ERR BIT(8) -#define S2_BIST_STAT_BIST_PH1_ERR BIT(7) -#define S2_BIST_STAT_BIST_PH0_ERR BIT(6) -#define S2_BIST_STAT_BIST_PV1_ERR BIT(5) -#define S2_BIST_STAT_BIST_PV0_ERR BIT(4) -#define S2_BIST_STAT_BIST_RUN BIT(3) -#define S2_BIST_STAT_BIST_ERR BIT(2) -#define S2_BIST_STAT_BIST_BUSY BIT(1) -#define S2_BIST_STAT_TCAM_RDY BIT(0) - -#endif /* _OCELOT_S2_CORE_H_ */ diff --git a/drivers/net/ethernet/mscc/ocelot_vcap.c b/drivers/net/ethernet/mscc/ocelot_vcap.c index 3ef620faf995..d8c778ee6f1b 100644 --- a/drivers/net/ethernet/mscc/ocelot_vcap.c +++ b/drivers/net/ethernet/mscc/ocelot_vcap.c @@ -9,9 +9,7 @@ #include <soc/mscc/ocelot_vcap.h> #include "ocelot_police.h" #include "ocelot_vcap.h" -#include "ocelot_s2.h" -#define OCELOT_POLICER_DISCARD 0x17f #define ENTRY_WIDTH 32 enum vcap_sel { @@ -48,145 +46,174 @@ struct vcap_data { u32 tg_mask; /* Current type-group mask */ }; -static u32 vcap_s2_read_update_ctrl(struct ocelot *ocelot) +static u32 vcap_read_update_ctrl(struct ocelot *ocelot, + const struct vcap_props *vcap) { - return ocelot_read(ocelot, S2_CORE_UPDATE_CTRL); + return ocelot_target_read(ocelot, vcap->target, VCAP_CORE_UPDATE_CTRL); } -static void vcap_cmd(struct ocelot *ocelot, u16 ix, int cmd, int sel) +static void vcap_cmd(struct ocelot *ocelot, const struct vcap_props *vcap, + u16 ix, int cmd, int sel) { - const struct vcap_props *vcap_is2 = &ocelot->vcap[VCAP_IS2]; + u32 value = (VCAP_CORE_UPDATE_CTRL_UPDATE_CMD(cmd) | + VCAP_CORE_UPDATE_CTRL_UPDATE_ADDR(ix) | + VCAP_CORE_UPDATE_CTRL_UPDATE_SHOT); - u32 value = (S2_CORE_UPDATE_CTRL_UPDATE_CMD(cmd) | - S2_CORE_UPDATE_CTRL_UPDATE_ADDR(ix) | - S2_CORE_UPDATE_CTRL_UPDATE_SHOT); - - if ((sel & VCAP_SEL_ENTRY) && ix >= vcap_is2->entry_count) + if ((sel & VCAP_SEL_ENTRY) && ix >= vcap->entry_count) return; if (!(sel & VCAP_SEL_ENTRY)) - value |= S2_CORE_UPDATE_CTRL_UPDATE_ENTRY_DIS; + value |= VCAP_CORE_UPDATE_CTRL_UPDATE_ENTRY_DIS; if (!(sel & VCAP_SEL_ACTION)) - value |= S2_CORE_UPDATE_CTRL_UPDATE_ACTION_DIS; + value |= VCAP_CORE_UPDATE_CTRL_UPDATE_ACTION_DIS; if (!(sel & VCAP_SEL_COUNTER)) - value |= S2_CORE_UPDATE_CTRL_UPDATE_CNT_DIS; + value |= VCAP_CORE_UPDATE_CTRL_UPDATE_CNT_DIS; + + ocelot_target_write(ocelot, vcap->target, value, VCAP_CORE_UPDATE_CTRL); - ocelot_write(ocelot, value, S2_CORE_UPDATE_CTRL); - readx_poll_timeout(vcap_s2_read_update_ctrl, ocelot, value, - (value & S2_CORE_UPDATE_CTRL_UPDATE_SHOT) == 0, - 10, 100000); + read_poll_timeout(vcap_read_update_ctrl, value, + (value & VCAP_CORE_UPDATE_CTRL_UPDATE_SHOT) == 0, + 10, 100000, false, ocelot, vcap); } /* Convert from 0-based row to VCAP entry row and run command */ -static void vcap_row_cmd(struct ocelot *ocelot, u32 row, int cmd, int sel) +static void vcap_row_cmd(struct ocelot *ocelot, const struct vcap_props *vcap, + u32 row, int cmd, int sel) { - const struct vcap_props *vcap_is2 = &ocelot->vcap[VCAP_IS2]; - - vcap_cmd(ocelot, vcap_is2->entry_count - row - 1, cmd, sel); + vcap_cmd(ocelot, vcap, vcap->entry_count - row - 1, cmd, sel); } -static void vcap_entry2cache(struct ocelot *ocelot, struct vcap_data *data) +static void vcap_entry2cache(struct ocelot *ocelot, + const struct vcap_props *vcap, + struct vcap_data *data) { - const struct vcap_props *vcap_is2 = &ocelot->vcap[VCAP_IS2]; u32 entry_words, i; - entry_words = DIV_ROUND_UP(vcap_is2->entry_width, ENTRY_WIDTH); + entry_words = DIV_ROUND_UP(vcap->entry_width, ENTRY_WIDTH); for (i = 0; i < entry_words; i++) { - ocelot_write_rix(ocelot, data->entry[i], S2_CACHE_ENTRY_DAT, i); - ocelot_write_rix(ocelot, ~data->mask[i], S2_CACHE_MASK_DAT, i); + ocelot_target_write_rix(ocelot, vcap->target, data->entry[i], + VCAP_CACHE_ENTRY_DAT, i); + ocelot_target_write_rix(ocelot, vcap->target, ~data->mask[i], + VCAP_CACHE_MASK_DAT, i); } - ocelot_write(ocelot, data->tg, S2_CACHE_TG_DAT); + ocelot_target_write(ocelot, vcap->target, data->tg, VCAP_CACHE_TG_DAT); } -static void vcap_cache2entry(struct ocelot *ocelot, struct vcap_data *data) +static void vcap_cache2entry(struct ocelot *ocelot, + const struct vcap_props *vcap, + struct vcap_data *data) { - const struct vcap_props *vcap_is2 = &ocelot->vcap[VCAP_IS2]; u32 entry_words, i; - entry_words = DIV_ROUND_UP(vcap_is2->entry_width, ENTRY_WIDTH); + entry_words = DIV_ROUND_UP(vcap->entry_width, ENTRY_WIDTH); for (i = 0; i < entry_words; i++) { - data->entry[i] = ocelot_read_rix(ocelot, S2_CACHE_ENTRY_DAT, i); + data->entry[i] = ocelot_target_read_rix(ocelot, vcap->target, + VCAP_CACHE_ENTRY_DAT, i); // Invert mask - data->mask[i] = ~ocelot_read_rix(ocelot, S2_CACHE_MASK_DAT, i); + data->mask[i] = ~ocelot_target_read_rix(ocelot, vcap->target, + VCAP_CACHE_MASK_DAT, i); } - data->tg = ocelot_read(ocelot, S2_CACHE_TG_DAT); + data->tg = ocelot_target_read(ocelot, vcap->target, VCAP_CACHE_TG_DAT); } -static void vcap_action2cache(struct ocelot *ocelot, struct vcap_data *data) +static void vcap_action2cache(struct ocelot *ocelot, + const struct vcap_props *vcap, + struct vcap_data *data) { - const struct vcap_props *vcap_is2 = &ocelot->vcap[VCAP_IS2]; u32 action_words, mask; int i, width; /* Encode action type */ - width = vcap_is2->action_type_width; + width = vcap->action_type_width; if (width) { mask = GENMASK(width, 0); data->action[0] = ((data->action[0] & ~mask) | data->type); } - action_words = DIV_ROUND_UP(vcap_is2->action_width, ENTRY_WIDTH); + action_words = DIV_ROUND_UP(vcap->action_width, ENTRY_WIDTH); for (i = 0; i < action_words; i++) - ocelot_write_rix(ocelot, data->action[i], S2_CACHE_ACTION_DAT, - i); + ocelot_target_write_rix(ocelot, vcap->target, data->action[i], + VCAP_CACHE_ACTION_DAT, i); - for (i = 0; i < vcap_is2->counter_words; i++) - ocelot_write_rix(ocelot, data->counter[i], S2_CACHE_CNT_DAT, i); + for (i = 0; i < vcap->counter_words; i++) + ocelot_target_write_rix(ocelot, vcap->target, data->counter[i], + VCAP_CACHE_CNT_DAT, i); } -static void vcap_cache2action(struct ocelot *ocelot, struct vcap_data *data) +static void vcap_cache2action(struct ocelot *ocelot, + const struct vcap_props *vcap, + struct vcap_data *data) { - const struct vcap_props *vcap_is2 = &ocelot->vcap[VCAP_IS2]; u32 action_words; int i, width; - action_words = DIV_ROUND_UP(vcap_is2->action_width, ENTRY_WIDTH); + action_words = DIV_ROUND_UP(vcap->action_width, ENTRY_WIDTH); for (i = 0; i < action_words; i++) - data->action[i] = ocelot_read_rix(ocelot, S2_CACHE_ACTION_DAT, - i); + data->action[i] = ocelot_target_read_rix(ocelot, vcap->target, + VCAP_CACHE_ACTION_DAT, + i); - for (i = 0; i < vcap_is2->counter_words; i++) - data->counter[i] = ocelot_read_rix(ocelot, S2_CACHE_CNT_DAT, i); + for (i = 0; i < vcap->counter_words; i++) + data->counter[i] = ocelot_target_read_rix(ocelot, vcap->target, + VCAP_CACHE_CNT_DAT, + i); /* Extract action type */ - width = vcap_is2->action_type_width; + width = vcap->action_type_width; data->type = (width ? (data->action[0] & GENMASK(width, 0)) : 0); } /* Calculate offsets for entry */ -static void is2_data_get(struct ocelot *ocelot, struct vcap_data *data, int ix) +static void vcap_data_offset_get(const struct vcap_props *vcap, + struct vcap_data *data, int ix) { - const struct vcap_props *vcap_is2 = &ocelot->vcap[VCAP_IS2]; - int i, col, offset, count, cnt, base; - int width = vcap_is2->tg_width; + int num_subwords_per_entry, num_subwords_per_action; + int i, col, offset, num_entries_per_row, base; + u32 width = vcap->tg_width; - count = (data->tg_sw == VCAP_TG_HALF ? 2 : 4); - col = (ix % 2); - cnt = (vcap_is2->sw_count / count); - base = (vcap_is2->sw_count - col * cnt - cnt); + switch (data->tg_sw) { + case VCAP_TG_FULL: + num_entries_per_row = 1; + break; + case VCAP_TG_HALF: + num_entries_per_row = 2; + break; + case VCAP_TG_QUARTER: + num_entries_per_row = 4; + break; + default: + return; + } + + col = (ix % num_entries_per_row); + num_subwords_per_entry = (vcap->sw_count / num_entries_per_row); + base = (vcap->sw_count - col * num_subwords_per_entry - + num_subwords_per_entry); data->tg_value = 0; data->tg_mask = 0; - for (i = 0; i < cnt; i++) { + for (i = 0; i < num_subwords_per_entry; i++) { offset = ((base + i) * width); data->tg_value |= (data->tg_sw << offset); data->tg_mask |= GENMASK(offset + width - 1, offset); } /* Calculate key/action/counter offsets */ - col = (count - col - 1); - data->key_offset = (base * vcap_is2->entry_width) / vcap_is2->sw_count; - data->counter_offset = (cnt * col * vcap_is2->counter_width); + col = (num_entries_per_row - col - 1); + data->key_offset = (base * vcap->entry_width) / vcap->sw_count; + data->counter_offset = (num_subwords_per_entry * col * + vcap->counter_width); i = data->type; - width = vcap_is2->action_table[i].width; - cnt = vcap_is2->action_table[i].count; - data->action_offset = - (((cnt * col * width) / count) + vcap_is2->action_type_width); + width = vcap->action_table[i].width; + num_subwords_per_action = vcap->action_table[i].count; + data->action_offset = ((num_subwords_per_action * col * width) / + num_entries_per_row); + data->action_offset += vcap->action_type_width; } static void vcap_data_set(u32 *data, u32 offset, u32 len, u32 value) @@ -224,22 +251,21 @@ static void vcap_key_field_set(struct vcap_data *data, u32 offset, u32 width, vcap_data_set(data->mask, offset + data->key_offset, width, mask); } -static void vcap_key_set(struct ocelot *ocelot, struct vcap_data *data, - enum vcap_is2_half_key_field field, - u32 value, u32 mask) +static void vcap_key_set(const struct vcap_props *vcap, struct vcap_data *data, + int field, u32 value, u32 mask) { - u32 offset = ocelot->vcap_is2_keys[field].offset; - u32 length = ocelot->vcap_is2_keys[field].length; + u32 offset = vcap->keys[field].offset; + u32 length = vcap->keys[field].length; vcap_key_field_set(data, offset, length, value, mask); } -static void vcap_key_bytes_set(struct ocelot *ocelot, struct vcap_data *data, - enum vcap_is2_half_key_field field, +static void vcap_key_bytes_set(const struct vcap_props *vcap, + struct vcap_data *data, int field, u8 *val, u8 *msk) { - u32 offset = ocelot->vcap_is2_keys[field].offset; - u32 count = ocelot->vcap_is2_keys[field].length; + u32 offset = vcap->keys[field].offset; + u32 count = vcap->keys[field].length; u32 i, j, n = 0, value = 0, mask = 0; WARN_ON(count % 8); @@ -265,37 +291,37 @@ static void vcap_key_bytes_set(struct ocelot *ocelot, struct vcap_data *data, } } -static void vcap_key_l4_port_set(struct ocelot *ocelot, struct vcap_data *data, - enum vcap_is2_half_key_field field, +static void vcap_key_l4_port_set(const struct vcap_props *vcap, + struct vcap_data *data, int field, struct ocelot_vcap_udp_tcp *port) { - u32 offset = ocelot->vcap_is2_keys[field].offset; - u32 length = ocelot->vcap_is2_keys[field].length; + u32 offset = vcap->keys[field].offset; + u32 length = vcap->keys[field].length; WARN_ON(length != 16); vcap_key_field_set(data, offset, length, port->value, port->mask); } -static void vcap_key_bit_set(struct ocelot *ocelot, struct vcap_data *data, - enum vcap_is2_half_key_field field, +static void vcap_key_bit_set(const struct vcap_props *vcap, + struct vcap_data *data, int field, enum ocelot_vcap_bit val) { - u32 offset = ocelot->vcap_is2_keys[field].offset; - u32 length = ocelot->vcap_is2_keys[field].length; u32 value = (val == OCELOT_VCAP_BIT_1 ? 1 : 0); u32 msk = (val == OCELOT_VCAP_BIT_ANY ? 0 : 1); + u32 offset = vcap->keys[field].offset; + u32 length = vcap->keys[field].length; WARN_ON(length != 1); vcap_key_field_set(data, offset, length, value, msk); } -static void vcap_action_set(struct ocelot *ocelot, struct vcap_data *data, - enum vcap_is2_action_field field, u32 value) +static void vcap_action_set(const struct vcap_props *vcap, + struct vcap_data *data, int field, u32 value) { - int offset = ocelot->vcap_is2_actions[field].offset; - int length = ocelot->vcap_is2_actions[field].length; + int offset = vcap->actions[field].offset; + int length = vcap->actions[field].length; vcap_data_set(data->action, offset + data->action_offset, length, value); @@ -304,40 +330,21 @@ static void vcap_action_set(struct ocelot *ocelot, struct vcap_data *data, static void is2_action_set(struct ocelot *ocelot, struct vcap_data *data, struct ocelot_vcap_filter *filter) { - switch (filter->action) { - case OCELOT_VCAP_ACTION_DROP: - vcap_action_set(ocelot, data, VCAP_IS2_ACT_PORT_MASK, 0); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_MASK_MODE, 1); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_POLICE_ENA, 1); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_POLICE_IDX, - OCELOT_POLICER_DISCARD); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_CPU_QU_NUM, 0); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_CPU_COPY_ENA, 0); - break; - case OCELOT_VCAP_ACTION_TRAP: - vcap_action_set(ocelot, data, VCAP_IS2_ACT_PORT_MASK, 0); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_MASK_MODE, 1); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_POLICE_ENA, 0); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_POLICE_IDX, 0); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_CPU_QU_NUM, 0); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_CPU_COPY_ENA, 1); - break; - case OCELOT_VCAP_ACTION_POLICE: - vcap_action_set(ocelot, data, VCAP_IS2_ACT_PORT_MASK, 0); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_MASK_MODE, 0); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_POLICE_ENA, 1); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_POLICE_IDX, - filter->pol_ix); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_CPU_QU_NUM, 0); - vcap_action_set(ocelot, data, VCAP_IS2_ACT_CPU_COPY_ENA, 0); - break; - } + const struct vcap_props *vcap = &ocelot->vcap[VCAP_IS2]; + struct ocelot_vcap_action *a = &filter->action; + + vcap_action_set(vcap, data, VCAP_IS2_ACT_MASK_MODE, a->mask_mode); + vcap_action_set(vcap, data, VCAP_IS2_ACT_PORT_MASK, a->port_mask); + vcap_action_set(vcap, data, VCAP_IS2_ACT_POLICE_ENA, a->police_ena); + vcap_action_set(vcap, data, VCAP_IS2_ACT_POLICE_IDX, a->pol_ix); + vcap_action_set(vcap, data, VCAP_IS2_ACT_CPU_QU_NUM, a->cpu_qu_num); + vcap_action_set(vcap, data, VCAP_IS2_ACT_CPU_COPY_ENA, a->cpu_copy_ena); } static void is2_entry_set(struct ocelot *ocelot, int ix, struct ocelot_vcap_filter *filter) { - const struct vcap_props *vcap_is2 = &ocelot->vcap[VCAP_IS2]; + const struct vcap_props *vcap = &ocelot->vcap[VCAP_IS2]; struct ocelot_vcap_key_vlan *tag = &filter->vlan; u32 val, msk, type, type_mask = 0xf, i, count; struct ocelot_vcap_u64 payload; @@ -348,52 +355,55 @@ static void is2_entry_set(struct ocelot *ocelot, int ix, memset(&data, 0, sizeof(data)); /* Read row */ - vcap_row_cmd(ocelot, row, VCAP_CMD_READ, VCAP_SEL_ALL); - vcap_cache2entry(ocelot, &data); - vcap_cache2action(ocelot, &data); + vcap_row_cmd(ocelot, vcap, row, VCAP_CMD_READ, VCAP_SEL_ALL); + vcap_cache2entry(ocelot, vcap, &data); + vcap_cache2action(ocelot, vcap, &data); data.tg_sw = VCAP_TG_HALF; - is2_data_get(ocelot, &data, ix); + vcap_data_offset_get(vcap, &data, ix); data.tg = (data.tg & ~data.tg_mask); if (filter->prio != 0) data.tg |= data.tg_value; data.type = IS2_ACTION_TYPE_NORMAL; - vcap_key_set(ocelot, &data, VCAP_IS2_HK_PAG, 0, 0); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_IGR_PORT_MASK, 0, + vcap_key_set(vcap, &data, VCAP_IS2_HK_PAG, filter->pag, 0xff); + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_FIRST, + (filter->lookup == 0) ? OCELOT_VCAP_BIT_1 : + OCELOT_VCAP_BIT_0); + vcap_key_set(vcap, &data, VCAP_IS2_HK_IGR_PORT_MASK, 0, ~filter->ingress_port_mask); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_FIRST, OCELOT_VCAP_BIT_1); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_HOST_MATCH, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_FIRST, OCELOT_VCAP_BIT_ANY); + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_HOST_MATCH, OCELOT_VCAP_BIT_ANY); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_L2_MC, filter->dmac_mc); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_L2_BC, filter->dmac_bc); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_VLAN_TAGGED, tag->tagged); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_VID, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L2_MC, filter->dmac_mc); + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L2_BC, filter->dmac_bc); + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_VLAN_TAGGED, tag->tagged); + vcap_key_set(vcap, &data, VCAP_IS2_HK_VID, tag->vid.value, tag->vid.mask); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_PCP, + vcap_key_set(vcap, &data, VCAP_IS2_HK_PCP, tag->pcp.value[0], tag->pcp.mask[0]); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_DEI, tag->dei); + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_DEI, tag->dei); switch (filter->key_type) { case OCELOT_VCAP_KEY_ETYPE: { struct ocelot_vcap_key_etype *etype = &filter->key.etype; type = IS2_TYPE_ETYPE; - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_L2_DMAC, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_L2_DMAC, etype->dmac.value, etype->dmac.mask); - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_L2_SMAC, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_L2_SMAC, etype->smac.value, etype->smac.mask); - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_MAC_ETYPE_ETYPE, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_MAC_ETYPE_ETYPE, etype->etype.value, etype->etype.mask); /* Clear unused bits */ - vcap_key_set(ocelot, &data, VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD0, + vcap_key_set(vcap, &data, VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD0, 0, 0); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD1, + vcap_key_set(vcap, &data, VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD1, 0, 0); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD2, + vcap_key_set(vcap, &data, VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD2, 0, 0); - vcap_key_bytes_set(ocelot, &data, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD0, etype->data.value, etype->data.mask); break; @@ -402,15 +412,15 @@ static void is2_entry_set(struct ocelot *ocelot, int ix, struct ocelot_vcap_key_llc *llc = &filter->key.llc; type = IS2_TYPE_LLC; - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_L2_DMAC, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_L2_DMAC, llc->dmac.value, llc->dmac.mask); - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_L2_SMAC, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_L2_SMAC, llc->smac.value, llc->smac.mask); for (i = 0; i < 4; i++) { payload.value[i] = llc->llc.value[i]; payload.mask[i] = llc->llc.mask[i]; } - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_MAC_LLC_L2_LLC, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_MAC_LLC_L2_LLC, payload.value, payload.mask); break; } @@ -418,11 +428,11 @@ static void is2_entry_set(struct ocelot *ocelot, int ix, struct ocelot_vcap_key_snap *snap = &filter->key.snap; type = IS2_TYPE_SNAP; - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_L2_DMAC, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_L2_DMAC, snap->dmac.value, snap->dmac.mask); - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_L2_SMAC, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_L2_SMAC, snap->smac.value, snap->smac.mask); - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_MAC_SNAP_L2_SNAP, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_MAC_SNAP_L2_SNAP, filter->key.snap.snap.value, filter->key.snap.snap.mask); break; @@ -431,24 +441,24 @@ static void is2_entry_set(struct ocelot *ocelot, int ix, struct ocelot_vcap_key_arp *arp = &filter->key.arp; type = IS2_TYPE_ARP; - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_MAC_ARP_SMAC, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_SMAC, arp->smac.value, arp->smac.mask); - vcap_key_bit_set(ocelot, &data, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_ADDR_SPACE_OK, arp->ethernet); - vcap_key_bit_set(ocelot, &data, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_PROTO_SPACE_OK, arp->ip); - vcap_key_bit_set(ocelot, &data, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_LEN_OK, arp->length); - vcap_key_bit_set(ocelot, &data, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_TARGET_MATCH, arp->dmac_match); - vcap_key_bit_set(ocelot, &data, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_SENDER_MATCH, arp->smac_match); - vcap_key_bit_set(ocelot, &data, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_OPCODE_UNKNOWN, arp->unknown); @@ -457,15 +467,15 @@ static void is2_entry_set(struct ocelot *ocelot, int ix, (arp->arp == OCELOT_VCAP_BIT_0 ? 2 : 0)); msk = ((arp->req == OCELOT_VCAP_BIT_ANY ? 0 : 1) | (arp->arp == OCELOT_VCAP_BIT_ANY ? 0 : 2)); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_MAC_ARP_OPCODE, + vcap_key_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_OPCODE, val, msk); - vcap_key_bytes_set(ocelot, &data, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_L3_IP4_DIP, arp->dip.value.addr, arp->dip.mask.addr); - vcap_key_bytes_set(ocelot, &data, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_L3_IP4_SIP, arp->sip.value.addr, arp->sip.mask.addr); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_MAC_ARP_DIP_EQ_SIP, + vcap_key_set(vcap, &data, VCAP_IS2_HK_MAC_ARP_DIP_EQ_SIP, 0, 0); break; } @@ -534,22 +544,22 @@ static void is2_entry_set(struct ocelot *ocelot, int ix, seq_zero = ipv6->seq_zero; } - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_IP4, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_IP4, ipv4 ? OCELOT_VCAP_BIT_1 : OCELOT_VCAP_BIT_0); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_L3_FRAGMENT, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L3_FRAGMENT, fragment); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_L3_FRAG_OFS_GT0, 0, 0); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_L3_OPTIONS, + vcap_key_set(vcap, &data, VCAP_IS2_HK_L3_FRAG_OFS_GT0, 0, 0); + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L3_OPTIONS, options); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_IP4_L3_TTL_GT0, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_IP4_L3_TTL_GT0, ttl); - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_L3_TOS, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_L3_TOS, ds.value, ds.mask); - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_L3_IP4_DIP, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_L3_IP4_DIP, dip.value.addr, dip.mask.addr); - vcap_key_bytes_set(ocelot, &data, VCAP_IS2_HK_L3_IP4_SIP, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_L3_IP4_SIP, sip.value.addr, sip.mask.addr); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_DIP_EQ_SIP, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_DIP_EQ_SIP, sip_eq_dip); val = proto.value[0]; msk = proto.mask[0]; @@ -558,33 +568,33 @@ static void is2_entry_set(struct ocelot *ocelot, int ix, /* UDP/TCP protocol match */ tcp = (val == 6 ? OCELOT_VCAP_BIT_1 : OCELOT_VCAP_BIT_0); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_TCP, tcp); - vcap_key_l4_port_set(ocelot, &data, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_TCP, tcp); + vcap_key_l4_port_set(vcap, &data, VCAP_IS2_HK_L4_DPORT, dport); - vcap_key_l4_port_set(ocelot, &data, + vcap_key_l4_port_set(vcap, &data, VCAP_IS2_HK_L4_SPORT, sport); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_L4_RNG, 0, 0); - vcap_key_bit_set(ocelot, &data, + vcap_key_set(vcap, &data, VCAP_IS2_HK_L4_RNG, 0, 0); + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L4_SPORT_EQ_DPORT, sport_eq_dport); - vcap_key_bit_set(ocelot, &data, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L4_SEQUENCE_EQ0, seq_zero); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_L4_FIN, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L4_FIN, tcp_fin); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_L4_SYN, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L4_SYN, tcp_syn); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_L4_RST, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L4_RST, tcp_rst); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_L4_PSH, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L4_PSH, tcp_psh); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_L4_ACK, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L4_ACK, tcp_ack); - vcap_key_bit_set(ocelot, &data, VCAP_IS2_HK_L4_URG, + vcap_key_bit_set(vcap, &data, VCAP_IS2_HK_L4_URG, tcp_urg); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_L4_1588_DOM, + vcap_key_set(vcap, &data, VCAP_IS2_HK_L4_1588_DOM, 0, 0); - vcap_key_set(ocelot, &data, VCAP_IS2_HK_L4_1588_VER, + vcap_key_set(vcap, &data, VCAP_IS2_HK_L4_1588_VER, 0, 0); } else { if (msk == 0) { @@ -598,10 +608,10 @@ static void is2_entry_set(struct ocelot *ocelot, int ix, payload.mask[i] = ip_data->mask[i]; } } - vcap_key_bytes_set(ocelot, &data, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_IP4_L3_PROTO, proto.value, proto.mask); - vcap_key_bytes_set(ocelot, &data, + vcap_key_bytes_set(vcap, &data, VCAP_IS2_HK_L3_PAYLOAD, payload.value, payload.mask); } @@ -611,46 +621,271 @@ static void is2_entry_set(struct ocelot *ocelot, int ix, default: type = 0; type_mask = 0; - count = vcap_is2->entry_width / 2; + count = vcap->entry_width / 2; /* Iterate over the non-common part of the key and * clear entry data */ - for (i = ocelot->vcap_is2_keys[VCAP_IS2_HK_L2_DMAC].offset; + for (i = vcap->keys[VCAP_IS2_HK_L2_DMAC].offset; i < count; i += ENTRY_WIDTH) { vcap_key_field_set(&data, i, min(32u, count - i), 0, 0); } break; } - vcap_key_set(ocelot, &data, VCAP_IS2_TYPE, type, type_mask); + vcap_key_set(vcap, &data, VCAP_IS2_TYPE, type, type_mask); is2_action_set(ocelot, &data, filter); vcap_data_set(data.counter, data.counter_offset, - vcap_is2->counter_width, filter->stats.pkts); + vcap->counter_width, filter->stats.pkts); /* Write row */ - vcap_entry2cache(ocelot, &data); - vcap_action2cache(ocelot, &data); - vcap_row_cmd(ocelot, row, VCAP_CMD_WRITE, VCAP_SEL_ALL); + vcap_entry2cache(ocelot, vcap, &data); + vcap_action2cache(ocelot, vcap, &data); + vcap_row_cmd(ocelot, vcap, row, VCAP_CMD_WRITE, VCAP_SEL_ALL); +} + +static void is1_action_set(struct ocelot *ocelot, struct vcap_data *data, + const struct ocelot_vcap_filter *filter) +{ + const struct vcap_props *vcap = &ocelot->vcap[VCAP_IS1]; + const struct ocelot_vcap_action *a = &filter->action; + + vcap_action_set(vcap, data, VCAP_IS1_ACT_VID_REPLACE_ENA, + a->vid_replace_ena); + vcap_action_set(vcap, data, VCAP_IS1_ACT_VID_ADD_VAL, a->vid); + vcap_action_set(vcap, data, VCAP_IS1_ACT_VLAN_POP_CNT_ENA, + a->vlan_pop_cnt_ena); + vcap_action_set(vcap, data, VCAP_IS1_ACT_VLAN_POP_CNT, + a->vlan_pop_cnt); + vcap_action_set(vcap, data, VCAP_IS1_ACT_PCP_DEI_ENA, a->pcp_dei_ena); + vcap_action_set(vcap, data, VCAP_IS1_ACT_PCP_VAL, a->pcp); + vcap_action_set(vcap, data, VCAP_IS1_ACT_DEI_VAL, a->dei); + vcap_action_set(vcap, data, VCAP_IS1_ACT_QOS_ENA, a->qos_ena); + vcap_action_set(vcap, data, VCAP_IS1_ACT_QOS_VAL, a->qos_val); + vcap_action_set(vcap, data, VCAP_IS1_ACT_PAG_OVERRIDE_MASK, + a->pag_override_mask); + vcap_action_set(vcap, data, VCAP_IS1_ACT_PAG_VAL, a->pag_val); } -static void is2_entry_get(struct ocelot *ocelot, struct ocelot_vcap_filter *filter, - int ix) +static void is1_entry_set(struct ocelot *ocelot, int ix, + struct ocelot_vcap_filter *filter) { - const struct vcap_props *vcap_is2 = &ocelot->vcap[VCAP_IS2]; + const struct vcap_props *vcap = &ocelot->vcap[VCAP_IS1]; + struct ocelot_vcap_key_vlan *tag = &filter->vlan; + struct ocelot_vcap_u64 payload; struct vcap_data data; - int row = (ix / 2); - u32 cnt; + int row = ix / 2; + u32 type; + + memset(&payload, 0, sizeof(payload)); + memset(&data, 0, sizeof(data)); + + /* Read row */ + vcap_row_cmd(ocelot, vcap, row, VCAP_CMD_READ, VCAP_SEL_ALL); + vcap_cache2entry(ocelot, vcap, &data); + vcap_cache2action(ocelot, vcap, &data); - vcap_row_cmd(ocelot, row, VCAP_CMD_READ, VCAP_SEL_COUNTER); - vcap_cache2action(ocelot, &data); data.tg_sw = VCAP_TG_HALF; - is2_data_get(ocelot, &data, ix); + data.type = IS1_ACTION_TYPE_NORMAL; + vcap_data_offset_get(vcap, &data, ix); + data.tg = (data.tg & ~data.tg_mask); + if (filter->prio != 0) + data.tg |= data.tg_value; + + vcap_key_set(vcap, &data, VCAP_IS1_HK_LOOKUP, filter->lookup, 0x3); + vcap_key_set(vcap, &data, VCAP_IS1_HK_IGR_PORT_MASK, 0, + ~filter->ingress_port_mask); + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_L2_MC, filter->dmac_mc); + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_L2_BC, filter->dmac_bc); + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_VLAN_TAGGED, tag->tagged); + vcap_key_set(vcap, &data, VCAP_IS1_HK_VID, + tag->vid.value, tag->vid.mask); + vcap_key_set(vcap, &data, VCAP_IS1_HK_PCP, + tag->pcp.value[0], tag->pcp.mask[0]); + type = IS1_TYPE_S1_NORMAL; + + switch (filter->key_type) { + case OCELOT_VCAP_KEY_ETYPE: { + struct ocelot_vcap_key_etype *etype = &filter->key.etype; + + vcap_key_bytes_set(vcap, &data, VCAP_IS1_HK_L2_SMAC, + etype->smac.value, etype->smac.mask); + vcap_key_bytes_set(vcap, &data, VCAP_IS1_HK_ETYPE, + etype->etype.value, etype->etype.mask); + break; + } + case OCELOT_VCAP_KEY_IPV4: { + struct ocelot_vcap_key_ipv4 *ipv4 = &filter->key.ipv4; + struct ocelot_vcap_udp_tcp *sport = &ipv4->sport; + struct ocelot_vcap_udp_tcp *dport = &ipv4->dport; + enum ocelot_vcap_bit tcp_udp = OCELOT_VCAP_BIT_0; + struct ocelot_vcap_u8 proto = ipv4->proto; + struct ocelot_vcap_ipv4 sip = ipv4->sip; + u32 val, msk; + + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_IP_SNAP, + OCELOT_VCAP_BIT_1); + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_IP4, + OCELOT_VCAP_BIT_1); + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_ETYPE_LEN, + OCELOT_VCAP_BIT_1); + vcap_key_bytes_set(vcap, &data, VCAP_IS1_HK_L3_IP4_SIP, + sip.value.addr, sip.mask.addr); + + val = proto.value[0]; + msk = proto.mask[0]; + + if ((val == NEXTHDR_TCP || val == NEXTHDR_UDP) && msk == 0xff) + tcp_udp = OCELOT_VCAP_BIT_1; + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_TCP_UDP, tcp_udp); + + if (tcp_udp) { + enum ocelot_vcap_bit tcp = OCELOT_VCAP_BIT_0; + + if (val == NEXTHDR_TCP) + tcp = OCELOT_VCAP_BIT_1; + + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_TCP, tcp); + vcap_key_l4_port_set(vcap, &data, VCAP_IS1_HK_L4_SPORT, + sport); + /* Overloaded field */ + vcap_key_l4_port_set(vcap, &data, VCAP_IS1_HK_ETYPE, + dport); + } else { + /* IPv4 "other" frame */ + struct ocelot_vcap_u16 etype = {0}; + + /* Overloaded field */ + etype.value[0] = proto.value[0]; + etype.mask[0] = proto.mask[0]; + + vcap_key_bytes_set(vcap, &data, VCAP_IS1_HK_ETYPE, + etype.value, etype.mask); + } + } + default: + break; + } + vcap_key_bit_set(vcap, &data, VCAP_IS1_HK_TYPE, + type ? OCELOT_VCAP_BIT_1 : OCELOT_VCAP_BIT_0); + + is1_action_set(ocelot, &data, filter); + vcap_data_set(data.counter, data.counter_offset, + vcap->counter_width, filter->stats.pkts); + + /* Write row */ + vcap_entry2cache(ocelot, vcap, &data); + vcap_action2cache(ocelot, vcap, &data); + vcap_row_cmd(ocelot, vcap, row, VCAP_CMD_WRITE, VCAP_SEL_ALL); +} + +static void es0_action_set(struct ocelot *ocelot, struct vcap_data *data, + const struct ocelot_vcap_filter *filter) +{ + const struct vcap_props *vcap = &ocelot->vcap[VCAP_ES0]; + const struct ocelot_vcap_action *a = &filter->action; + + vcap_action_set(vcap, data, VCAP_ES0_ACT_PUSH_OUTER_TAG, + a->push_outer_tag); + vcap_action_set(vcap, data, VCAP_ES0_ACT_PUSH_INNER_TAG, + a->push_inner_tag); + vcap_action_set(vcap, data, VCAP_ES0_ACT_TAG_A_TPID_SEL, + a->tag_a_tpid_sel); + vcap_action_set(vcap, data, VCAP_ES0_ACT_TAG_A_VID_SEL, + a->tag_a_vid_sel); + vcap_action_set(vcap, data, VCAP_ES0_ACT_TAG_A_PCP_SEL, + a->tag_a_pcp_sel); + vcap_action_set(vcap, data, VCAP_ES0_ACT_VID_A_VAL, a->vid_a_val); + vcap_action_set(vcap, data, VCAP_ES0_ACT_PCP_A_VAL, a->pcp_a_val); + vcap_action_set(vcap, data, VCAP_ES0_ACT_TAG_B_TPID_SEL, + a->tag_b_tpid_sel); + vcap_action_set(vcap, data, VCAP_ES0_ACT_TAG_B_VID_SEL, + a->tag_b_vid_sel); + vcap_action_set(vcap, data, VCAP_ES0_ACT_TAG_B_PCP_SEL, + a->tag_b_pcp_sel); + vcap_action_set(vcap, data, VCAP_ES0_ACT_VID_B_VAL, a->vid_b_val); + vcap_action_set(vcap, data, VCAP_ES0_ACT_PCP_B_VAL, a->pcp_b_val); +} + +static void es0_entry_set(struct ocelot *ocelot, int ix, + struct ocelot_vcap_filter *filter) +{ + const struct vcap_props *vcap = &ocelot->vcap[VCAP_ES0]; + struct ocelot_vcap_key_vlan *tag = &filter->vlan; + struct ocelot_vcap_u64 payload; + struct vcap_data data; + int row = ix; + + memset(&payload, 0, sizeof(payload)); + memset(&data, 0, sizeof(data)); + + /* Read row */ + vcap_row_cmd(ocelot, vcap, row, VCAP_CMD_READ, VCAP_SEL_ALL); + vcap_cache2entry(ocelot, vcap, &data); + vcap_cache2action(ocelot, vcap, &data); + + data.tg_sw = VCAP_TG_FULL; + data.type = ES0_ACTION_TYPE_NORMAL; + vcap_data_offset_get(vcap, &data, ix); + data.tg = (data.tg & ~data.tg_mask); + if (filter->prio != 0) + data.tg |= data.tg_value; + + vcap_key_set(vcap, &data, VCAP_ES0_IGR_PORT, filter->ingress_port.value, + filter->ingress_port.mask); + vcap_key_set(vcap, &data, VCAP_ES0_EGR_PORT, filter->egress_port.value, + filter->egress_port.mask); + vcap_key_bit_set(vcap, &data, VCAP_ES0_L2_MC, filter->dmac_mc); + vcap_key_bit_set(vcap, &data, VCAP_ES0_L2_BC, filter->dmac_bc); + vcap_key_set(vcap, &data, VCAP_ES0_VID, + tag->vid.value, tag->vid.mask); + vcap_key_set(vcap, &data, VCAP_ES0_PCP, + tag->pcp.value[0], tag->pcp.mask[0]); + + es0_action_set(ocelot, &data, filter); + vcap_data_set(data.counter, data.counter_offset, + vcap->counter_width, filter->stats.pkts); + + /* Write row */ + vcap_entry2cache(ocelot, vcap, &data); + vcap_action2cache(ocelot, vcap, &data); + vcap_row_cmd(ocelot, vcap, row, VCAP_CMD_WRITE, VCAP_SEL_ALL); +} + +static void vcap_entry_get(struct ocelot *ocelot, int ix, + struct ocelot_vcap_filter *filter) +{ + const struct vcap_props *vcap = &ocelot->vcap[filter->block_id]; + struct vcap_data data; + int row, count; + u32 cnt; + + if (filter->block_id == VCAP_ES0) + data.tg_sw = VCAP_TG_FULL; + else + data.tg_sw = VCAP_TG_HALF; + + count = (1 << (data.tg_sw - 1)); + row = (ix / count); + vcap_row_cmd(ocelot, vcap, row, VCAP_CMD_READ, VCAP_SEL_COUNTER); + vcap_cache2action(ocelot, vcap, &data); + vcap_data_offset_get(vcap, &data, ix); cnt = vcap_data_get(data.counter, data.counter_offset, - vcap_is2->counter_width); + vcap->counter_width); filter->stats.pkts = cnt; } +static void vcap_entry_set(struct ocelot *ocelot, int ix, + struct ocelot_vcap_filter *filter) +{ + if (filter->block_id == VCAP_IS1) + return is1_entry_set(ocelot, ix, filter); + if (filter->block_id == VCAP_IS2) + return is2_entry_set(ocelot, ix, filter); + if (filter->block_id == VCAP_ES0) + return es0_entry_set(ocelot, ix, filter); +} + static int ocelot_vcap_policer_add(struct ocelot *ocelot, u32 pol_ix, struct ocelot_policer *pol) { @@ -679,11 +914,12 @@ static void ocelot_vcap_policer_del(struct ocelot *ocelot, list_for_each_entry(filter, &block->rules, list) { index++; - if (filter->action == OCELOT_VCAP_ACTION_POLICE && - filter->pol_ix < pol_ix) { - filter->pol_ix += 1; - ocelot_vcap_policer_add(ocelot, filter->pol_ix, - &filter->pol); + if (filter->block_id == VCAP_IS2 && + filter->action.police_ena && + filter->action.pol_ix < pol_ix) { + filter->action.pol_ix += 1; + ocelot_vcap_policer_add(ocelot, filter->action.pol_ix, + &filter->action.pol); is2_entry_set(ocelot, index, filter); } } @@ -701,10 +937,11 @@ static void ocelot_vcap_filter_add_to_block(struct ocelot *ocelot, struct ocelot_vcap_filter *tmp; struct list_head *pos, *n; - if (filter->action == OCELOT_VCAP_ACTION_POLICE) { + if (filter->block_id == VCAP_IS2 && filter->action.police_ena) { block->pol_lpr--; - filter->pol_ix = block->pol_lpr; - ocelot_vcap_policer_add(ocelot, filter->pol_ix, &filter->pol); + filter->action.pol_ix = block->pol_lpr; + ocelot_vcap_policer_add(ocelot, filter->action.pol_ix, + &filter->action.pol); } block->count++; @@ -726,19 +963,20 @@ static int ocelot_vcap_block_get_filter_index(struct ocelot_vcap_block *block, struct ocelot_vcap_filter *filter) { struct ocelot_vcap_filter *tmp; - int index = -1; + int index = 0; list_for_each_entry(tmp, &block->rules, list) { - ++index; if (filter->id == tmp->id) - break; + return index; + index++; } - return index; + + return -ENOENT; } static struct ocelot_vcap_filter* -ocelot_vcap_block_find_filter(struct ocelot_vcap_block *block, - int index) +ocelot_vcap_block_find_filter_by_index(struct ocelot_vcap_block *block, + int index) { struct ocelot_vcap_filter *tmp; int i = 0; @@ -752,6 +990,18 @@ ocelot_vcap_block_find_filter(struct ocelot_vcap_block *block, return NULL; } +struct ocelot_vcap_filter * +ocelot_vcap_block_find_filter_by_id(struct ocelot_vcap_block *block, int id) +{ + struct ocelot_vcap_filter *filter; + + list_for_each_entry(filter, &block->rules, list) + if (filter->id == id) + return filter; + + return NULL; +} + /* If @on=false, then SNAP, ARP, IP and OAM frames will not match on keys based * on destination and source MAC addresses, but only on higher-level protocol * information. The only frame types to match on keys containing MAC addresses @@ -763,23 +1013,23 @@ ocelot_vcap_block_find_filter(struct ocelot_vcap_block *block, * on any _other_ keys than MAC_ETYPE ones. */ static void ocelot_match_all_as_mac_etype(struct ocelot *ocelot, int port, - bool on) + int lookup, bool on) { u32 val = 0; if (on) - val = ANA_PORT_VCAP_S2_CFG_S2_SNAP_DIS(3) | - ANA_PORT_VCAP_S2_CFG_S2_ARP_DIS(3) | - ANA_PORT_VCAP_S2_CFG_S2_IP_TCPUDP_DIS(3) | - ANA_PORT_VCAP_S2_CFG_S2_IP_OTHER_DIS(3) | - ANA_PORT_VCAP_S2_CFG_S2_OAM_DIS(3); + val = ANA_PORT_VCAP_S2_CFG_S2_SNAP_DIS(BIT(lookup)) | + ANA_PORT_VCAP_S2_CFG_S2_ARP_DIS(BIT(lookup)) | + ANA_PORT_VCAP_S2_CFG_S2_IP_TCPUDP_DIS(BIT(lookup)) | + ANA_PORT_VCAP_S2_CFG_S2_IP_OTHER_DIS(BIT(lookup)) | + ANA_PORT_VCAP_S2_CFG_S2_OAM_DIS(BIT(lookup)); ocelot_rmw_gix(ocelot, val, - ANA_PORT_VCAP_S2_CFG_S2_SNAP_DIS_M | - ANA_PORT_VCAP_S2_CFG_S2_ARP_DIS_M | - ANA_PORT_VCAP_S2_CFG_S2_IP_TCPUDP_DIS_M | - ANA_PORT_VCAP_S2_CFG_S2_IP_OTHER_DIS_M | - ANA_PORT_VCAP_S2_CFG_S2_OAM_DIS_M, + ANA_PORT_VCAP_S2_CFG_S2_SNAP_DIS(BIT(lookup)) | + ANA_PORT_VCAP_S2_CFG_S2_ARP_DIS(BIT(lookup)) | + ANA_PORT_VCAP_S2_CFG_S2_IP_TCPUDP_DIS(BIT(lookup)) | + ANA_PORT_VCAP_S2_CFG_S2_IP_OTHER_DIS(BIT(lookup)) | + ANA_PORT_VCAP_S2_CFG_S2_OAM_DIS(BIT(lookup)), ANA_PORT_VCAP_S2_CFG, port); } @@ -825,35 +1075,43 @@ static bool ocelot_exclusive_mac_etype_filter_rules(struct ocelot *ocelot, struct ocelot_vcap_filter *filter) { - struct ocelot_vcap_block *block = &ocelot->block; + struct ocelot_vcap_block *block = &ocelot->block[filter->block_id]; struct ocelot_vcap_filter *tmp; unsigned long port; int i; + /* We only have the S2_IP_TCPUDP_DIS set of knobs for VCAP IS2 */ + if (filter->block_id != VCAP_IS2) + return true; + if (ocelot_vcap_is_problematic_mac_etype(filter)) { /* Search for any non-MAC_ETYPE rules on the port */ for (i = 0; i < block->count; i++) { - tmp = ocelot_vcap_block_find_filter(block, i); + tmp = ocelot_vcap_block_find_filter_by_index(block, i); if (tmp->ingress_port_mask & filter->ingress_port_mask && + tmp->lookup == filter->lookup && ocelot_vcap_is_problematic_non_mac_etype(tmp)) return false; } for_each_set_bit(port, &filter->ingress_port_mask, ocelot->num_phys_ports) - ocelot_match_all_as_mac_etype(ocelot, port, true); + ocelot_match_all_as_mac_etype(ocelot, port, + filter->lookup, true); } else if (ocelot_vcap_is_problematic_non_mac_etype(filter)) { /* Search for any MAC_ETYPE rules on the port */ for (i = 0; i < block->count; i++) { - tmp = ocelot_vcap_block_find_filter(block, i); + tmp = ocelot_vcap_block_find_filter_by_index(block, i); if (tmp->ingress_port_mask & filter->ingress_port_mask && + tmp->lookup == filter->lookup && ocelot_vcap_is_problematic_mac_etype(tmp)) return false; } for_each_set_bit(port, &filter->ingress_port_mask, ocelot->num_phys_ports) - ocelot_match_all_as_mac_etype(ocelot, port, false); + ocelot_match_all_as_mac_etype(ocelot, port, + filter->lookup, false); } return true; @@ -863,12 +1121,12 @@ int ocelot_vcap_filter_add(struct ocelot *ocelot, struct ocelot_vcap_filter *filter, struct netlink_ext_ack *extack) { - struct ocelot_vcap_block *block = &ocelot->block; + struct ocelot_vcap_block *block = &ocelot->block[filter->block_id]; int i, index; if (!ocelot_exclusive_mac_etype_filter_rules(ocelot, filter)) { NL_SET_ERR_MSG_MOD(extack, - "Cannot mix MAC_ETYPE with non-MAC_ETYPE rules"); + "Cannot mix MAC_ETYPE with non-MAC_ETYPE rules, use the other IS2 lookup"); return -EBUSY; } @@ -877,17 +1135,19 @@ int ocelot_vcap_filter_add(struct ocelot *ocelot, /* Get the index of the inserted filter */ index = ocelot_vcap_block_get_filter_index(block, filter); + if (index < 0) + return index; /* Move down the rules to make place for the new filter */ for (i = block->count - 1; i > index; i--) { struct ocelot_vcap_filter *tmp; - tmp = ocelot_vcap_block_find_filter(block, i); - is2_entry_set(ocelot, i, tmp); + tmp = ocelot_vcap_block_find_filter_by_index(block, i); + vcap_entry_set(ocelot, i, tmp); } /* Now insert the new filter */ - is2_entry_set(ocelot, index, filter); + vcap_entry_set(ocelot, index, filter); return 0; } @@ -901,9 +1161,10 @@ static void ocelot_vcap_block_remove_filter(struct ocelot *ocelot, list_for_each_safe(pos, q, &block->rules) { tmp = list_entry(pos, struct ocelot_vcap_filter, list); if (tmp->id == filter->id) { - if (tmp->action == OCELOT_VCAP_ACTION_POLICE) + if (tmp->block_id == VCAP_IS2 && + tmp->action.police_ena) ocelot_vcap_policer_del(ocelot, block, - tmp->pol_ix); + tmp->action.pol_ix); list_del(pos); kfree(tmp); @@ -916,7 +1177,7 @@ static void ocelot_vcap_block_remove_filter(struct ocelot *ocelot, int ocelot_vcap_filter_del(struct ocelot *ocelot, struct ocelot_vcap_filter *filter) { - struct ocelot_vcap_block *block = &ocelot->block; + struct ocelot_vcap_block *block = &ocelot->block[filter->block_id]; struct ocelot_vcap_filter del_filter; int i, index; @@ -924,6 +1185,8 @@ int ocelot_vcap_filter_del(struct ocelot *ocelot, /* Gets index of the filter */ index = ocelot_vcap_block_get_filter_index(block, filter); + if (index < 0) + return index; /* Delete filter */ ocelot_vcap_block_remove_filter(ocelot, block, filter); @@ -932,12 +1195,12 @@ int ocelot_vcap_filter_del(struct ocelot *ocelot, for (i = index; i < block->count; i++) { struct ocelot_vcap_filter *tmp; - tmp = ocelot_vcap_block_find_filter(block, i); - is2_entry_set(ocelot, i, tmp); + tmp = ocelot_vcap_block_find_filter_by_index(block, i); + vcap_entry_set(ocelot, i, tmp); } /* Now delete the last filter, because it is duplicated */ - is2_entry_set(ocelot, block->count, &del_filter); + vcap_entry_set(ocelot, block->count, &del_filter); return 0; } @@ -945,37 +1208,115 @@ int ocelot_vcap_filter_del(struct ocelot *ocelot, int ocelot_vcap_filter_stats_update(struct ocelot *ocelot, struct ocelot_vcap_filter *filter) { - struct ocelot_vcap_block *block = &ocelot->block; - struct ocelot_vcap_filter *tmp; + struct ocelot_vcap_block *block = &ocelot->block[filter->block_id]; + struct ocelot_vcap_filter tmp; int index; index = ocelot_vcap_block_get_filter_index(block, filter); - is2_entry_get(ocelot, filter, index); + if (index < 0) + return index; + + vcap_entry_get(ocelot, index, filter); /* After we get the result we need to clear the counters */ - tmp = ocelot_vcap_block_find_filter(block, index); - tmp->stats.pkts = 0; - is2_entry_set(ocelot, index, tmp); + tmp = *filter; + tmp.stats.pkts = 0; + vcap_entry_set(ocelot, index, &tmp); return 0; } -int ocelot_vcap_init(struct ocelot *ocelot) +static void ocelot_vcap_init_one(struct ocelot *ocelot, + const struct vcap_props *vcap) { - const struct vcap_props *vcap_is2 = &ocelot->vcap[VCAP_IS2]; - struct ocelot_vcap_block *block = &ocelot->block; struct vcap_data data; memset(&data, 0, sizeof(data)); - vcap_entry2cache(ocelot, &data); - ocelot_write(ocelot, vcap_is2->entry_count, S2_CORE_MV_CFG); - vcap_cmd(ocelot, 0, VCAP_CMD_INITIALIZE, VCAP_SEL_ENTRY); + vcap_entry2cache(ocelot, vcap, &data); + ocelot_target_write(ocelot, vcap->target, vcap->entry_count, + VCAP_CORE_MV_CFG); + vcap_cmd(ocelot, vcap, 0, VCAP_CMD_INITIALIZE, VCAP_SEL_ENTRY); - vcap_action2cache(ocelot, &data); - ocelot_write(ocelot, vcap_is2->action_count, S2_CORE_MV_CFG); - vcap_cmd(ocelot, 0, VCAP_CMD_INITIALIZE, + vcap_action2cache(ocelot, vcap, &data); + ocelot_target_write(ocelot, vcap->target, vcap->action_count, + VCAP_CORE_MV_CFG); + vcap_cmd(ocelot, vcap, 0, VCAP_CMD_INITIALIZE, VCAP_SEL_ACTION | VCAP_SEL_COUNTER); +} + +static void ocelot_vcap_detect_constants(struct ocelot *ocelot, + struct vcap_props *vcap) +{ + int counter_memory_width; + int num_default_actions; + int version; + + version = ocelot_target_read(ocelot, vcap->target, + VCAP_CONST_VCAP_VER); + /* Only version 0 VCAP supported for now */ + if (WARN_ON(version != 0)) + return; + + /* Width in bits of type-group field */ + vcap->tg_width = ocelot_target_read(ocelot, vcap->target, + VCAP_CONST_ENTRY_TG_WIDTH); + /* Number of subwords per TCAM row */ + vcap->sw_count = ocelot_target_read(ocelot, vcap->target, + VCAP_CONST_ENTRY_SWCNT); + /* Number of rows in TCAM. There can be this many full keys, or double + * this number half keys, or 4 times this number quarter keys. + */ + vcap->entry_count = ocelot_target_read(ocelot, vcap->target, + VCAP_CONST_ENTRY_CNT); + /* Assuming there are 4 subwords per TCAM row, their layout in the + * actual TCAM (not in the cache) would be: + * + * | SW 3 | TG 3 | SW 2 | TG 2 | SW 1 | TG 1 | SW 0 | TG 0 | + * + * (where SW=subword and TG=Type-Group). + * + * What VCAP_CONST_ENTRY_CNT is giving us is the width of one full TCAM + * row. But when software accesses the TCAM through the cache + * registers, the Type-Group values are written through another set of + * registers VCAP_TG_DAT, and therefore, it appears as though the 4 + * subwords are contiguous in the cache memory. + * Important mention: regardless of the number of key entries per row + * (and therefore of key size: 1 full key or 2 half keys or 4 quarter + * keys), software always has to configure 4 Type-Group values. For + * example, in the case of 1 full key, the driver needs to set all 4 + * Type-Group to be full key. + * + * For this reason, we need to fix up the value that the hardware is + * giving us. We don't actually care about the width of the entry in + * the TCAM. What we care about is the width of the entry in the cache + * registers, which is how we get to interact with it. And since the + * VCAP_ENTRY_DAT cache registers access only the subwords and not the + * Type-Groups, this means we need to subtract the width of the + * Type-Groups when packing and unpacking key entry data in a TCAM row. + */ + vcap->entry_width = ocelot_target_read(ocelot, vcap->target, + VCAP_CONST_ENTRY_WIDTH); + vcap->entry_width -= vcap->tg_width * vcap->sw_count; + num_default_actions = ocelot_target_read(ocelot, vcap->target, + VCAP_CONST_ACTION_DEF_CNT); + vcap->action_count = vcap->entry_count + num_default_actions; + vcap->action_width = ocelot_target_read(ocelot, vcap->target, + VCAP_CONST_ACTION_WIDTH); + /* The width of the counter memory, this is the complete width of all + * counter-fields associated with one full-word entry. There is one + * counter per entry sub-word (see CAP_CORE::ENTRY_SWCNT for number of + * subwords.) + */ + vcap->counter_words = vcap->sw_count; + counter_memory_width = ocelot_target_read(ocelot, vcap->target, + VCAP_CONST_CNT_WIDTH); + vcap->counter_width = counter_memory_width / vcap->counter_words; +} + +int ocelot_vcap_init(struct ocelot *ocelot) +{ + int i; /* Create a policer that will drop the frames for the cpu. * This policer will be used as action in the acl rules to drop @@ -992,9 +1333,18 @@ int ocelot_vcap_init(struct ocelot *ocelot) ocelot_write_gix(ocelot, 0x3fffff, ANA_POL_CIR_STATE, OCELOT_POLICER_DISCARD); - block->pol_lpr = OCELOT_POLICER_DISCARD - 1; + for (i = 0; i < OCELOT_NUM_VCAP_BLOCKS; i++) { + struct ocelot_vcap_block *block = &ocelot->block[i]; + struct vcap_props *vcap = &ocelot->vcap[i]; + + INIT_LIST_HEAD(&block->rules); + block->pol_lpr = OCELOT_POLICER_DISCARD - 1; + + ocelot_vcap_detect_constants(ocelot, vcap); + ocelot_vcap_init_one(ocelot, vcap); + } - INIT_LIST_HEAD(&ocelot->block.rules); + INIT_LIST_HEAD(&ocelot->dummy_rules); return 0; } diff --git a/drivers/net/ethernet/mscc/ocelot_vcap.h b/drivers/net/ethernet/mscc/ocelot_vcap.h index 0dfbfc011b2e..82fd10581a14 100644 --- a/drivers/net/ethernet/mscc/ocelot_vcap.h +++ b/drivers/net/ethernet/mscc/ocelot_vcap.h @@ -11,6 +11,8 @@ #include <net/sch_generic.h> #include <net/pkt_cls.h> +#define OCELOT_POLICER_DISCARD 0x17f + struct ocelot_ipv4 { u8 addr[4]; }; @@ -76,6 +78,11 @@ struct ocelot_vcap_udp_tcp { u16 mask; }; +struct ocelot_vcap_port { + u8 value; + u8 mask; +}; + enum ocelot_vcap_key_type { OCELOT_VCAP_KEY_ANY, OCELOT_VCAP_KEY_ETYPE, @@ -158,6 +165,7 @@ struct ocelot_vcap_key_ipv4 { struct ocelot_vcap_key_ipv6 { struct ocelot_vcap_u8 proto; /* IPv6 protocol */ struct ocelot_vcap_u128 sip; /* IPv6 source (byte 0-7 ignored) */ + struct ocelot_vcap_u128 dip; /* IPv6 destination (byte 0-7 ignored) */ enum ocelot_vcap_bit ttl; /* TTL zero */ struct ocelot_vcap_u8 ds; struct ocelot_vcap_u48 data; /* Not UDP/TCP: IP data */ @@ -174,10 +182,71 @@ struct ocelot_vcap_key_ipv6 { enum ocelot_vcap_bit seq_zero; /* TCP sequence number is zero */ }; -enum ocelot_vcap_action { - OCELOT_VCAP_ACTION_DROP, - OCELOT_VCAP_ACTION_TRAP, - OCELOT_VCAP_ACTION_POLICE, +enum ocelot_mask_mode { + OCELOT_MASK_MODE_NONE, + OCELOT_MASK_MODE_PERMIT_DENY, + OCELOT_MASK_MODE_POLICY, + OCELOT_MASK_MODE_REDIRECT, +}; + +enum ocelot_es0_tag { + OCELOT_NO_ES0_TAG, + OCELOT_ES0_TAG, + OCELOT_FORCE_PORT_TAG, + OCELOT_FORCE_UNTAG, +}; + +enum ocelot_tag_tpid_sel { + OCELOT_TAG_TPID_SEL_8021Q, + OCELOT_TAG_TPID_SEL_8021AD, +}; + +struct ocelot_vcap_action { + union { + /* VCAP ES0 */ + struct { + enum ocelot_es0_tag push_outer_tag; + enum ocelot_es0_tag push_inner_tag; + enum ocelot_tag_tpid_sel tag_a_tpid_sel; + int tag_a_vid_sel; + int tag_a_pcp_sel; + u16 vid_a_val; + u8 pcp_a_val; + u8 dei_a_val; + enum ocelot_tag_tpid_sel tag_b_tpid_sel; + int tag_b_vid_sel; + int tag_b_pcp_sel; + u16 vid_b_val; + u8 pcp_b_val; + u8 dei_b_val; + }; + + /* VCAP IS1 */ + struct { + bool vid_replace_ena; + u16 vid; + bool vlan_pop_cnt_ena; + int vlan_pop_cnt; + bool pcp_dei_ena; + u8 pcp; + u8 dei; + bool qos_ena; + u8 qos_val; + u8 pag_override_mask; + u8 pag_val; + }; + + /* VCAP IS2 */ + struct { + bool cpu_copy_ena; + u8 cpu_qu_num; + enum ocelot_mask_mode mask_mode; + unsigned long port_mask; + bool police_ena; + struct ocelot_policer pol; + u32 pol_ix; + }; + }; }; struct ocelot_vcap_stats { @@ -186,15 +255,30 @@ struct ocelot_vcap_stats { u64 used; }; +enum ocelot_vcap_filter_type { + OCELOT_VCAP_FILTER_DUMMY, + OCELOT_VCAP_FILTER_PAG, + OCELOT_VCAP_FILTER_OFFLOAD, +}; + struct ocelot_vcap_filter { struct list_head list; + enum ocelot_vcap_filter_type type; + int block_id; + int goto_target; + int lookup; + u8 pag; u16 prio; u32 id; - enum ocelot_vcap_action action; + struct ocelot_vcap_action action; struct ocelot_vcap_stats stats; + /* For VCAP IS1 and IS2 */ unsigned long ingress_port_mask; + /* For VCAP ES0 */ + struct ocelot_vcap_port ingress_port; + struct ocelot_vcap_port egress_port; enum ocelot_vcap_bit dmac_mc; enum ocelot_vcap_bit dmac_bc; @@ -210,8 +294,6 @@ struct ocelot_vcap_filter { struct ocelot_vcap_key_ipv4 ipv4; struct ocelot_vcap_key_ipv6 ipv6; } key; - struct ocelot_policer pol; - u32 pol_ix; }; int ocelot_vcap_filter_add(struct ocelot *ocelot, @@ -221,7 +303,10 @@ int ocelot_vcap_filter_del(struct ocelot *ocelot, struct ocelot_vcap_filter *rule); int ocelot_vcap_filter_stats_update(struct ocelot *ocelot, struct ocelot_vcap_filter *rule); +struct ocelot_vcap_filter * +ocelot_vcap_block_find_filter_by_id(struct ocelot_vcap_block *block, int id); +void ocelot_detect_vcap_constants(struct ocelot *ocelot); int ocelot_vcap_init(struct ocelot *ocelot); int ocelot_setup_tc_cls_flower(struct ocelot_port_private *priv, diff --git a/drivers/net/ethernet/mscc/ocelot_vsc7514.c b/drivers/net/ethernet/mscc/ocelot_vsc7514.c index d7aef2fb9848..f3e54589e6d6 100644 --- a/drivers/net/ethernet/mscc/ocelot_vsc7514.c +++ b/drivers/net/ethernet/mscc/ocelot_vsc7514.c @@ -19,10 +19,6 @@ #include "ocelot.h" #define IFH_EXTRACT_BITFIELD64(x, o, w) (((x) >> (o)) & GENMASK_ULL((w) - 1, 0)) -#define VSC7514_VCAP_IS2_CNT 64 -#define VSC7514_VCAP_IS2_ENTRY_WIDTH 376 -#define VSC7514_VCAP_IS2_ACTION_WIDTH 99 -#define VSC7514_VCAP_PORT_CNT 11 static const u32 ocelot_ana_regmap[] = { REG(ANA_ADVLEARN, 0x009000), @@ -241,14 +237,27 @@ static const u32 ocelot_sys_regmap[] = { REG(SYS_PTP_CFG, 0x0006c4), }; -static const u32 ocelot_s2_regmap[] = { - REG(S2_CORE_UPDATE_CTRL, 0x000000), - REG(S2_CORE_MV_CFG, 0x000004), - REG(S2_CACHE_ENTRY_DAT, 0x000008), - REG(S2_CACHE_MASK_DAT, 0x000108), - REG(S2_CACHE_ACTION_DAT, 0x000208), - REG(S2_CACHE_CNT_DAT, 0x000308), - REG(S2_CACHE_TG_DAT, 0x000388), +static const u32 ocelot_vcap_regmap[] = { + /* VCAP_CORE_CFG */ + REG(VCAP_CORE_UPDATE_CTRL, 0x000000), + REG(VCAP_CORE_MV_CFG, 0x000004), + /* VCAP_CORE_CACHE */ + REG(VCAP_CACHE_ENTRY_DAT, 0x000008), + REG(VCAP_CACHE_MASK_DAT, 0x000108), + REG(VCAP_CACHE_ACTION_DAT, 0x000208), + REG(VCAP_CACHE_CNT_DAT, 0x000308), + REG(VCAP_CACHE_TG_DAT, 0x000388), + /* VCAP_CONST */ + REG(VCAP_CONST_VCAP_VER, 0x000398), + REG(VCAP_CONST_ENTRY_WIDTH, 0x00039c), + REG(VCAP_CONST_ENTRY_CNT, 0x0003a0), + REG(VCAP_CONST_ENTRY_SWCNT, 0x0003a4), + REG(VCAP_CONST_ENTRY_TG_WIDTH, 0x0003a8), + REG(VCAP_CONST_ACTION_DEF_CNT, 0x0003ac), + REG(VCAP_CONST_ACTION_WIDTH, 0x0003b0), + REG(VCAP_CONST_CNT_WIDTH, 0x0003b4), + REG(VCAP_CONST_CORE_CNT, 0x0003b8), + REG(VCAP_CONST_IF_CNT, 0x0003bc), }; static const u32 ocelot_ptp_regmap[] = { @@ -311,7 +320,7 @@ static const u32 *ocelot_regmap[TARGET_MAX] = { [QSYS] = ocelot_qsys_regmap, [REW] = ocelot_rew_regmap, [SYS] = ocelot_sys_regmap, - [S2] = ocelot_s2_regmap, + [S2] = ocelot_vcap_regmap, [PTP] = ocelot_ptp_regmap, [DEV_GMII] = ocelot_dev_gmii_regmap, }; @@ -754,6 +763,115 @@ static u16 ocelot_wm_enc(u16 value) static const struct ocelot_ops ocelot_ops = { .reset = ocelot_reset, .wm_enc = ocelot_wm_enc, + .port_to_netdev = ocelot_port_to_netdev, + .netdev_to_port = ocelot_netdev_to_port, +}; + +static const struct vcap_field vsc7514_vcap_es0_keys[] = { + [VCAP_ES0_EGR_PORT] = { 0, 4}, + [VCAP_ES0_IGR_PORT] = { 4, 4}, + [VCAP_ES0_RSV] = { 8, 2}, + [VCAP_ES0_L2_MC] = { 10, 1}, + [VCAP_ES0_L2_BC] = { 11, 1}, + [VCAP_ES0_VID] = { 12, 12}, + [VCAP_ES0_DP] = { 24, 1}, + [VCAP_ES0_PCP] = { 25, 3}, +}; + +static const struct vcap_field vsc7514_vcap_es0_actions[] = { + [VCAP_ES0_ACT_PUSH_OUTER_TAG] = { 0, 2}, + [VCAP_ES0_ACT_PUSH_INNER_TAG] = { 2, 1}, + [VCAP_ES0_ACT_TAG_A_TPID_SEL] = { 3, 2}, + [VCAP_ES0_ACT_TAG_A_VID_SEL] = { 5, 1}, + [VCAP_ES0_ACT_TAG_A_PCP_SEL] = { 6, 2}, + [VCAP_ES0_ACT_TAG_A_DEI_SEL] = { 8, 2}, + [VCAP_ES0_ACT_TAG_B_TPID_SEL] = { 10, 2}, + [VCAP_ES0_ACT_TAG_B_VID_SEL] = { 12, 1}, + [VCAP_ES0_ACT_TAG_B_PCP_SEL] = { 13, 2}, + [VCAP_ES0_ACT_TAG_B_DEI_SEL] = { 15, 2}, + [VCAP_ES0_ACT_VID_A_VAL] = { 17, 12}, + [VCAP_ES0_ACT_PCP_A_VAL] = { 29, 3}, + [VCAP_ES0_ACT_DEI_A_VAL] = { 32, 1}, + [VCAP_ES0_ACT_VID_B_VAL] = { 33, 12}, + [VCAP_ES0_ACT_PCP_B_VAL] = { 45, 3}, + [VCAP_ES0_ACT_DEI_B_VAL] = { 48, 1}, + [VCAP_ES0_ACT_RSV] = { 49, 24}, + [VCAP_ES0_ACT_HIT_STICKY] = { 73, 1}, +}; + +static const struct vcap_field vsc7514_vcap_is1_keys[] = { + [VCAP_IS1_HK_TYPE] = { 0, 1}, + [VCAP_IS1_HK_LOOKUP] = { 1, 2}, + [VCAP_IS1_HK_IGR_PORT_MASK] = { 3, 12}, + [VCAP_IS1_HK_RSV] = { 15, 9}, + [VCAP_IS1_HK_OAM_Y1731] = { 24, 1}, + [VCAP_IS1_HK_L2_MC] = { 25, 1}, + [VCAP_IS1_HK_L2_BC] = { 26, 1}, + [VCAP_IS1_HK_IP_MC] = { 27, 1}, + [VCAP_IS1_HK_VLAN_TAGGED] = { 28, 1}, + [VCAP_IS1_HK_VLAN_DBL_TAGGED] = { 29, 1}, + [VCAP_IS1_HK_TPID] = { 30, 1}, + [VCAP_IS1_HK_VID] = { 31, 12}, + [VCAP_IS1_HK_DEI] = { 43, 1}, + [VCAP_IS1_HK_PCP] = { 44, 3}, + /* Specific Fields for IS1 Half Key S1_NORMAL */ + [VCAP_IS1_HK_L2_SMAC] = { 47, 48}, + [VCAP_IS1_HK_ETYPE_LEN] = { 95, 1}, + [VCAP_IS1_HK_ETYPE] = { 96, 16}, + [VCAP_IS1_HK_IP_SNAP] = {112, 1}, + [VCAP_IS1_HK_IP4] = {113, 1}, + /* Layer-3 Information */ + [VCAP_IS1_HK_L3_FRAGMENT] = {114, 1}, + [VCAP_IS1_HK_L3_FRAG_OFS_GT0] = {115, 1}, + [VCAP_IS1_HK_L3_OPTIONS] = {116, 1}, + [VCAP_IS1_HK_L3_DSCP] = {117, 6}, + [VCAP_IS1_HK_L3_IP4_SIP] = {123, 32}, + /* Layer-4 Information */ + [VCAP_IS1_HK_TCP_UDP] = {155, 1}, + [VCAP_IS1_HK_TCP] = {156, 1}, + [VCAP_IS1_HK_L4_SPORT] = {157, 16}, + [VCAP_IS1_HK_L4_RNG] = {173, 8}, + /* Specific Fields for IS1 Half Key S1_5TUPLE_IP4 */ + [VCAP_IS1_HK_IP4_INNER_TPID] = { 47, 1}, + [VCAP_IS1_HK_IP4_INNER_VID] = { 48, 12}, + [VCAP_IS1_HK_IP4_INNER_DEI] = { 60, 1}, + [VCAP_IS1_HK_IP4_INNER_PCP] = { 61, 3}, + [VCAP_IS1_HK_IP4_IP4] = { 64, 1}, + [VCAP_IS1_HK_IP4_L3_FRAGMENT] = { 65, 1}, + [VCAP_IS1_HK_IP4_L3_FRAG_OFS_GT0] = { 66, 1}, + [VCAP_IS1_HK_IP4_L3_OPTIONS] = { 67, 1}, + [VCAP_IS1_HK_IP4_L3_DSCP] = { 68, 6}, + [VCAP_IS1_HK_IP4_L3_IP4_DIP] = { 74, 32}, + [VCAP_IS1_HK_IP4_L3_IP4_SIP] = {106, 32}, + [VCAP_IS1_HK_IP4_L3_PROTO] = {138, 8}, + [VCAP_IS1_HK_IP4_TCP_UDP] = {146, 1}, + [VCAP_IS1_HK_IP4_TCP] = {147, 1}, + [VCAP_IS1_HK_IP4_L4_RNG] = {148, 8}, + [VCAP_IS1_HK_IP4_IP_PAYLOAD_S1_5TUPLE] = {156, 32}, +}; + +static const struct vcap_field vsc7514_vcap_is1_actions[] = { + [VCAP_IS1_ACT_DSCP_ENA] = { 0, 1}, + [VCAP_IS1_ACT_DSCP_VAL] = { 1, 6}, + [VCAP_IS1_ACT_QOS_ENA] = { 7, 1}, + [VCAP_IS1_ACT_QOS_VAL] = { 8, 3}, + [VCAP_IS1_ACT_DP_ENA] = { 11, 1}, + [VCAP_IS1_ACT_DP_VAL] = { 12, 1}, + [VCAP_IS1_ACT_PAG_OVERRIDE_MASK] = { 13, 8}, + [VCAP_IS1_ACT_PAG_VAL] = { 21, 8}, + [VCAP_IS1_ACT_RSV] = { 29, 9}, + /* The fields below are incorrectly shifted by 2 in the manual */ + [VCAP_IS1_ACT_VID_REPLACE_ENA] = { 38, 1}, + [VCAP_IS1_ACT_VID_ADD_VAL] = { 39, 12}, + [VCAP_IS1_ACT_FID_SEL] = { 51, 2}, + [VCAP_IS1_ACT_FID_VAL] = { 53, 13}, + [VCAP_IS1_ACT_PCP_DEI_ENA] = { 66, 1}, + [VCAP_IS1_ACT_PCP_VAL] = { 67, 3}, + [VCAP_IS1_ACT_DEI_VAL] = { 70, 1}, + [VCAP_IS1_ACT_VLAN_POP_CNT_ENA] = { 71, 1}, + [VCAP_IS1_ACT_VLAN_POP_CNT] = { 72, 2}, + [VCAP_IS1_ACT_CUSTOM_ACE_TYPE_ENA] = { 74, 4}, + [VCAP_IS1_ACT_HIT_STICKY] = { 78, 1}, }; static const struct vcap_field vsc7514_vcap_is2_keys[] = { @@ -854,15 +972,32 @@ static const struct vcap_field vsc7514_vcap_is2_actions[] = { [VCAP_IS2_ACT_HIT_CNT] = { 49, 32}, }; -static const struct vcap_props vsc7514_vcap_props[] = { +static struct vcap_props vsc7514_vcap_props[] = { + [VCAP_ES0] = { + .action_type_width = 0, + .action_table = { + [ES0_ACTION_TYPE_NORMAL] = { + .width = 73, /* HIT_STICKY not included */ + .count = 1, + }, + }, + .target = S0, + .keys = vsc7514_vcap_es0_keys, + .actions = vsc7514_vcap_es0_actions, + }, + [VCAP_IS1] = { + .action_type_width = 0, + .action_table = { + [IS1_ACTION_TYPE_NORMAL] = { + .width = 78, /* HIT_STICKY not included */ + .count = 4, + }, + }, + .target = S1, + .keys = vsc7514_vcap_is1_keys, + .actions = vsc7514_vcap_is1_actions, + }, [VCAP_IS2] = { - .tg_width = 2, - .sw_count = 4, - .entry_count = VSC7514_VCAP_IS2_CNT, - .entry_width = VSC7514_VCAP_IS2_ENTRY_WIDTH, - .action_count = VSC7514_VCAP_IS2_CNT + - VSC7514_VCAP_PORT_CNT + 2, - .action_width = 99, .action_type_width = 1, .action_table = { [IS2_ACTION_TYPE_NORMAL] = { @@ -874,8 +1009,9 @@ static const struct vcap_props vsc7514_vcap_props[] = { .count = 4 }, }, - .counter_words = 4, - .counter_width = 32, + .target = S2, + .keys = vsc7514_vcap_is2_keys, + .actions = vsc7514_vcap_is2_actions, }, }; @@ -1037,6 +1173,8 @@ static int mscc_ocelot_probe(struct platform_device *pdev) { QSYS, "qsys" }, { ANA, "ana" }, { QS, "qs" }, + { S0, "s0" }, + { S1, "s1" }, { S2, "s2" }, { PTP, "ptp", 1 }, }; @@ -1113,8 +1251,6 @@ static int mscc_ocelot_probe(struct platform_device *pdev) ocelot->num_phys_ports = of_get_child_count(ports); - ocelot->vcap_is2_keys = vsc7514_vcap_is2_keys; - ocelot->vcap_is2_actions = vsc7514_vcap_is2_actions; ocelot->vcap = vsc7514_vcap_props; ocelot->inj_prefix = OCELOT_TAG_PREFIX_NONE; ocelot->xtr_prefix = OCELOT_TAG_PREFIX_NONE; diff --git a/drivers/net/ethernet/natsemi/sonic.c b/drivers/net/ethernet/natsemi/sonic.c index dd3605aa5f23..d17d1b4f2585 100644 --- a/drivers/net/ethernet/natsemi/sonic.c +++ b/drivers/net/ethernet/natsemi/sonic.c @@ -143,7 +143,7 @@ static int sonic_open(struct net_device *dev) /* * Initialize the SONIC */ - sonic_init(dev); + sonic_init(dev, true); netif_start_queue(dev); @@ -153,7 +153,7 @@ static int sonic_open(struct net_device *dev) } /* Wait for the SONIC to become idle. */ -static void sonic_quiesce(struct net_device *dev, u16 mask) +static void sonic_quiesce(struct net_device *dev, u16 mask, bool may_sleep) { struct sonic_local * __maybe_unused lp = netdev_priv(dev); int i; @@ -163,7 +163,7 @@ static void sonic_quiesce(struct net_device *dev, u16 mask) bits = SONIC_READ(SONIC_CMD) & mask; if (!bits) return; - if (irqs_disabled() || in_interrupt()) + if (!may_sleep) udelay(20); else usleep_range(100, 200); @@ -187,7 +187,7 @@ static int sonic_close(struct net_device *dev) * stop the SONIC, disable interrupts */ SONIC_WRITE(SONIC_CMD, SONIC_CR_RXDIS); - sonic_quiesce(dev, SONIC_CR_ALL); + sonic_quiesce(dev, SONIC_CR_ALL, true); SONIC_WRITE(SONIC_IMR, 0); SONIC_WRITE(SONIC_ISR, 0x7fff); @@ -229,7 +229,7 @@ static void sonic_tx_timeout(struct net_device *dev, unsigned int txqueue) * disable all interrupts before releasing DMA buffers */ SONIC_WRITE(SONIC_CMD, SONIC_CR_RXDIS); - sonic_quiesce(dev, SONIC_CR_ALL); + sonic_quiesce(dev, SONIC_CR_ALL, false); SONIC_WRITE(SONIC_IMR, 0); SONIC_WRITE(SONIC_ISR, 0x7fff); @@ -246,7 +246,7 @@ static void sonic_tx_timeout(struct net_device *dev, unsigned int txqueue) } } /* Try to restart the adaptor. */ - sonic_init(dev); + sonic_init(dev, false); lp->stats.tx_errors++; netif_trans_update(dev); /* prevent tx timeout */ netif_wake_queue(dev); @@ -692,9 +692,9 @@ static void sonic_multicast_list(struct net_device *dev) /* LCAM and TXP commands can't be used simultaneously */ spin_lock_irqsave(&lp->lock, flags); - sonic_quiesce(dev, SONIC_CR_TXP); + sonic_quiesce(dev, SONIC_CR_TXP, false); SONIC_WRITE(SONIC_CMD, SONIC_CR_LCAM); - sonic_quiesce(dev, SONIC_CR_LCAM); + sonic_quiesce(dev, SONIC_CR_LCAM, false); spin_unlock_irqrestore(&lp->lock, flags); } } @@ -708,7 +708,7 @@ static void sonic_multicast_list(struct net_device *dev) /* * Initialize the SONIC ethernet controller. */ -static int sonic_init(struct net_device *dev) +static int sonic_init(struct net_device *dev, bool may_sleep) { struct sonic_local *lp = netdev_priv(dev); int i; @@ -730,7 +730,7 @@ static int sonic_init(struct net_device *dev) */ SONIC_WRITE(SONIC_CMD, 0); SONIC_WRITE(SONIC_CMD, SONIC_CR_RXDIS | SONIC_CR_STP); - sonic_quiesce(dev, SONIC_CR_ALL); + sonic_quiesce(dev, SONIC_CR_ALL, may_sleep); /* * initialize the receive resource area @@ -759,7 +759,7 @@ static int sonic_init(struct net_device *dev) netif_dbg(lp, ifup, dev, "%s: issuing RRRA command\n", __func__); SONIC_WRITE(SONIC_CMD, SONIC_CR_RRRA); - sonic_quiesce(dev, SONIC_CR_RRRA); + sonic_quiesce(dev, SONIC_CR_RRRA, may_sleep); /* * Initialize the receive descriptors so that they @@ -834,7 +834,7 @@ static int sonic_init(struct net_device *dev) * load the CAM */ SONIC_WRITE(SONIC_CMD, SONIC_CR_LCAM); - sonic_quiesce(dev, SONIC_CR_LCAM); + sonic_quiesce(dev, SONIC_CR_LCAM, may_sleep); /* * enable receiver, disable loopback diff --git a/drivers/net/ethernet/natsemi/sonic.h b/drivers/net/ethernet/natsemi/sonic.h index 3cbb62c860c8..a5b803eb8c8a 100644 --- a/drivers/net/ethernet/natsemi/sonic.h +++ b/drivers/net/ethernet/natsemi/sonic.h @@ -338,7 +338,7 @@ static void sonic_rx(struct net_device *dev); static int sonic_close(struct net_device *dev); static struct net_device_stats *sonic_get_stats(struct net_device *dev); static void sonic_multicast_list(struct net_device *dev); -static int sonic_init(struct net_device *dev); +static int sonic_init(struct net_device *dev, bool may_sleep); static void sonic_tx_timeout(struct net_device *dev, unsigned int txqueue); static void sonic_msg_init(struct net_device *dev); static int sonic_alloc_descriptors(struct net_device *dev); diff --git a/drivers/net/ethernet/neterion/vxge/vxge-config.c b/drivers/net/ethernet/neterion/vxge/vxge-config.c index 6aba267036dd..f5d48d7c4ce2 100644 --- a/drivers/net/ethernet/neterion/vxge/vxge-config.c +++ b/drivers/net/ethernet/neterion/vxge/vxge-config.c @@ -2306,16 +2306,9 @@ exit: static inline void vxge_os_dma_malloc_async(struct pci_dev *pdev, void *devh, unsigned long size) { - gfp_t flags; void *vaddr; - if (in_interrupt()) - flags = GFP_ATOMIC | GFP_DMA; - else - flags = GFP_KERNEL | GFP_DMA; - - vaddr = kmalloc((size), flags); - + vaddr = kmalloc(size, GFP_KERNEL | GFP_DMA); vxge_hw_blockpool_block_add(devh, vaddr, size, pdev, pdev); } diff --git a/drivers/net/ethernet/neterion/vxge/vxge-config.h b/drivers/net/ethernet/neterion/vxge/vxge-config.h index 373165119850..0cd0750484ae 100644 --- a/drivers/net/ethernet/neterion/vxge/vxge-config.h +++ b/drivers/net/ethernet/neterion/vxge/vxge-config.h @@ -1899,18 +1899,13 @@ static inline void *vxge_os_dma_malloc(struct pci_dev *pdev, struct pci_dev **p_dmah, struct pci_dev **p_dma_acch) { - gfp_t flags; void *vaddr; unsigned long misaligned = 0; int realloc_flag = 0; *p_dma_acch = *p_dmah = NULL; - if (in_interrupt()) - flags = GFP_ATOMIC | GFP_DMA; - else - flags = GFP_KERNEL | GFP_DMA; realloc: - vaddr = kmalloc((size), flags); + vaddr = kmalloc(size, GFP_KERNEL | GFP_DMA); if (vaddr == NULL) return vaddr; misaligned = (unsigned long)VXGE_ALIGN((unsigned long)vaddr, diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c index d1d6fb6669e5..2749ce009ebc 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c @@ -350,7 +350,7 @@ err_out_port_reset: err_out_reset: ionic_reset(ionic); err_out_teardown: - ionic_dev_teardown(ionic); + del_timer_sync(&ionic->watchdog_timer); pci_clear_master(pdev); /* Don't fail the probe for these errors, keep * the hw interface around for inspection @@ -378,6 +378,8 @@ static void ionic_remove(struct pci_dev *pdev) if (!ionic) return; + del_timer_sync(&ionic->watchdog_timer); + if (ionic->lif) { ionic_devlink_unregister(ionic); ionic_lif_unregister(ionic->lif); @@ -389,7 +391,6 @@ static void ionic_remove(struct pci_dev *pdev) ionic_port_reset(ionic); ionic_reset(ionic); - ionic_dev_teardown(ionic); pci_clear_master(pdev); ionic_unmap_bars(ionic); pci_release_regions(pdev); diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.c b/drivers/net/ethernet/pensando/ionic/ionic_dev.c index 6068f51a11d9..545c99b15df8 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_dev.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.c @@ -19,10 +19,13 @@ static void ionic_watchdog_cb(struct timer_list *t) mod_timer(&ionic->watchdog_timer, round_jiffies(jiffies + ionic->watchdog_period)); + if (!ionic->lif) + return; + hb = ionic_heartbeat_check(ionic); - if (hb >= 0 && ionic->lif) - ionic_link_status_check_request(ionic->lif); + if (hb >= 0) + ionic_link_status_check_request(ionic->lif, false); } void ionic_init_devinfo(struct ionic *ionic) @@ -98,11 +101,6 @@ int ionic_dev_setup(struct ionic *ionic) return 0; } -void ionic_dev_teardown(struct ionic *ionic) -{ - del_timer_sync(&ionic->watchdog_timer); -} - /* Devcmd Interface */ int ionic_heartbeat_check(struct ionic *ionic) { diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h index 8842dc4a716f..c109cd5a0471 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h @@ -283,7 +283,6 @@ static inline bool ionic_q_has_space(struct ionic_queue *q, unsigned int want) void ionic_init_devinfo(struct ionic *ionic); int ionic_dev_setup(struct ionic *ionic); -void ionic_dev_teardown(struct ionic *ionic); void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd); u8 ionic_dev_cmd_status(struct ionic_dev *idev); diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index eb6fe37c0df6..1b4d5eb9bbc9 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -151,7 +151,7 @@ static void ionic_link_status_check(struct ionic_lif *lif) clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state); } -void ionic_link_status_check_request(struct ionic_lif *lif) +void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep) { struct ionic_deferred_work *work; @@ -159,7 +159,7 @@ void ionic_link_status_check_request(struct ionic_lif *lif) if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state)) return; - if (in_interrupt()) { + if (!can_sleep) { work = kzalloc(sizeof(*work), GFP_ATOMIC); if (!work) return; @@ -798,7 +798,7 @@ static bool ionic_notifyq_service(struct ionic_cq *cq, switch (le16_to_cpu(comp->event.ecode)) { case IONIC_EVENT_LINK_CHANGE: - ionic_link_status_check_request(lif); + ionic_link_status_check_request(lif, false); break; case IONIC_EVENT_RESET: work = kzalloc(sizeof(*work), GFP_ATOMIC); @@ -981,7 +981,8 @@ static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr) return 0; } -static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add) +static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add, + bool can_sleep) { struct ionic *ionic = lif->ionic; struct ionic_deferred_work *work; @@ -1010,7 +1011,7 @@ static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add) lif->nucast--; } - if (in_interrupt()) { + if (!can_sleep) { work = kzalloc(sizeof(*work), GFP_ATOMIC); if (!work) { netdev_err(lif->netdev, "%s OOM\n", __func__); @@ -1036,12 +1037,22 @@ static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add) static int ionic_addr_add(struct net_device *netdev, const u8 *addr) { - return ionic_lif_addr(netdev_priv(netdev), addr, true); + return ionic_lif_addr(netdev_priv(netdev), addr, true, true); +} + +static int ionic_ndo_addr_add(struct net_device *netdev, const u8 *addr) +{ + return ionic_lif_addr(netdev_priv(netdev), addr, true, false); } static int ionic_addr_del(struct net_device *netdev, const u8 *addr) { - return ionic_lif_addr(netdev_priv(netdev), addr, false); + return ionic_lif_addr(netdev_priv(netdev), addr, false, true); +} + +static int ionic_ndo_addr_del(struct net_device *netdev, const u8 *addr) +{ + return ionic_lif_addr(netdev_priv(netdev), addr, false, false); } static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode) @@ -1081,11 +1092,12 @@ static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode) lif->rx_mode = rx_mode; } -static void _ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode) +static void _ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode, + bool from_ndo) { struct ionic_deferred_work *work; - if (in_interrupt()) { + if (from_ndo) { work = kzalloc(sizeof(*work), GFP_ATOMIC); if (!work) { netdev_err(lif->netdev, "%s OOM\n", __func__); @@ -1100,7 +1112,16 @@ static void _ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode) } } -static void ionic_set_rx_mode(struct net_device *netdev) +static void ionic_dev_uc_sync(struct net_device *netdev, bool from_ndo) +{ + if (from_ndo) + __dev_uc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del); + else + __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del); + +} + +static void ionic_set_rx_mode(struct net_device *netdev, bool from_ndo) { struct ionic_lif *lif = netdev_priv(netdev); struct ionic_identity *ident; @@ -1122,7 +1143,7 @@ static void ionic_set_rx_mode(struct net_device *netdev) * we remove our overflow flag and check the netdev flags * to see if we can disable NIC PROMISC */ - __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del); + ionic_dev_uc_sync(netdev, from_ndo); nfilters = le32_to_cpu(ident->lif.eth.max_ucast_filters); if (netdev_uc_count(netdev) + 1 > nfilters) { rx_mode |= IONIC_RX_MODE_F_PROMISC; @@ -1134,7 +1155,7 @@ static void ionic_set_rx_mode(struct net_device *netdev) } /* same for multicast */ - __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del); + ionic_dev_uc_sync(netdev, from_ndo); nfilters = le32_to_cpu(ident->lif.eth.max_mcast_filters); if (netdev_mc_count(netdev) > nfilters) { rx_mode |= IONIC_RX_MODE_F_ALLMULTI; @@ -1146,7 +1167,12 @@ static void ionic_set_rx_mode(struct net_device *netdev) } if (lif->rx_mode != rx_mode) - _ionic_lif_rx_mode(lif, rx_mode); + _ionic_lif_rx_mode(lif, rx_mode, from_ndo); +} + +static void ionic_ndo_set_rx_mode(struct net_device *netdev) +{ + ionic_set_rx_mode(netdev, true); } static __le64 ionic_netdev_features_to_nic(netdev_features_t features) @@ -1391,7 +1417,7 @@ static int ionic_start_queues_reconfig(struct ionic_lif *lif) */ err = ionic_txrx_init(lif); mutex_unlock(&lif->queue_lock); - ionic_link_status_check_request(lif); + ionic_link_status_check_request(lif, true); netif_device_attach(lif->netdev); return err; @@ -1720,7 +1746,7 @@ static int ionic_txrx_init(struct ionic_lif *lif) if (lif->netdev->features & NETIF_F_RXHASH) ionic_lif_rss_init(lif); - ionic_set_rx_mode(lif->netdev); + ionic_set_rx_mode(lif->netdev, false); return 0; @@ -2093,7 +2119,7 @@ static const struct net_device_ops ionic_netdev_ops = { .ndo_stop = ionic_stop, .ndo_start_xmit = ionic_start_xmit, .ndo_get_stats64 = ionic_get_stats64, - .ndo_set_rx_mode = ionic_set_rx_mode, + .ndo_set_rx_mode = ionic_ndo_set_rx_mode, .ndo_set_features = ionic_set_features, .ndo_set_mac_address = ionic_set_mac_address, .ndo_validate_addr = eth_validate_addr, @@ -2521,7 +2547,7 @@ static void ionic_lif_handle_fw_up(struct ionic_lif *lif) } clear_bit(IONIC_LIF_F_FW_RESET, lif->state); - ionic_link_status_check_request(lif); + ionic_link_status_check_request(lif, true); netif_device_attach(lif->netdev); dev_info(ionic->dev, "FW Up: LIFs restarted\n"); @@ -2713,7 +2739,7 @@ static int ionic_station_set(struct ionic_lif *lif) */ if (!ether_addr_equal(ctx.comp.lif_getattr.mac, netdev->dev_addr)) - ionic_lif_addr(lif, netdev->dev_addr, true); + ionic_lif_addr(lif, netdev->dev_addr, true, true); } else { /* Update the netdev mac with the device's mac */ memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len); @@ -2730,7 +2756,7 @@ static int ionic_station_set(struct ionic_lif *lif) netdev_dbg(lif->netdev, "adding station MAC addr %pM\n", netdev->dev_addr); - ionic_lif_addr(lif, netdev->dev_addr, true); + ionic_lif_addr(lif, netdev->dev_addr, true, true); return 0; } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h index c65a5e6c26f4..0224dfd24b8a 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h @@ -245,7 +245,7 @@ static inline u32 ionic_coal_usec_to_hw(struct ionic *ionic, u32 usecs) typedef void (*ionic_reset_cb)(struct ionic_lif *lif, void *arg); -void ionic_link_status_check_request(struct ionic_lif *lif); +void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep); void ionic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *ns); void ionic_lif_deferred_enqueue(struct ionic_deferred *def, diff --git a/drivers/net/ethernet/pensando/ionic/ionic_main.c b/drivers/net/ethernet/pensando/ionic/ionic_main.c index e339216949a6..c7a67c5cda42 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_main.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_main.c @@ -255,8 +255,6 @@ static int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) struct ionic_queue *q; int err = 0; - WARN_ON(in_interrupt()); - if (!lif->adminqcq) return -EIO; @@ -328,8 +326,6 @@ int ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_seconds) int done; int err; - WARN_ON(in_interrupt()); - /* Wait for dev cmd to complete, retrying if we get EAGAIN, * but don't wait any longer than max_seconds. */ diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.c b/drivers/net/ethernet/qlogic/qed/qed_ll2.c index 0452b728c527..49783f365079 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_ll2.c +++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.c @@ -1185,7 +1185,7 @@ static int qed_ll2_acquire_connection_tx(struct qed_hwfn *p_hwfn, .elem_size = sizeof(struct core_tx_bd), }; struct qed_ll2_tx_packet *p_descq; - u32 desc_size; + size_t desc_size; u32 capacity; int rc = 0; @@ -1198,10 +1198,9 @@ static int qed_ll2_acquire_connection_tx(struct qed_hwfn *p_hwfn, goto out; capacity = qed_chain_get_capacity(&p_ll2_info->tx_queue.txq_chain); - /* First element is part of the packet, rest are flexibly added */ - desc_size = (sizeof(*p_descq) + - (p_ll2_info->input.tx_max_bds_per_packet - 1) * - sizeof(p_descq->bds_set)); + /* All bds_set elements are flexibily added. */ + desc_size = struct_size(p_descq, bds_set, + p_ll2_info->input.tx_max_bds_per_packet); p_descq = kcalloc(capacity, desc_size, GFP_KERNEL); if (!p_descq) { @@ -1524,7 +1523,7 @@ int qed_ll2_establish_connection(void *cxt, u8 connection_handle) struct qed_ptt *p_ptt; int rc = -EINVAL; u32 i, capacity; - u32 desc_size; + size_t desc_size; u8 qid; p_ptt = qed_ptt_acquire(p_hwfn); @@ -1558,10 +1557,9 @@ int qed_ll2_establish_connection(void *cxt, u8 connection_handle) INIT_LIST_HEAD(&p_tx->sending_descq); spin_lock_init(&p_tx->lock); capacity = qed_chain_get_capacity(&p_tx->txq_chain); - /* First element is part of the packet, rest are flexibly added */ - desc_size = (sizeof(*p_pkt) + - (p_ll2_conn->input.tx_max_bds_per_packet - 1) * - sizeof(p_pkt->bds_set)); + /* All bds_set elements are flexibily added. */ + desc_size = struct_size(p_pkt, bds_set, + p_ll2_conn->input.tx_max_bds_per_packet); for (i = 0; i < capacity; i++) { p_pkt = p_tx->descq_mem + desc_size * i; diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.h b/drivers/net/ethernet/qlogic/qed/qed_ll2.h index 500d0c4f8077..df88d00053a2 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_ll2.h +++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.h @@ -56,7 +56,7 @@ struct qed_ll2_tx_packet { struct core_tx_bd *txq_bd; dma_addr_t tx_frag; u16 frag_len; - } bds_set[1]; + } bds_set[]; }; struct qed_ll2_rx_queue { @@ -86,9 +86,6 @@ struct qed_ll2_tx_queue { struct list_head active_descq; struct list_head free_descq; struct list_head sending_descq; - void *descq_mem; /* memory for variable sized qed_ll2_tx_packet*/ - struct qed_ll2_tx_packet *cur_send_packet; - struct qed_ll2_tx_packet cur_completing_packet; u16 cur_completing_bd_idx; void __iomem *doorbell_addr; struct core_db_data db_msg; @@ -96,6 +93,9 @@ struct qed_ll2_tx_queue { u16 cur_send_frag_num; u16 cur_completing_frag_num; bool b_completing_packet; + void *descq_mem; /* memory for variable sized qed_ll2_tx_packet*/ + struct qed_ll2_tx_packet *cur_send_packet; + struct qed_ll2_tx_packet cur_completing_packet; }; struct qed_ll2_info { diff --git a/drivers/net/ethernet/renesas/ravb.h b/drivers/net/ethernet/renesas/ravb.h index 9f88b5db4f89..7453b17a37a2 100644 --- a/drivers/net/ethernet/renesas/ravb.h +++ b/drivers/net/ethernet/renesas/ravb.h @@ -1036,7 +1036,10 @@ struct ravb_private { unsigned no_avb_link:1; unsigned avb_link_active_low:1; unsigned wol_enabled:1; - int num_tx_desc; /* TX descriptors per packet */ + unsigned rxcidm:1; /* RX Clock Internal Delay Mode */ + unsigned txcidm:1; /* TX Clock Internal Delay Mode */ + unsigned rgmii_override:1; /* Deprecated rgmii-*id behavior */ + int num_tx_desc; /* TX descriptors per packet */ }; static inline u32 ravb_read(struct net_device *ndev, enum ravb_reg reg) diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index f684296df871..5082c16bf9c0 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -1034,11 +1034,8 @@ static int ravb_phy_init(struct net_device *ndev) pn = of_node_get(np); } - iface = priv->phy_interface; - if (priv->chip_id != RCAR_GEN2 && phy_interface_mode_is_rgmii(iface)) { - /* ravb_set_delay_mode() takes care of internal delay mode */ - iface = PHY_INTERFACE_MODE_RGMII; - } + iface = priv->rgmii_override ? PHY_INTERFACE_MODE_RGMII + : priv->phy_interface; phydev = of_phy_connect(ndev, pn, ravb_adjust_link, 0, iface); of_node_put(pn); if (!phydev) { @@ -1999,23 +1996,53 @@ static const struct soc_device_attribute ravb_delay_mode_quirk_match[] = { }; /* Set tx and rx clock internal delay modes */ -static void ravb_set_delay_mode(struct net_device *ndev) +static void ravb_parse_delay_mode(struct device_node *np, struct net_device *ndev) { struct ravb_private *priv = netdev_priv(ndev); - int set = 0; + bool explicit_delay = false; + u32 delay; + + if (!of_property_read_u32(np, "rx-internal-delay-ps", &delay)) { + /* Valid values are 0 and 1800, according to DT bindings */ + priv->rxcidm = !!delay; + explicit_delay = true; + } + if (!of_property_read_u32(np, "tx-internal-delay-ps", &delay)) { + /* Valid values are 0 and 2000, according to DT bindings */ + priv->txcidm = !!delay; + explicit_delay = true; + } + if (explicit_delay) + return; + + /* Fall back to legacy rgmii-*id behavior */ if (priv->phy_interface == PHY_INTERFACE_MODE_RGMII_ID || - priv->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID) - set |= APSR_DM_RDM; + priv->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID) { + priv->rxcidm = 1; + priv->rgmii_override = 1; + } if (priv->phy_interface == PHY_INTERFACE_MODE_RGMII_ID || priv->phy_interface == PHY_INTERFACE_MODE_RGMII_TXID) { if (!WARN(soc_device_match(ravb_delay_mode_quirk_match), "phy-mode %s requires TX clock internal delay mode which is not supported by this hardware revision. Please update device tree", - phy_modes(priv->phy_interface))) - set |= APSR_DM_TDM; + phy_modes(priv->phy_interface))) { + priv->txcidm = 1; + priv->rgmii_override = 1; + } } +} + +static void ravb_set_delay_mode(struct net_device *ndev) +{ + struct ravb_private *priv = netdev_priv(ndev); + u32 set = 0; + if (priv->rxcidm) + set |= APSR_DM_RDM; + if (priv->txcidm) + set |= APSR_DM_TDM; ravb_modify(ndev, APSR, APSR_DM, set); } @@ -2148,8 +2175,10 @@ static int ravb_probe(struct platform_device *pdev) /* Request GTI loading */ ravb_modify(ndev, GCCR, GCCR_LTI, GCCR_LTI); - if (priv->chip_id != RCAR_GEN2) + if (priv->chip_id != RCAR_GEN2) { + ravb_parse_delay_mode(np, ndev); ravb_set_delay_mode(ndev); + } /* Allocate descriptor base address table */ priv->desc_bat_size = sizeof(struct ravb_desc) * DBAT_ENTRY_NUM; diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c index c9df2e96ebe4..da6886dcac37 100644 --- a/drivers/net/ethernet/sfc/ef10.c +++ b/drivers/net/ethernet/sfc/ef10.c @@ -1871,18 +1871,9 @@ static int efx_ef10_try_update_nic_stats_vf(struct efx_nic *efx) spin_unlock_bh(&efx->stats_lock); - if (in_interrupt()) { - /* If in atomic context, cannot update stats. Just update the - * software stats and return so the caller can continue. - */ - spin_lock_bh(&efx->stats_lock); - efx_update_sw_stats(efx, stats); - return 0; - } - efx_ef10_get_stat_mask(efx, mask); - rc = efx_nic_alloc_buffer(efx, &stats_buf, dma_len, GFP_ATOMIC); + rc = efx_nic_alloc_buffer(efx, &stats_buf, dma_len, GFP_KERNEL); if (rc) { spin_lock_bh(&efx->stats_lock); return rc; @@ -1938,6 +1929,18 @@ static size_t efx_ef10_update_stats_vf(struct efx_nic *efx, u64 *full_stats, return efx_ef10_update_stats_common(efx, full_stats, core_stats); } +static size_t efx_ef10_update_stats_atomic_vf(struct efx_nic *efx, u64 *full_stats, + struct rtnl_link_stats64 *core_stats) +{ + struct efx_ef10_nic_data *nic_data = efx->nic_data; + + /* In atomic context, cannot update HW stats. Just update the + * software stats and return so the caller can continue. + */ + efx_update_sw_stats(efx, nic_data->stats); + return efx_ef10_update_stats_common(efx, full_stats, core_stats); +} + static void efx_ef10_push_irq_moderation(struct efx_channel *channel) { struct efx_nic *efx = channel->efx; @@ -3998,6 +4001,7 @@ const struct efx_nic_type efx_hunt_a0_vf_nic_type = { .finish_flr = efx_port_dummy_op_void, .describe_stats = efx_ef10_describe_stats, .update_stats = efx_ef10_update_stats_vf, + .update_stats_atomic = efx_ef10_update_stats_atomic_vf, .start_stats = efx_port_dummy_op_void, .pull_stats = efx_port_dummy_op_void, .stop_stats = efx_port_dummy_op_void, diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c index c256db241570..72a3f0e09f52 100644 --- a/drivers/net/ethernet/sfc/efx_common.c +++ b/drivers/net/ethernet/sfc/efx_common.c @@ -602,7 +602,7 @@ void efx_net_stats(struct net_device *net_dev, struct rtnl_link_stats64 *stats) struct efx_nic *efx = netdev_priv(net_dev); spin_lock_bh(&efx->stats_lock); - efx->type->update_stats(efx, NULL, stats); + efx_nic_update_stats_atomic(efx, NULL, stats); spin_unlock_bh(&efx->stats_lock); } diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 47aa753e64bd..9f7dfdf708cf 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -1172,6 +1172,9 @@ struct efx_udp_tunnel { * @describe_stats: Describe statistics for ethtool * @update_stats: Update statistics not provided by event handling. * Either argument may be %NULL. + * @update_stats_atomic: Update statistics while in atomic context, if that + * is more limiting than @update_stats. Otherwise, leave %NULL and + * driver core will call @update_stats. * @start_stats: Start the regular fetching of statistics * @pull_stats: Pull stats from the NIC and wait until they arrive. * @stop_stats: Stop the regular fetching of statistics @@ -1316,6 +1319,8 @@ struct efx_nic_type { size_t (*describe_stats)(struct efx_nic *efx, u8 *names); size_t (*update_stats)(struct efx_nic *efx, u64 *full_stats, struct rtnl_link_stats64 *core_stats); + size_t (*update_stats_atomic)(struct efx_nic *efx, u64 *full_stats, + struct rtnl_link_stats64 *core_stats); void (*start_stats)(struct efx_nic *efx); void (*pull_stats)(struct efx_nic *efx); void (*stop_stats)(struct efx_nic *efx); diff --git a/drivers/net/ethernet/sfc/nic_common.h b/drivers/net/ethernet/sfc/nic_common.h index 82271f0b8627..b9cafe9cd568 100644 --- a/drivers/net/ethernet/sfc/nic_common.h +++ b/drivers/net/ethernet/sfc/nic_common.h @@ -244,6 +244,13 @@ void efx_nic_update_stats(const struct efx_hw_stat_desc *desc, size_t count, const unsigned long *mask, u64 *stats, const void *dma_buf, bool accumulate); void efx_nic_fix_nodesc_drop_stat(struct efx_nic *efx, u64 *stat); +static inline size_t efx_nic_update_stats_atomic(struct efx_nic *efx, u64 *full_stats, + struct rtnl_link_stats64 *core_stats) +{ + if (efx->type->update_stats_atomic) + return efx->type->update_stats_atomic(efx, full_stats, core_stats); + return efx->type->update_stats(efx, full_stats, core_stats); +} #define EFX_MAX_FLUSH_TIME 5000 diff --git a/drivers/net/ethernet/socionext/sni_ave.c b/drivers/net/ethernet/socionext/sni_ave.c index 81b554dd7221..501b9c7aba56 100644 --- a/drivers/net/ethernet/socionext/sni_ave.c +++ b/drivers/net/ethernet/socionext/sni_ave.c @@ -1585,7 +1585,7 @@ static int ave_probe(struct platform_device *pdev) if (IS_ERR(base)) return PTR_ERR(base); - ndev = alloc_etherdev(sizeof(struct ave_private)); + ndev = devm_alloc_etherdev(dev, sizeof(struct ave_private)); if (!ndev) { dev_err(dev, "can't allocate ethernet device\n"); return -ENOMEM; @@ -1632,7 +1632,7 @@ static int ave_probe(struct platform_device *pdev) } ret = dma_set_mask(dev, dma_mask); if (ret) - goto out_free_netdev; + return ret; priv->tx.ndesc = AVE_NR_TXDESC; priv->rx.ndesc = AVE_NR_RXDESC; @@ -1645,10 +1645,8 @@ static int ave_probe(struct platform_device *pdev) if (!name) break; priv->clk[i] = devm_clk_get(dev, name); - if (IS_ERR(priv->clk[i])) { - ret = PTR_ERR(priv->clk[i]); - goto out_free_netdev; - } + if (IS_ERR(priv->clk[i])) + return PTR_ERR(priv->clk[i]); priv->nclks++; } @@ -1657,10 +1655,8 @@ static int ave_probe(struct platform_device *pdev) if (!name) break; priv->rst[i] = devm_reset_control_get_shared(dev, name); - if (IS_ERR(priv->rst[i])) { - ret = PTR_ERR(priv->rst[i]); - goto out_free_netdev; - } + if (IS_ERR(priv->rst[i])) + return PTR_ERR(priv->rst[i]); priv->nrsts++; } @@ -1669,26 +1665,23 @@ static int ave_probe(struct platform_device *pdev) 1, 0, &args); if (ret) { dev_err(dev, "can't get syscon-phy-mode property\n"); - goto out_free_netdev; + return ret; } priv->regmap = syscon_node_to_regmap(args.np); of_node_put(args.np); if (IS_ERR(priv->regmap)) { dev_err(dev, "can't map syscon-phy-mode\n"); - ret = PTR_ERR(priv->regmap); - goto out_free_netdev; + return PTR_ERR(priv->regmap); } ret = priv->data->get_pinmode(priv, phy_mode, args.args[0]); if (ret) { dev_err(dev, "invalid phy-mode setting\n"); - goto out_free_netdev; + return ret; } priv->mdio = devm_mdiobus_alloc(dev); - if (!priv->mdio) { - ret = -ENOMEM; - goto out_free_netdev; - } + if (!priv->mdio) + return -ENOMEM; priv->mdio->priv = ndev; priv->mdio->parent = dev; priv->mdio->read = ave_mdiobus_read; @@ -1725,8 +1718,6 @@ static int ave_probe(struct platform_device *pdev) out_del_napi: netif_napi_del(&priv->napi_rx); netif_napi_del(&priv->napi_tx); -out_free_netdev: - free_netdev(ndev); return ret; } @@ -1739,7 +1730,6 @@ static int ave_remove(struct platform_device *pdev) unregister_netdev(ndev); netif_napi_del(&priv->napi_rx); netif_napi_del(&priv->napi_tx); - free_netdev(ndev); return 0; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel-plat.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel-plat.c index b1323d5c95b5..f61cb997a8f6 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel-plat.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel-plat.c @@ -11,6 +11,7 @@ #include <linux/platform_device.h> #include <linux/stmmac.h> +#include "dwmac4.h" #include "stmmac.h" #include "stmmac_platform.h" @@ -146,6 +147,14 @@ static int intel_eth_plat_probe(struct platform_device *pdev) } plat_dat->bsp_priv = dwmac; + plat_dat->eee_usecs_rate = plat_dat->clk_ptp_rate; + + if (plat_dat->eee_usecs_rate > 0) { + u32 tx_lpi_usec; + + tx_lpi_usec = (plat_dat->eee_usecs_rate / 1000000) - 1; + writel(tx_lpi_usec, stmmac_res.addr + GMAC_1US_TIC_COUNTER); + } ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); if (ret) { diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index ab0a81e0fea1..2af9458be95f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -6,6 +6,7 @@ #include <linux/pci.h> #include <linux/dmi.h> #include "dwmac-intel.h" +#include "dwmac4.h" #include "stmmac.h" struct intel_priv_data { @@ -295,6 +296,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, plat->axi->axi_blen[2] = 16; plat->ptp_max_adj = plat->clk_ptp_rate; + plat->eee_usecs_rate = plat->clk_ptp_rate; /* Set system clock */ plat->stmmac_clk = clk_register_fixed_rate(&pdev->dev, @@ -623,6 +625,13 @@ static int intel_eth_pci_probe(struct pci_dev *pdev, if (ret) return ret; + if (plat->eee_usecs_rate > 0) { + u32 tx_lpi_usec; + + tx_lpi_usec = (plat->eee_usecs_rate / 1000000) - 1; + writel(tx_lpi_usec, res.addr + GMAC_1US_TIC_COUNTER); + } + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); if (ret < 0) return ret; diff --git a/drivers/net/ethernet/sun/sunbmac.c b/drivers/net/ethernet/sun/sunbmac.c index 34fdbc6d6031..c646575e79d5 100644 --- a/drivers/net/ethernet/sun/sunbmac.c +++ b/drivers/net/ethernet/sun/sunbmac.c @@ -209,13 +209,13 @@ static void bigmac_clean_rings(struct bigmac *bp) } } -static void bigmac_init_rings(struct bigmac *bp, int from_irq) +static void bigmac_init_rings(struct bigmac *bp, bool non_blocking) { struct bmac_init_block *bb = bp->bmac_block; int i; gfp_t gfp_flags = GFP_KERNEL; - if (from_irq || in_interrupt()) + if (non_blocking) gfp_flags = GFP_ATOMIC; bp->rx_new = bp->rx_old = bp->tx_new = bp->tx_old = 0; @@ -489,7 +489,7 @@ static void bigmac_tcvr_init(struct bigmac *bp) } } -static int bigmac_init_hw(struct bigmac *, int); +static int bigmac_init_hw(struct bigmac *, bool); static int try_next_permutation(struct bigmac *bp, void __iomem *tregs) { @@ -549,7 +549,7 @@ static void bigmac_timer(struct timer_list *t) if (ret == -1) { printk(KERN_ERR "%s: Link down, cable problem?\n", bp->dev->name); - ret = bigmac_init_hw(bp, 0); + ret = bigmac_init_hw(bp, true); if (ret) { printk(KERN_ERR "%s: Error, cannot re-init the " "BigMAC.\n", bp->dev->name); @@ -617,7 +617,7 @@ static void bigmac_begin_auto_negotiation(struct bigmac *bp) add_timer(&bp->bigmac_timer); } -static int bigmac_init_hw(struct bigmac *bp, int from_irq) +static int bigmac_init_hw(struct bigmac *bp, bool non_blocking) { void __iomem *gregs = bp->gregs; void __iomem *cregs = bp->creg; @@ -635,7 +635,7 @@ static int bigmac_init_hw(struct bigmac *bp, int from_irq) qec_init(bp); /* Alloc and reset the tx/rx descriptor chains. */ - bigmac_init_rings(bp, from_irq); + bigmac_init_rings(bp, non_blocking); /* Initialize the PHY. */ bigmac_tcvr_init(bp); @@ -749,7 +749,7 @@ static void bigmac_is_medium_rare(struct bigmac *bp, u32 qec_status, u32 bmac_st } printk(" RESET\n"); - bigmac_init_hw(bp, 1); + bigmac_init_hw(bp, true); } /* BigMAC transmit complete service routines. */ @@ -921,7 +921,7 @@ static int bigmac_open(struct net_device *dev) return ret; } timer_setup(&bp->bigmac_timer, bigmac_timer, 0); - ret = bigmac_init_hw(bp, 0); + ret = bigmac_init_hw(bp, false); if (ret) free_irq(dev->irq, bp); return ret; @@ -945,7 +945,7 @@ static void bigmac_tx_timeout(struct net_device *dev, unsigned int txqueue) { struct bigmac *bp = netdev_priv(dev); - bigmac_init_hw(bp, 0); + bigmac_init_hw(bp, true); netif_wake_queue(dev); } diff --git a/drivers/net/fddi/skfp/h/smc.h b/drivers/net/fddi/skfp/h/smc.h index 991857f6a83c..706fa619b703 100644 --- a/drivers/net/fddi/skfp/h/smc.h +++ b/drivers/net/fddi/skfp/h/smc.h @@ -122,7 +122,7 @@ struct s_rmt { u_char timer1_exp ; /* flag : timer 1 expired */ u_char timer2_exp ; /* flag : timer 2 expired */ - u_char rm_pad1[1] ; + u_char rm_pad1; } ; /* diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index cb75f7d54057..6bfac1efe037 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -254,8 +254,8 @@ static void gsi_irq_enable(struct gsi *gsi) /* We don't use inter-EE channel or event interrupts */ val = GSI_CNTXT_TYPE_IRQ_MSK_ALL; - val &= ~MSK_INTER_EE_CH_CTRL_FMASK; - val &= ~MSK_INTER_EE_EV_CTRL_FMASK; + val &= ~INTER_EE_CH_CTRL_FMASK; + val &= ~INTER_EE_EV_CTRL_FMASK; iowrite32(val, gsi->virt + GSI_CNTXT_TYPE_IRQ_MSK_OFFSET); val = GENMASK(gsi->channel_count - 1, 0); @@ -271,7 +271,7 @@ static void gsi_irq_enable(struct gsi *gsi) iowrite32(val, gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); /* Never enable GSI_BREAK_POINT */ - val = GSI_CNTXT_GSI_IRQ_ALL & ~EN_BREAK_POINT_FMASK; + val = GSI_CNTXT_GSI_IRQ_ALL & ~BREAK_POINT_FMASK; iowrite32(val, gsi->virt + GSI_CNTXT_GSI_IRQ_EN_OFFSET); } @@ -1074,8 +1074,8 @@ static void gsi_isr_glob_ee(struct gsi *gsi) val &= ~ERROR_INT_FMASK; - if (val & EN_GP_INT1_FMASK) { - val ^= EN_GP_INT1_FMASK; + if (val & GP_INT1_FMASK) { + val ^= GP_INT1_FMASK; gsi_isr_gp_int1(gsi); } @@ -1600,7 +1600,7 @@ err_unwind_modem: /* Compute which modem channels need to be deallocated */ mask ^= gsi->modem_channel_bitmap; while (mask) { - u32 channel_id = __fls(mask); + channel_id = __fls(mask); mask ^= BIT(channel_id); @@ -1628,7 +1628,7 @@ static void gsi_channel_teardown(struct gsi *gsi) mutex_lock(&gsi->mutex); while (mask) { - u32 channel_id = __fls(mask); + channel_id = __fls(mask); mask ^= BIT(channel_id); @@ -1972,7 +1972,6 @@ int gsi_init(struct gsi *gsi, struct platform_device *pdev, bool prefetch, */ init_dummy_netdev(&gsi->dummy_dev); - /* Get the GSI IRQ and request for it to wake the system */ ret = platform_get_irq_byname(pdev, "gsi"); if (ret <= 0) { dev_err(dev, "DT error %d getting \"gsi\" IRQ property\n", ret); diff --git a/drivers/net/ipa/gsi_reg.h b/drivers/net/ipa/gsi_reg.h index acc9e744c67d..8e0e9350c383 100644 --- a/drivers/net/ipa/gsi_reg.h +++ b/drivers/net/ipa/gsi_reg.h @@ -258,6 +258,11 @@ GSI_EE_N_CNTXT_TYPE_IRQ_OFFSET(GSI_EE_AP) #define GSI_EE_N_CNTXT_TYPE_IRQ_OFFSET(ee) \ (0x0001f080 + 0x4000 * (ee)) +#define GSI_CNTXT_TYPE_IRQ_MSK_OFFSET \ + GSI_EE_N_CNTXT_TYPE_IRQ_MSK_OFFSET(GSI_EE_AP) +#define GSI_EE_N_CNTXT_TYPE_IRQ_MSK_OFFSET(ee) \ + (0x0001f088 + 0x4000 * (ee)) +/* The masks below are used for the TYPE_IRQ and TYPE_IRQ_MASK registers */ #define CH_CTRL_FMASK GENMASK(0, 0) #define EV_CTRL_FMASK GENMASK(1, 1) #define GLOB_EE_FMASK GENMASK(2, 2) @@ -265,18 +270,6 @@ #define INTER_EE_CH_CTRL_FMASK GENMASK(4, 4) #define INTER_EE_EV_CTRL_FMASK GENMASK(5, 5) #define GENERAL_FMASK GENMASK(6, 6) - -#define GSI_CNTXT_TYPE_IRQ_MSK_OFFSET \ - GSI_EE_N_CNTXT_TYPE_IRQ_MSK_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_TYPE_IRQ_MSK_OFFSET(ee) \ - (0x0001f088 + 0x4000 * (ee)) -#define MSK_CH_CTRL_FMASK GENMASK(0, 0) -#define MSK_EV_CTRL_FMASK GENMASK(1, 1) -#define MSK_GLOB_EE_FMASK GENMASK(2, 2) -#define MSK_IEOB_FMASK GENMASK(3, 3) -#define MSK_INTER_EE_CH_CTRL_FMASK GENMASK(4, 4) -#define MSK_INTER_EE_EV_CTRL_FMASK GENMASK(5, 5) -#define MSK_GENERAL_FMASK GENMASK(6, 6) #define GSI_CNTXT_TYPE_IRQ_MSK_ALL GENMASK(6, 0) #define GSI_CNTXT_SRC_CH_IRQ_OFFSET \ @@ -328,57 +321,39 @@ GSI_EE_N_CNTXT_GLOB_IRQ_STTS_OFFSET(GSI_EE_AP) #define GSI_EE_N_CNTXT_GLOB_IRQ_STTS_OFFSET(ee) \ (0x0001f100 + 0x4000 * (ee)) -#define ERROR_INT_FMASK GENMASK(0, 0) -#define GP_INT1_FMASK GENMASK(1, 1) -#define GP_INT2_FMASK GENMASK(2, 2) -#define GP_INT3_FMASK GENMASK(3, 3) - #define GSI_CNTXT_GLOB_IRQ_EN_OFFSET \ GSI_EE_N_CNTXT_GLOB_IRQ_EN_OFFSET(GSI_EE_AP) #define GSI_EE_N_CNTXT_GLOB_IRQ_EN_OFFSET(ee) \ (0x0001f108 + 0x4000 * (ee)) -#define EN_ERROR_INT_FMASK GENMASK(0, 0) -#define EN_GP_INT1_FMASK GENMASK(1, 1) -#define EN_GP_INT2_FMASK GENMASK(2, 2) -#define EN_GP_INT3_FMASK GENMASK(3, 3) -#define GSI_CNTXT_GLOB_IRQ_ALL GENMASK(3, 0) - #define GSI_CNTXT_GLOB_IRQ_CLR_OFFSET \ GSI_EE_N_CNTXT_GLOB_IRQ_CLR_OFFSET(GSI_EE_AP) #define GSI_EE_N_CNTXT_GLOB_IRQ_CLR_OFFSET(ee) \ (0x0001f110 + 0x4000 * (ee)) -#define CLR_ERROR_INT_FMASK GENMASK(0, 0) -#define CLR_GP_INT1_FMASK GENMASK(1, 1) -#define CLR_GP_INT2_FMASK GENMASK(2, 2) -#define CLR_GP_INT3_FMASK GENMASK(3, 3) +/* The masks below are used for the general IRQ STTS, EN, and CLR registers */ +#define ERROR_INT_FMASK GENMASK(0, 0) +#define GP_INT1_FMASK GENMASK(1, 1) +#define GP_INT2_FMASK GENMASK(2, 2) +#define GP_INT3_FMASK GENMASK(3, 3) +#define GSI_CNTXT_GLOB_IRQ_ALL GENMASK(3, 0) #define GSI_CNTXT_GSI_IRQ_STTS_OFFSET \ GSI_EE_N_CNTXT_GSI_IRQ_STTS_OFFSET(GSI_EE_AP) #define GSI_EE_N_CNTXT_GSI_IRQ_STTS_OFFSET(ee) \ (0x0001f118 + 0x4000 * (ee)) -#define BREAK_POINT_FMASK GENMASK(0, 0) -#define BUS_ERROR_FMASK GENMASK(1, 1) -#define CMD_FIFO_OVRFLOW_FMASK GENMASK(2, 2) -#define MCS_STACK_OVRFLOW_FMASK GENMASK(3, 3) - #define GSI_CNTXT_GSI_IRQ_EN_OFFSET \ GSI_EE_N_CNTXT_GSI_IRQ_EN_OFFSET(GSI_EE_AP) #define GSI_EE_N_CNTXT_GSI_IRQ_EN_OFFSET(ee) \ (0x0001f120 + 0x4000 * (ee)) -#define EN_BREAK_POINT_FMASK GENMASK(0, 0) -#define EN_BUS_ERROR_FMASK GENMASK(1, 1) -#define EN_CMD_FIFO_OVRFLOW_FMASK GENMASK(2, 2) -#define EN_MCS_STACK_OVRFLOW_FMASK GENMASK(3, 3) -#define GSI_CNTXT_GSI_IRQ_ALL GENMASK(3, 0) - #define GSI_CNTXT_GSI_IRQ_CLR_OFFSET \ GSI_EE_N_CNTXT_GSI_IRQ_CLR_OFFSET(GSI_EE_AP) #define GSI_EE_N_CNTXT_GSI_IRQ_CLR_OFFSET(ee) \ (0x0001f128 + 0x4000 * (ee)) -#define CLR_BREAK_POINT_FMASK GENMASK(0, 0) -#define CLR_BUS_ERROR_FMASK GENMASK(1, 1) -#define CLR_CMD_FIFO_OVRFLOW_FMASK GENMASK(2, 2) -#define CLR_MCS_STACK_OVRFLOW_FMASK GENMASK(3, 3) +/* The masks below are used for the general IRQ STTS, EN, and CLR registers */ +#define BREAK_POINT_FMASK GENMASK(0, 0) +#define BUS_ERROR_FMASK GENMASK(1, 1) +#define CMD_FIFO_OVRFLOW_FMASK GENMASK(2, 2) +#define MCS_STACK_OVRFLOW_FMASK GENMASK(3, 3) +#define GSI_CNTXT_GSI_IRQ_ALL GENMASK(3, 0) #define GSI_CNTXT_INTSET_OFFSET \ GSI_EE_N_CNTXT_INTSET_OFFSET(GSI_EE_AP) diff --git a/drivers/net/ipa/gsi_trans.c b/drivers/net/ipa/gsi_trans.c index bdbfeed359db..43f5f5d93cb0 100644 --- a/drivers/net/ipa/gsi_trans.c +++ b/drivers/net/ipa/gsi_trans.c @@ -81,7 +81,6 @@ struct gsi_tre { /* gsi_tre->flags mask values (in CPU byte order) */ #define TRE_FLAGS_CHAIN_FMASK GENMASK(0, 0) -#define TRE_FLAGS_IEOB_FMASK GENMASK(8, 8) #define TRE_FLAGS_IEOT_FMASK GENMASK(9, 9) #define TRE_FLAGS_BEI_FMASK GENMASK(10, 10) #define TRE_FLAGS_TYPE_FMASK GENMASK(23, 16) diff --git a/drivers/net/ipa/ipa_endpoint.c b/drivers/net/ipa/ipa_endpoint.c index b7efd7c95e9c..24d688e3cd93 100644 --- a/drivers/net/ipa/ipa_endpoint.c +++ b/drivers/net/ipa/ipa_endpoint.c @@ -42,11 +42,8 @@ /** enum ipa_status_opcode - status element opcode hardware values */ enum ipa_status_opcode { IPA_STATUS_OPCODE_PACKET = 0x01, - IPA_STATUS_OPCODE_NEW_FRAG_RULE = 0x02, IPA_STATUS_OPCODE_DROPPED_PACKET = 0x04, IPA_STATUS_OPCODE_SUSPENDED_PACKET = 0x08, - IPA_STATUS_OPCODE_LOG = 0x10, - IPA_STATUS_OPCODE_DCMP = 0x20, IPA_STATUS_OPCODE_PACKET_2ND_PASS = 0x40, }; @@ -54,13 +51,6 @@ enum ipa_status_opcode { enum ipa_status_exception { /* 0 means no exception */ IPA_STATUS_EXCEPTION_DEAGGR = 0x01, - IPA_STATUS_EXCEPTION_IPTYPE = 0x04, - IPA_STATUS_EXCEPTION_PACKET_LENGTH = 0x08, - IPA_STATUS_EXCEPTION_FRAG_RULE_MISS = 0x10, - IPA_STATUS_EXCEPTION_SW_FILT = 0x20, - /* The meaning of the next value depends on whether the IP version */ - IPA_STATUS_EXCEPTION_NAT = 0x40, /* IPv4 */ - IPA_STATUS_EXCEPTION_IPV6CT = IPA_STATUS_EXCEPTION_NAT, }; /* Status element provided by hardware */ @@ -79,36 +69,9 @@ struct ipa_status { }; /* Field masks for struct ipa_status structure fields */ - -#define IPA_STATUS_SRC_IDX_FMASK GENMASK(4, 0) - #define IPA_STATUS_DST_IDX_FMASK GENMASK(4, 0) - -#define IPA_STATUS_FLAGS1_FLT_LOCAL_FMASK GENMASK(0, 0) -#define IPA_STATUS_FLAGS1_FLT_HASH_FMASK GENMASK(1, 1) -#define IPA_STATUS_FLAGS1_FLT_GLOBAL_FMASK GENMASK(2, 2) -#define IPA_STATUS_FLAGS1_FLT_RET_HDR_FMASK GENMASK(3, 3) -#define IPA_STATUS_FLAGS1_FLT_RULE_ID_FMASK GENMASK(13, 4) -#define IPA_STATUS_FLAGS1_RT_LOCAL_FMASK GENMASK(14, 14) -#define IPA_STATUS_FLAGS1_RT_HASH_FMASK GENMASK(15, 15) -#define IPA_STATUS_FLAGS1_UCP_FMASK GENMASK(16, 16) -#define IPA_STATUS_FLAGS1_RT_TBL_IDX_FMASK GENMASK(21, 17) #define IPA_STATUS_FLAGS1_RT_RULE_ID_FMASK GENMASK(31, 22) -#define IPA_STATUS_FLAGS2_NAT_HIT_FMASK GENMASK_ULL(0, 0) -#define IPA_STATUS_FLAGS2_NAT_ENTRY_IDX_FMASK GENMASK_ULL(13, 1) -#define IPA_STATUS_FLAGS2_NAT_TYPE_FMASK GENMASK_ULL(15, 14) -#define IPA_STATUS_FLAGS2_TAG_INFO_FMASK GENMASK_ULL(63, 16) - -#define IPA_STATUS_FLAGS3_SEQ_NUM_FMASK GENMASK(7, 0) -#define IPA_STATUS_FLAGS3_TOD_CTR_FMASK GENMASK(31, 8) - -#define IPA_STATUS_FLAGS4_HDR_LOCAL_FMASK GENMASK(0, 0) -#define IPA_STATUS_FLAGS4_HDR_OFFSET_FMASK GENMASK(10, 1) -#define IPA_STATUS_FLAGS4_FRAG_HIT_FMASK GENMASK(11, 11) -#define IPA_STATUS_FLAGS4_FRAG_RULE_FMASK GENMASK(15, 12) -#define IPA_STATUS_FLAGS4_HW_SPECIFIC_FMASK GENMASK(31, 16) - #ifdef IPA_VALIDATE static void ipa_endpoint_validate_build(void) @@ -1048,8 +1011,7 @@ static bool ipa_endpoint_skb_build(struct ipa_endpoint *endpoint, } /* The format of a packet status element is the same for several status - * types (opcodes). The NEW_FRAG_RULE, LOG, DCMP (decompression) types - * aren't currently supported + * types (opcodes). Other types aren't currently supported. */ static bool ipa_status_format_packet(enum ipa_status_opcode opcode) { @@ -1086,7 +1048,7 @@ static bool ipa_status_drop_packet(const struct ipa_status *status) { u32 val; - /* Deaggregation exceptions we drop; others we consume */ + /* Deaggregation exceptions we drop; all other types we consume */ if (status->exception) return status->exception == IPA_STATUS_EXCEPTION_DEAGGR; @@ -1432,11 +1394,10 @@ void ipa_endpoint_suspend_one(struct ipa_endpoint *endpoint) if (!(endpoint->ipa->enabled & BIT(endpoint->endpoint_id))) return; - if (!endpoint->toward_ipa) + if (!endpoint->toward_ipa) { ipa_endpoint_replenish_disable(endpoint); - - if (!endpoint->toward_ipa) (void)ipa_endpoint_program_suspend(endpoint, true); + } /* IPA v3.5.1 doesn't use channel stop for suspend */ stop_channel = endpoint->ipa->version != IPA_VERSION_3_5_1; diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c index 1044758b501d..cd4d993b0bbb 100644 --- a/drivers/net/ipa/ipa_main.c +++ b/drivers/net/ipa/ipa_main.c @@ -723,8 +723,8 @@ static int ipa_probe(struct platform_device *pdev) bool modem_alloc; bool modem_init; struct ipa *ipa; - phandle phandle; bool prefetch; + phandle ph; int ret; ipa_validate_build(); @@ -736,13 +736,13 @@ static int ipa_probe(struct platform_device *pdev) return -EPROBE_DEFER; /* We rely on remoteproc to tell us about modem state changes */ - phandle = of_property_read_phandle(dev->of_node, "modem-remoteproc"); - if (!phandle) { + ph = of_property_read_phandle(dev->of_node, "modem-remoteproc"); + if (!ph) { dev_err(dev, "DT missing \"modem-remoteproc\" property\n"); return -EINVAL; } - rproc = rproc_get_by_phandle(phandle); + rproc = rproc_get_by_phandle(ph); if (!rproc) return -EPROBE_DEFER; diff --git a/drivers/net/ipa/ipa_reg.h b/drivers/net/ipa/ipa_reg.h index eb4e39fa7d4b..e542598fd775 100644 --- a/drivers/net/ipa/ipa_reg.h +++ b/drivers/net/ipa/ipa_reg.h @@ -426,7 +426,7 @@ enum ipa_cs_offload_en { IPA_CS_RSVD }; -/** enum ipa_aggr_en - aggregation type field in ENDP_INIT_AGGR_N */ +/** enum ipa_aggr_en - aggregation enable field in ENDP_INIT_AGGR_N */ enum ipa_aggr_en { IPA_BYPASS_AGGR = 0, IPA_ENABLE_AGGR = 1, diff --git a/drivers/net/ipa/ipa_uc.c b/drivers/net/ipa/ipa_uc.c index 1a0b04e0ab74..b382d47bc70d 100644 --- a/drivers/net/ipa/ipa_uc.c +++ b/drivers/net/ipa/ipa_uc.c @@ -144,7 +144,7 @@ static void ipa_uc_response_hdlr(struct ipa *ipa, enum ipa_irq_id irq_id) * should only receive responses from the microcontroller when it has * sent it a request message. * - * We can drop the clock reference taken in ipa_uc_init() once we + * We can drop the clock reference taken in ipa_uc_setup() once we * know the microcontroller has finished its initialization. */ switch (shared->response) { diff --git a/drivers/net/netdevsim/netdevsim.h b/drivers/net/netdevsim/netdevsim.h index 0a1b28aea894..827fc80f50a0 100644 --- a/drivers/net/netdevsim/netdevsim.h +++ b/drivers/net/netdevsim/netdevsim.h @@ -20,6 +20,7 @@ #include <linux/netdevice.h> #include <linux/u64_stats_sync.h> #include <net/devlink.h> +#include <net/udp_tunnel.h> #include <net/xdp.h> #define DRV_NAME "netdevsim" @@ -84,7 +85,8 @@ struct netdevsim { struct { u32 inject_error; u32 sleep; - u32 ports[2][NSIM_UDP_TUNNEL_N_PORTS]; + u32 __ports[2][NSIM_UDP_TUNNEL_N_PORTS]; + u32 (*ports)[NSIM_UDP_TUNNEL_N_PORTS]; struct debugfs_u32_array dfs_ports[2]; } udp_ports; @@ -209,9 +211,13 @@ struct nsim_dev { bool fail_trap_policer_set; bool fail_trap_policer_counter_get; struct { + struct udp_tunnel_nic_shared utn_shared; + u32 __ports[2][NSIM_UDP_TUNNEL_N_PORTS]; bool sync_all; bool open_only; bool ipv4_only; + bool shared; + bool static_iana_vxlan; u32 sleep; } udp_ports; }; diff --git a/drivers/net/netdevsim/udp_tunnels.c b/drivers/net/netdevsim/udp_tunnels.c index 22c06a76033c..6ab023acefd6 100644 --- a/drivers/net/netdevsim/udp_tunnels.c +++ b/drivers/net/netdevsim/udp_tunnels.c @@ -22,11 +22,13 @@ nsim_udp_tunnel_set_port(struct net_device *dev, unsigned int table, msleep(ns->udp_ports.sleep); if (!ret) { - if (ns->udp_ports.ports[table][entry]) + if (ns->udp_ports.ports[table][entry]) { + WARN(1, "entry already in use\n"); ret = -EBUSY; - else + } else { ns->udp_ports.ports[table][entry] = be16_to_cpu(ti->port) << 16 | ti->type; + } } netdev_info(dev, "set [%d, %d] type %d family %d port %d - %d\n", @@ -50,10 +52,13 @@ nsim_udp_tunnel_unset_port(struct net_device *dev, unsigned int table, if (!ret) { u32 val = be16_to_cpu(ti->port) << 16 | ti->type; - if (val == ns->udp_ports.ports[table][entry]) + if (val == ns->udp_ports.ports[table][entry]) { ns->udp_ports.ports[table][entry] = 0; - else + } else { + WARN(1, "entry not installed %x vs %x\n", + val, ns->udp_ports.ports[table][entry]); ret = -ENOENT; + } } netdev_info(dev, "unset [%d, %d] type %d family %d port %d - %d\n", @@ -107,7 +112,7 @@ nsim_udp_tunnels_info_reset_write(struct file *file, const char __user *data, struct net_device *dev = file->private_data; struct netdevsim *ns = netdev_priv(dev); - memset(&ns->udp_ports.ports, 0, sizeof(ns->udp_ports.ports)); + memset(ns->udp_ports.ports, 0, sizeof(ns->udp_ports.__ports)); rtnl_lock(); udp_tunnel_nic_reset_ntf(dev); rtnl_unlock(); @@ -127,6 +132,17 @@ int nsim_udp_tunnels_info_create(struct nsim_dev *nsim_dev, struct netdevsim *ns = netdev_priv(dev); struct udp_tunnel_nic_info *info; + if (nsim_dev->udp_ports.shared && nsim_dev->udp_ports.open_only) { + dev_err(&nsim_dev->nsim_bus_dev->dev, + "shared can't be used in conjunction with open_only\n"); + return -EINVAL; + } + + if (!nsim_dev->udp_ports.shared) + ns->udp_ports.ports = ns->udp_ports.__ports; + else + ns->udp_ports.ports = nsim_dev->udp_ports.__ports; + debugfs_create_u32("udp_ports_inject_error", 0600, ns->nsim_dev_port->ddir, &ns->udp_ports.inject_error); @@ -168,6 +184,10 @@ int nsim_udp_tunnels_info_create(struct nsim_dev *nsim_dev, info->flags |= UDP_TUNNEL_NIC_INFO_OPEN_ONLY; if (nsim_dev->udp_ports.ipv4_only) info->flags |= UDP_TUNNEL_NIC_INFO_IPV4_ONLY; + if (nsim_dev->udp_ports.shared) + info->shared = &nsim_dev->udp_ports.utn_shared; + if (nsim_dev->udp_ports.static_iana_vxlan) + info->flags |= UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN; dev->udp_tunnel_nic_info = info; return 0; @@ -187,6 +207,10 @@ void nsim_udp_tunnels_debugfs_create(struct nsim_dev *nsim_dev) &nsim_dev->udp_ports.open_only); debugfs_create_bool("udp_ports_ipv4_only", 0600, nsim_dev->ddir, &nsim_dev->udp_ports.ipv4_only); + debugfs_create_bool("udp_ports_shared", 0600, nsim_dev->ddir, + &nsim_dev->udp_ports.shared); + debugfs_create_bool("udp_ports_static_iana_vxlan", 0600, nsim_dev->ddir, + &nsim_dev->udp_ports.static_iana_vxlan); debugfs_create_u32("udp_ports_sleep", 0600, nsim_dev->ddir, &nsim_dev->udp_ports.sleep); } diff --git a/drivers/net/phy/dp83869.c b/drivers/net/phy/dp83869.c index 81899bc99add..0aee5f645b71 100644 --- a/drivers/net/phy/dp83869.c +++ b/drivers/net/phy/dp83869.c @@ -4,12 +4,14 @@ */ #include <linux/ethtool.h> +#include <linux/etherdevice.h> #include <linux/kernel.h> #include <linux/mii.h> #include <linux/module.h> #include <linux/of.h> #include <linux/phy.h> #include <linux/delay.h> +#include <linux/bitfield.h> #include <dt-bindings/net/ti-dp83869.h> @@ -19,6 +21,7 @@ #define MII_DP83869_PHYCTRL 0x10 #define MII_DP83869_MICR 0x12 #define MII_DP83869_ISR 0x13 +#define DP83869_CFG2 0x14 #define DP83869_CTRL 0x1f #define DP83869_CFG4 0x1e @@ -27,6 +30,13 @@ #define DP83869_RGMIICTL 0x0032 #define DP83869_STRAP_STS1 0x006e #define DP83869_RGMIIDCTL 0x0086 +#define DP83869_RXFCFG 0x0134 +#define DP83869_RXFPMD1 0x0136 +#define DP83869_RXFPMD2 0x0137 +#define DP83869_RXFPMD3 0x0138 +#define DP83869_RXFSOP1 0x0139 +#define DP83869_RXFSOP2 0x013A +#define DP83869_RXFSOP3 0x013B #define DP83869_IO_MUX_CFG 0x0170 #define DP83869_OP_MODE 0x01df #define DP83869_FX_CTRL 0x0c00 @@ -104,6 +114,26 @@ #define DP83869_OP_MODE_MII BIT(5) #define DP83869_SGMII_RGMII_BRIDGE BIT(6) +/* RXFCFG bits*/ +#define DP83869_WOL_MAGIC_EN BIT(0) +#define DP83869_WOL_PATTERN_EN BIT(1) +#define DP83869_WOL_BCAST_EN BIT(2) +#define DP83869_WOL_UCAST_EN BIT(4) +#define DP83869_WOL_SEC_EN BIT(5) +#define DP83869_WOL_ENH_MAC BIT(7) + +/* CFG2 bits */ +#define DP83869_DOWNSHIFT_EN (BIT(8) | BIT(9)) +#define DP83869_DOWNSHIFT_ATTEMPT_MASK (BIT(10) | BIT(11)) +#define DP83869_DOWNSHIFT_1_COUNT_VAL 0 +#define DP83869_DOWNSHIFT_2_COUNT_VAL 1 +#define DP83869_DOWNSHIFT_4_COUNT_VAL 2 +#define DP83869_DOWNSHIFT_8_COUNT_VAL 3 +#define DP83869_DOWNSHIFT_1_COUNT 1 +#define DP83869_DOWNSHIFT_2_COUNT 2 +#define DP83869_DOWNSHIFT_4_COUNT 4 +#define DP83869_DOWNSHIFT_8_COUNT 8 + enum { DP83869_PORT_MIRRORING_KEEP, DP83869_PORT_MIRRORING_EN, @@ -177,6 +207,256 @@ static int dp83869_config_intr(struct phy_device *phydev) return phy_write(phydev, MII_DP83869_MICR, micr_status); } +static int dp83869_set_wol(struct phy_device *phydev, + struct ethtool_wolinfo *wol) +{ + struct net_device *ndev = phydev->attached_dev; + int val_rxcfg, val_micr; + u8 *mac; + int ret; + + val_rxcfg = phy_read_mmd(phydev, DP83869_DEVADDR, DP83869_RXFCFG); + if (val_rxcfg < 0) + return val_rxcfg; + + val_micr = phy_read(phydev, MII_DP83869_MICR); + if (val_micr < 0) + return val_micr; + + if (wol->wolopts & (WAKE_MAGIC | WAKE_MAGICSECURE | WAKE_UCAST | + WAKE_BCAST)) { + val_rxcfg |= DP83869_WOL_ENH_MAC; + val_micr |= MII_DP83869_MICR_WOL_INT_EN; + + if (wol->wolopts & WAKE_MAGIC || + wol->wolopts & WAKE_MAGICSECURE) { + mac = (u8 *)ndev->dev_addr; + + if (!is_valid_ether_addr(mac)) + return -EINVAL; + + ret = phy_write_mmd(phydev, DP83869_DEVADDR, + DP83869_RXFPMD1, + mac[1] << 8 | mac[0]); + if (ret) + return ret; + + ret = phy_write_mmd(phydev, DP83869_DEVADDR, + DP83869_RXFPMD2, + mac[3] << 8 | mac[2]); + if (ret) + return ret; + + ret = phy_write_mmd(phydev, DP83869_DEVADDR, + DP83869_RXFPMD3, + mac[5] << 8 | mac[4]); + if (ret) + return ret; + + val_rxcfg |= DP83869_WOL_MAGIC_EN; + } else { + val_rxcfg &= ~DP83869_WOL_MAGIC_EN; + } + + if (wol->wolopts & WAKE_MAGICSECURE) { + ret = phy_write_mmd(phydev, DP83869_DEVADDR, + DP83869_RXFSOP1, + (wol->sopass[1] << 8) | wol->sopass[0]); + if (ret) + return ret; + + ret = phy_write_mmd(phydev, DP83869_DEVADDR, + DP83869_RXFSOP2, + (wol->sopass[3] << 8) | wol->sopass[2]); + if (ret) + return ret; + ret = phy_write_mmd(phydev, DP83869_DEVADDR, + DP83869_RXFSOP3, + (wol->sopass[5] << 8) | wol->sopass[4]); + if (ret) + return ret; + + val_rxcfg |= DP83869_WOL_SEC_EN; + } else { + val_rxcfg &= ~DP83869_WOL_SEC_EN; + } + + if (wol->wolopts & WAKE_UCAST) + val_rxcfg |= DP83869_WOL_UCAST_EN; + else + val_rxcfg &= ~DP83869_WOL_UCAST_EN; + + if (wol->wolopts & WAKE_BCAST) + val_rxcfg |= DP83869_WOL_BCAST_EN; + else + val_rxcfg &= ~DP83869_WOL_BCAST_EN; + } else { + val_rxcfg &= ~DP83869_WOL_ENH_MAC; + val_micr &= ~MII_DP83869_MICR_WOL_INT_EN; + } + + ret = phy_write_mmd(phydev, DP83869_DEVADDR, DP83869_RXFCFG, val_rxcfg); + if (ret) + return ret; + + return phy_write(phydev, MII_DP83869_MICR, val_micr); +} + +static void dp83869_get_wol(struct phy_device *phydev, + struct ethtool_wolinfo *wol) +{ + u16 value, sopass_val; + + wol->supported = (WAKE_UCAST | WAKE_BCAST | WAKE_MAGIC | + WAKE_MAGICSECURE); + wol->wolopts = 0; + + value = phy_read_mmd(phydev, DP83869_DEVADDR, DP83869_RXFCFG); + if (value < 0) { + phydev_err(phydev, "Failed to read RX CFG\n"); + return; + } + + if (value & DP83869_WOL_UCAST_EN) + wol->wolopts |= WAKE_UCAST; + + if (value & DP83869_WOL_BCAST_EN) + wol->wolopts |= WAKE_BCAST; + + if (value & DP83869_WOL_MAGIC_EN) + wol->wolopts |= WAKE_MAGIC; + + if (value & DP83869_WOL_SEC_EN) { + sopass_val = phy_read_mmd(phydev, DP83869_DEVADDR, + DP83869_RXFSOP1); + if (sopass_val < 0) { + phydev_err(phydev, "Failed to read RX SOP 1\n"); + return; + } + + wol->sopass[0] = (sopass_val & 0xff); + wol->sopass[1] = (sopass_val >> 8); + + sopass_val = phy_read_mmd(phydev, DP83869_DEVADDR, + DP83869_RXFSOP2); + if (sopass_val < 0) { + phydev_err(phydev, "Failed to read RX SOP 2\n"); + return; + } + + wol->sopass[2] = (sopass_val & 0xff); + wol->sopass[3] = (sopass_val >> 8); + + sopass_val = phy_read_mmd(phydev, DP83869_DEVADDR, + DP83869_RXFSOP3); + if (sopass_val < 0) { + phydev_err(phydev, "Failed to read RX SOP 3\n"); + return; + } + + wol->sopass[4] = (sopass_val & 0xff); + wol->sopass[5] = (sopass_val >> 8); + + wol->wolopts |= WAKE_MAGICSECURE; + } + + if (!(value & DP83869_WOL_ENH_MAC)) + wol->wolopts = 0; +} + +static int dp83869_get_downshift(struct phy_device *phydev, u8 *data) +{ + int val, cnt, enable, count; + + val = phy_read(phydev, DP83869_CFG2); + if (val < 0) + return val; + + enable = FIELD_GET(DP83869_DOWNSHIFT_EN, val); + cnt = FIELD_GET(DP83869_DOWNSHIFT_ATTEMPT_MASK, val); + + switch (cnt) { + case DP83869_DOWNSHIFT_1_COUNT_VAL: + count = DP83869_DOWNSHIFT_1_COUNT; + break; + case DP83869_DOWNSHIFT_2_COUNT_VAL: + count = DP83869_DOWNSHIFT_2_COUNT; + break; + case DP83869_DOWNSHIFT_4_COUNT_VAL: + count = DP83869_DOWNSHIFT_4_COUNT; + break; + case DP83869_DOWNSHIFT_8_COUNT_VAL: + count = DP83869_DOWNSHIFT_8_COUNT; + break; + default: + return -EINVAL; + } + + *data = enable ? count : DOWNSHIFT_DEV_DISABLE; + + return 0; +} + +static int dp83869_set_downshift(struct phy_device *phydev, u8 cnt) +{ + int val, count; + + if (cnt > DP83869_DOWNSHIFT_8_COUNT) + return -EINVAL; + + if (!cnt) + return phy_clear_bits(phydev, DP83869_CFG2, + DP83869_DOWNSHIFT_EN); + + switch (cnt) { + case DP83869_DOWNSHIFT_1_COUNT: + count = DP83869_DOWNSHIFT_1_COUNT_VAL; + break; + case DP83869_DOWNSHIFT_2_COUNT: + count = DP83869_DOWNSHIFT_2_COUNT_VAL; + break; + case DP83869_DOWNSHIFT_4_COUNT: + count = DP83869_DOWNSHIFT_4_COUNT_VAL; + break; + case DP83869_DOWNSHIFT_8_COUNT: + count = DP83869_DOWNSHIFT_8_COUNT_VAL; + break; + default: + phydev_err(phydev, + "Downshift count must be 1, 2, 4 or 8\n"); + return -EINVAL; + } + + val = DP83869_DOWNSHIFT_EN; + val |= FIELD_PREP(DP83869_DOWNSHIFT_ATTEMPT_MASK, count); + + return phy_modify(phydev, DP83869_CFG2, + DP83869_DOWNSHIFT_EN | DP83869_DOWNSHIFT_ATTEMPT_MASK, + val); +} + +static int dp83869_get_tunable(struct phy_device *phydev, + struct ethtool_tunable *tuna, void *data) +{ + switch (tuna->id) { + case ETHTOOL_PHY_DOWNSHIFT: + return dp83869_get_downshift(phydev, data); + default: + return -EOPNOTSUPP; + } +} + +static int dp83869_set_tunable(struct phy_device *phydev, + struct ethtool_tunable *tuna, const void *data) +{ + switch (tuna->id) { + case ETHTOOL_PHY_DOWNSHIFT: + return dp83869_set_downshift(phydev, *(const u8 *)data); + default: + return -EOPNOTSUPP; + } +} + static int dp83869_config_port_mirroring(struct phy_device *phydev) { struct dp83869_private *dp83869 = phydev->priv; @@ -469,6 +749,12 @@ static int dp83869_config_init(struct phy_device *phydev) struct dp83869_private *dp83869 = phydev->priv; int ret, val; + /* Force speed optimization for the PHY even if it strapped */ + ret = phy_modify(phydev, DP83869_CFG2, DP83869_DOWNSHIFT_EN, + DP83869_DOWNSHIFT_EN); + if (ret) + return ret; + ret = dp83869_configure_mode(phydev, dp83869); if (ret) return ret; @@ -568,6 +854,12 @@ static struct phy_driver dp83869_driver[] = { .config_intr = dp83869_config_intr, .read_status = dp83869_read_status, + .get_tunable = dp83869_get_tunable, + .set_tunable = dp83869_set_tunable, + + .get_wol = dp83869_get_wol, + .set_wol = dp83869_set_wol, + .suspend = genphy_suspend, .resume = genphy_resume, }, diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c index 0af20faad69d..757e950fb745 100644 --- a/drivers/net/phy/mdio_bus.c +++ b/drivers/net/phy/mdio_bus.c @@ -825,9 +825,6 @@ int mdiobus_read_nested(struct mii_bus *bus, int addr, u32 regnum) { int retval; - if (WARN_ON_ONCE(in_interrupt())) - return -EINVAL; - mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED); retval = __mdiobus_read(bus, addr, regnum); mutex_unlock(&bus->mdio_lock); @@ -850,9 +847,6 @@ int mdiobus_read(struct mii_bus *bus, int addr, u32 regnum) { int retval; - if (WARN_ON_ONCE(in_interrupt())) - return -EINVAL; - mutex_lock(&bus->mdio_lock); retval = __mdiobus_read(bus, addr, regnum); mutex_unlock(&bus->mdio_lock); @@ -879,9 +873,6 @@ int mdiobus_write_nested(struct mii_bus *bus, int addr, u32 regnum, u16 val) { int err; - if (WARN_ON_ONCE(in_interrupt())) - return -EINVAL; - mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED); err = __mdiobus_write(bus, addr, regnum, val); mutex_unlock(&bus->mdio_lock); @@ -905,9 +896,6 @@ int mdiobus_write(struct mii_bus *bus, int addr, u32 regnum, u16 val) { int err; - if (WARN_ON_ONCE(in_interrupt())) - return -EINVAL; - mutex_lock(&bus->mdio_lock); err = __mdiobus_write(bus, addr, regnum, val); mutex_unlock(&bus->mdio_lock); @@ -929,9 +917,6 @@ int mdiobus_modify(struct mii_bus *bus, int addr, u32 regnum, u16 mask, u16 set) { int err; - if (WARN_ON_ONCE(in_interrupt())) - return -EINVAL; - mutex_lock(&bus->mdio_lock); err = __mdiobus_modify_changed(bus, addr, regnum, mask, set); mutex_unlock(&bus->mdio_lock); diff --git a/drivers/net/phy/realtek.c b/drivers/net/phy/realtek.c index 4bf54cded48a..1af874d36ced 100644 --- a/drivers/net/phy/realtek.c +++ b/drivers/net/phy/realtek.c @@ -409,7 +409,7 @@ static int rtlgen_write_mmd(struct phy_device *phydev, int devnum, u16 regnum, return ret; } -static int rtl8125_read_mmd(struct phy_device *phydev, int devnum, u16 regnum) +static int rtl822x_read_mmd(struct phy_device *phydev, int devnum, u16 regnum) { int ret = rtlgen_read_mmd(phydev, devnum, regnum); @@ -433,7 +433,7 @@ static int rtl8125_read_mmd(struct phy_device *phydev, int devnum, u16 regnum) return ret; } -static int rtl8125_write_mmd(struct phy_device *phydev, int devnum, u16 regnum, +static int rtl822x_write_mmd(struct phy_device *phydev, int devnum, u16 regnum, u16 val) { int ret = rtlgen_write_mmd(phydev, devnum, regnum, val); @@ -450,7 +450,7 @@ static int rtl8125_write_mmd(struct phy_device *phydev, int devnum, u16 regnum, return ret; } -static int rtl8125_get_features(struct phy_device *phydev) +static int rtl822x_get_features(struct phy_device *phydev) { int val; @@ -468,7 +468,7 @@ static int rtl8125_get_features(struct phy_device *phydev) return genphy_read_abilities(phydev); } -static int rtl8125_config_aneg(struct phy_device *phydev) +static int rtl822x_config_aneg(struct phy_device *phydev) { int ret = 0; @@ -488,7 +488,7 @@ static int rtl8125_config_aneg(struct phy_device *phydev) return __genphy_config_aneg(phydev, ret); } -static int rtl8125_read_status(struct phy_device *phydev) +static int rtl822x_read_status(struct phy_device *phydev) { int ret; @@ -530,7 +530,7 @@ static int rtlgen_match_phy_device(struct phy_device *phydev) !rtlgen_supports_2_5gbps(phydev); } -static int rtl8125_match_phy_device(struct phy_device *phydev) +static int rtl8226_match_phy_device(struct phy_device *phydev) { return phydev->phy_id == RTL_GENERIC_PHYID && rtlgen_supports_2_5gbps(phydev); @@ -635,29 +635,29 @@ static struct phy_driver realtek_drvs[] = { .read_mmd = rtlgen_read_mmd, .write_mmd = rtlgen_write_mmd, }, { - .name = "RTL8125 2.5Gbps internal", - .match_phy_device = rtl8125_match_phy_device, - .get_features = rtl8125_get_features, - .config_aneg = rtl8125_config_aneg, - .read_status = rtl8125_read_status, + .name = "RTL8226 2.5Gbps PHY", + .match_phy_device = rtl8226_match_phy_device, + .get_features = rtl822x_get_features, + .config_aneg = rtl822x_config_aneg, + .read_status = rtl822x_read_status, .suspend = genphy_suspend, .resume = rtlgen_resume, .read_page = rtl821x_read_page, .write_page = rtl821x_write_page, - .read_mmd = rtl8125_read_mmd, - .write_mmd = rtl8125_write_mmd, + .read_mmd = rtl822x_read_mmd, + .write_mmd = rtl822x_write_mmd, }, { PHY_ID_MATCH_EXACT(0x001cc840), - .name = "RTL8125B 2.5Gbps internal", - .get_features = rtl8125_get_features, - .config_aneg = rtl8125_config_aneg, - .read_status = rtl8125_read_status, + .name = "RTL8226B_RTL8221B 2.5Gbps PHY", + .get_features = rtl822x_get_features, + .config_aneg = rtl822x_config_aneg, + .read_status = rtl822x_read_status, .suspend = genphy_suspend, .resume = rtlgen_resume, .read_page = rtl821x_read_page, .write_page = rtl821x_write_page, - .read_mmd = rtl8125_read_mmd, - .write_mmd = rtl8125_write_mmd, + .read_mmd = rtl822x_read_mmd, + .write_mmd = rtl822x_write_mmd, }, { PHY_ID_MATCH_EXACT(0x001cc961), .name = "RTL8366RB Gigabit Ethernet", diff --git a/drivers/net/phy/spi_ks8995.c b/drivers/net/phy/spi_ks8995.c index 7475cef17cf7..ca49c1ad3efc 100644 --- a/drivers/net/phy/spi_ks8995.c +++ b/drivers/net/phy/spi_ks8995.c @@ -300,7 +300,7 @@ static ssize_t ks8995_registers_read(struct file *filp, struct kobject *kobj, struct device *dev; struct ks8995_switch *ks8995; - dev = container_of(kobj, struct device, kobj); + dev = kobj_to_dev(kobj); ks8995 = dev_get_drvdata(dev); return ks8995_read(ks8995, buf, off, count); @@ -312,7 +312,7 @@ static ssize_t ks8995_registers_write(struct file *filp, struct kobject *kobj, struct device *dev; struct ks8995_switch *ks8995; - dev = container_of(kobj, struct device, kobj); + dev = kobj_to_dev(kobj); ks8995 = dev_get_drvdata(dev); return ks8995_write(ks8995, buf, off, count); diff --git a/drivers/net/usb/kaweth.c b/drivers/net/usb/kaweth.c index ed01dc964c99..144c686b4333 100644 --- a/drivers/net/usb/kaweth.c +++ b/drivers/net/usb/kaweth.c @@ -103,10 +103,6 @@ static int kaweth_probe( const struct usb_device_id *id /* from id_table */ ); static void kaweth_disconnect(struct usb_interface *intf); -static int kaweth_internal_control_msg(struct usb_device *usb_dev, - unsigned int pipe, - struct usb_ctrlrequest *cmd, void *data, - int len, int timeout); static int kaweth_suspend(struct usb_interface *intf, pm_message_t message); static int kaweth_resume(struct usb_interface *intf); @@ -236,65 +232,17 @@ struct kaweth_device }; /**************************************************************** - * kaweth_control - ****************************************************************/ -static int kaweth_control(struct kaweth_device *kaweth, - unsigned int pipe, - __u8 request, - __u8 requesttype, - __u16 value, - __u16 index, - void *data, - __u16 size, - int timeout) -{ - struct usb_ctrlrequest *dr; - int retval; - - if(in_interrupt()) { - netdev_dbg(kaweth->net, "in_interrupt()\n"); - return -EBUSY; - } - - dr = kmalloc(sizeof(struct usb_ctrlrequest), GFP_ATOMIC); - if (!dr) - return -ENOMEM; - - dr->bRequestType = requesttype; - dr->bRequest = request; - dr->wValue = cpu_to_le16(value); - dr->wIndex = cpu_to_le16(index); - dr->wLength = cpu_to_le16(size); - - retval = kaweth_internal_control_msg(kaweth->dev, - pipe, - dr, - data, - size, - timeout); - - kfree(dr); - return retval; -} - -/**************************************************************** * kaweth_read_configuration ****************************************************************/ static int kaweth_read_configuration(struct kaweth_device *kaweth) { - int retval; - - retval = kaweth_control(kaweth, - usb_rcvctrlpipe(kaweth->dev, 0), + return usb_control_msg(kaweth->dev, usb_rcvctrlpipe(kaweth->dev, 0), KAWETH_COMMAND_GET_ETHERNET_DESC, USB_TYPE_VENDOR | USB_DIR_IN | USB_RECIP_DEVICE, - 0, - 0, - (void *)&kaweth->configuration, + 0, 0, + &kaweth->configuration, sizeof(kaweth->configuration), KAWETH_CONTROL_TIMEOUT); - - return retval; } /**************************************************************** @@ -302,21 +250,14 @@ static int kaweth_read_configuration(struct kaweth_device *kaweth) ****************************************************************/ static int kaweth_set_urb_size(struct kaweth_device *kaweth, __u16 urb_size) { - int retval; - netdev_dbg(kaweth->net, "Setting URB size to %d\n", (unsigned)urb_size); - retval = kaweth_control(kaweth, - usb_sndctrlpipe(kaweth->dev, 0), - KAWETH_COMMAND_SET_URB_SIZE, - USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, - urb_size, - 0, - (void *)&kaweth->scratch, - 0, - KAWETH_CONTROL_TIMEOUT); - - return retval; + return usb_control_msg(kaweth->dev, usb_sndctrlpipe(kaweth->dev, 0), + KAWETH_COMMAND_SET_URB_SIZE, + USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, + urb_size, 0, + &kaweth->scratch, 0, + KAWETH_CONTROL_TIMEOUT); } /**************************************************************** @@ -324,21 +265,14 @@ static int kaweth_set_urb_size(struct kaweth_device *kaweth, __u16 urb_size) ****************************************************************/ static int kaweth_set_sofs_wait(struct kaweth_device *kaweth, __u16 sofs_wait) { - int retval; - netdev_dbg(kaweth->net, "Set SOFS wait to %d\n", (unsigned)sofs_wait); - retval = kaweth_control(kaweth, - usb_sndctrlpipe(kaweth->dev, 0), - KAWETH_COMMAND_SET_SOFS_WAIT, - USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, - sofs_wait, - 0, - (void *)&kaweth->scratch, - 0, - KAWETH_CONTROL_TIMEOUT); - - return retval; + return usb_control_msg(kaweth->dev, usb_sndctrlpipe(kaweth->dev, 0), + KAWETH_COMMAND_SET_SOFS_WAIT, + USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, + sofs_wait, 0, + &kaweth->scratch, 0, + KAWETH_CONTROL_TIMEOUT); } /**************************************************************** @@ -347,22 +281,15 @@ static int kaweth_set_sofs_wait(struct kaweth_device *kaweth, __u16 sofs_wait) static int kaweth_set_receive_filter(struct kaweth_device *kaweth, __u16 receive_filter) { - int retval; - netdev_dbg(kaweth->net, "Set receive filter to %d\n", (unsigned)receive_filter); - retval = kaweth_control(kaweth, - usb_sndctrlpipe(kaweth->dev, 0), - KAWETH_COMMAND_SET_PACKET_FILTER, - USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, - receive_filter, - 0, - (void *)&kaweth->scratch, - 0, - KAWETH_CONTROL_TIMEOUT); - - return retval; + return usb_control_msg(kaweth->dev, usb_sndctrlpipe(kaweth->dev, 0), + KAWETH_COMMAND_SET_PACKET_FILTER, + USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, + receive_filter, 0, + &kaweth->scratch, 0, + KAWETH_CONTROL_TIMEOUT); } /**************************************************************** @@ -407,14 +334,11 @@ static int kaweth_download_firmware(struct kaweth_device *kaweth, kaweth->firmware_buf, kaweth); netdev_dbg(kaweth->net, "Firmware length: %d\n", data_len); - return kaweth_control(kaweth, - usb_sndctrlpipe(kaweth->dev, 0), + return usb_control_msg(kaweth->dev, usb_sndctrlpipe(kaweth->dev, 0), KAWETH_COMMAND_SCAN, USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, - 0, - 0, - (void *)kaweth->firmware_buf, - data_len, + 0, 0, + kaweth->firmware_buf, data_len, KAWETH_CONTROL_TIMEOUT); } @@ -433,15 +357,12 @@ static int kaweth_trigger_firmware(struct kaweth_device *kaweth, kaweth->firmware_buf[6] = 0x00; kaweth->firmware_buf[7] = 0x00; - return kaweth_control(kaweth, - usb_sndctrlpipe(kaweth->dev, 0), - KAWETH_COMMAND_SCAN, - USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, - 0, - 0, - (void *)kaweth->firmware_buf, - 8, - KAWETH_CONTROL_TIMEOUT); + return usb_control_msg(kaweth->dev, usb_sndctrlpipe(kaweth->dev, 0), + KAWETH_COMMAND_SCAN, + USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, + 0, 0, + (void *)kaweth->firmware_buf, 8, + KAWETH_CONTROL_TIMEOUT); } /**************************************************************** @@ -564,7 +485,8 @@ static int kaweth_resubmit_rx_urb(struct kaweth_device *kaweth, return result; } -static void kaweth_async_set_rx_mode(struct kaweth_device *kaweth); +static void kaweth_async_set_rx_mode(struct kaweth_device *kaweth, + bool may_sleep); /**************************************************************** * kaweth_usb_receive @@ -694,7 +616,7 @@ static int kaweth_open(struct net_device *net) netif_start_queue(net); - kaweth_async_set_rx_mode(kaweth); + kaweth_async_set_rx_mode(kaweth, true); return 0; err_out: @@ -782,7 +704,7 @@ static netdev_tx_t kaweth_start_xmit(struct sk_buff *skb, spin_lock_irq(&kaweth->device_lock); - kaweth_async_set_rx_mode(kaweth); + kaweth_async_set_rx_mode(kaweth, false); netif_stop_queue(net); if (IS_BLOCKED(kaweth->status)) { goto skip; @@ -859,36 +781,31 @@ static void kaweth_set_rx_mode(struct net_device *net) /**************************************************************** * kaweth_async_set_rx_mode ****************************************************************/ -static void kaweth_async_set_rx_mode(struct kaweth_device *kaweth) +static void kaweth_async_set_rx_mode(struct kaweth_device *kaweth, + bool may_sleep) { - int result; + int ret; __u16 packet_filter_bitmap = kaweth->packet_filter_bitmap; kaweth->packet_filter_bitmap = 0; if (packet_filter_bitmap == 0) return; - if (in_interrupt()) + if (!may_sleep) return; - result = kaweth_control(kaweth, - usb_sndctrlpipe(kaweth->dev, 0), - KAWETH_COMMAND_SET_PACKET_FILTER, - USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, - packet_filter_bitmap, - 0, - (void *)&kaweth->scratch, - 0, - KAWETH_CONTROL_TIMEOUT); - - if(result < 0) { + ret = usb_control_msg(kaweth->dev, usb_sndctrlpipe(kaweth->dev, 0), + KAWETH_COMMAND_SET_PACKET_FILTER, + USB_TYPE_VENDOR | USB_DIR_OUT | USB_RECIP_DEVICE, + packet_filter_bitmap, 0, + &kaweth->scratch, 0, + KAWETH_CONTROL_TIMEOUT); + if (ret < 0) dev_err(&kaweth->intf->dev, "Failed to set Rx mode: %d\n", - result); - } - else { + ret); + else netdev_dbg(kaweth->net, "Set Rx mode to %d\n", packet_filter_bitmap); - } } /**************************************************************** @@ -1196,88 +1113,4 @@ static void kaweth_disconnect(struct usb_interface *intf) } -// FIXME this completion stuff is a modified clone of -// an OLD version of some stuff in usb.c ... -struct usb_api_data { - wait_queue_head_t wqh; - int done; -}; - -/*-------------------------------------------------------------------* - * completion handler for compatibility wrappers (sync control/bulk) * - *-------------------------------------------------------------------*/ -static void usb_api_blocking_completion(struct urb *urb) -{ - struct usb_api_data *awd = (struct usb_api_data *)urb->context; - - awd->done=1; - wake_up(&awd->wqh); -} - -/*-------------------------------------------------------------------* - * COMPATIBILITY STUFF * - *-------------------------------------------------------------------*/ - -// Starts urb and waits for completion or timeout -static int usb_start_wait_urb(struct urb *urb, int timeout, int* actual_length) -{ - struct usb_api_data awd; - int status; - - init_waitqueue_head(&awd.wqh); - awd.done = 0; - - urb->context = &awd; - status = usb_submit_urb(urb, GFP_ATOMIC); - if (status) { - // something went wrong - usb_free_urb(urb); - return status; - } - - if (!wait_event_timeout(awd.wqh, awd.done, timeout)) { - // timeout - dev_warn(&urb->dev->dev, "usb_control/bulk_msg: timeout\n"); - usb_kill_urb(urb); // remove urb safely - status = -ETIMEDOUT; - } - else { - status = urb->status; - } - - if (actual_length) { - *actual_length = urb->actual_length; - } - - usb_free_urb(urb); - return status; -} - -/*-------------------------------------------------------------------*/ -// returns status (negative) or length (positive) -static int kaweth_internal_control_msg(struct usb_device *usb_dev, - unsigned int pipe, - struct usb_ctrlrequest *cmd, void *data, - int len, int timeout) -{ - struct urb *urb; - int retv; - int length = 0; /* shut up GCC */ - - urb = usb_alloc_urb(0, GFP_ATOMIC); - if (!urb) - return -ENOMEM; - - usb_fill_control_urb(urb, usb_dev, pipe, (unsigned char*)cmd, data, - len, usb_api_blocking_completion, NULL); - - retv = usb_start_wait_urb(urb, timeout, &length); - if (retv < 0) { - return retv; - } - else { - return length; - } -} - module_usb_driver(kaweth_driver); diff --git a/drivers/net/usb/net1080.c b/drivers/net/usb/net1080.c index 1f04f1720426..b0c0c9dd6a02 100644 --- a/drivers/net/usb/net1080.c +++ b/drivers/net/usb/net1080.c @@ -113,7 +113,6 @@ nc_register_read(struct usbnet *dev, u8 regnum, u16 *retval_ptr) return nc_vendor_read(dev, REQUEST_REGISTER, regnum, retval_ptr); } -// no retval ... can become async, usable in_interrupt() static void nc_vendor_write(struct usbnet *dev, u8 req, u8 regnum, u16 value) { diff --git a/drivers/net/usb/smsc75xx.c b/drivers/net/usb/smsc75xx.c index 9556d431885f..8689835a5214 100644 --- a/drivers/net/usb/smsc75xx.c +++ b/drivers/net/usb/smsc75xx.c @@ -757,13 +757,14 @@ static int smsc75xx_ioctl(struct net_device *netdev, struct ifreq *rq, int cmd) static void smsc75xx_init_mac_address(struct usbnet *dev) { - const u8 *mac_addr; - /* maybe the boot loader passed the MAC address in devicetree */ - mac_addr = of_get_mac_address(dev->udev->dev.of_node); - if (!IS_ERR(mac_addr)) { - ether_addr_copy(dev->net->dev_addr, mac_addr); - return; + if (!eth_platform_get_mac_address(&dev->udev->dev, + dev->net->dev_addr)) { + if (is_valid_ether_addr(dev->net->dev_addr)) { + /* device tree values are valid so use them */ + netif_dbg(dev, ifup, dev->net, "MAC address read from the device tree\n"); + return; + } } /* try reading mac address from EEPROM */ diff --git a/drivers/net/usb/smsc95xx.c b/drivers/net/usb/smsc95xx.c index 601fb40a2a0a..ea0d5f04dc3a 100644 --- a/drivers/net/usb/smsc95xx.c +++ b/drivers/net/usb/smsc95xx.c @@ -755,13 +755,14 @@ static int smsc95xx_ioctl(struct net_device *netdev, struct ifreq *rq, int cmd) static void smsc95xx_init_mac_address(struct usbnet *dev) { - const u8 *mac_addr; - /* maybe the boot loader passed the MAC address in devicetree */ - mac_addr = of_get_mac_address(dev->udev->dev.of_node); - if (!IS_ERR(mac_addr)) { - ether_addr_copy(dev->net->dev_addr, mac_addr); - return; + if (!eth_platform_get_mac_address(&dev->udev->dev, + dev->net->dev_addr)) { + if (is_valid_ether_addr(dev->net->dev_addr)) { + /* device tree values are valid so use them */ + netif_dbg(dev, ifup, dev->net, "MAC address read from the device tree\n"); + return; + } } /* try reading mac address from EEPROM */ diff --git a/drivers/net/wan/hdlc_fr.c b/drivers/net/wan/hdlc_fr.c index d6cfd51613ed..3a44dad87602 100644 --- a/drivers/net/wan/hdlc_fr.c +++ b/drivers/net/wan/hdlc_fr.c @@ -273,63 +273,69 @@ static inline struct net_device **get_dev_p(struct pvc_device *pvc, static int fr_hard_header(struct sk_buff **skb_p, u16 dlci) { - u16 head_len; struct sk_buff *skb = *skb_p; - switch (skb->protocol) { - case cpu_to_be16(NLPID_CCITT_ANSI_LMI): - head_len = 4; - skb_push(skb, head_len); - skb->data[3] = NLPID_CCITT_ANSI_LMI; - break; - - case cpu_to_be16(NLPID_CISCO_LMI): - head_len = 4; - skb_push(skb, head_len); - skb->data[3] = NLPID_CISCO_LMI; - break; - - case cpu_to_be16(ETH_P_IP): - head_len = 4; - skb_push(skb, head_len); - skb->data[3] = NLPID_IP; - break; - - case cpu_to_be16(ETH_P_IPV6): - head_len = 4; - skb_push(skb, head_len); - skb->data[3] = NLPID_IPV6; - break; - - case cpu_to_be16(ETH_P_802_3): - head_len = 10; - if (skb_headroom(skb) < head_len) { - struct sk_buff *skb2 = skb_realloc_headroom(skb, - head_len); + if (!skb->dev) { /* Control packets */ + switch (dlci) { + case LMI_CCITT_ANSI_DLCI: + skb_push(skb, 4); + skb->data[3] = NLPID_CCITT_ANSI_LMI; + break; + + case LMI_CISCO_DLCI: + skb_push(skb, 4); + skb->data[3] = NLPID_CISCO_LMI; + break; + + default: + return -EINVAL; + } + + } else if (skb->dev->type == ARPHRD_DLCI) { + switch (skb->protocol) { + case htons(ETH_P_IP): + skb_push(skb, 4); + skb->data[3] = NLPID_IP; + break; + + case htons(ETH_P_IPV6): + skb_push(skb, 4); + skb->data[3] = NLPID_IPV6; + break; + + default: + skb_push(skb, 10); + skb->data[3] = FR_PAD; + skb->data[4] = NLPID_SNAP; + /* OUI 00-00-00 indicates an Ethertype follows */ + skb->data[5] = 0x00; + skb->data[6] = 0x00; + skb->data[7] = 0x00; + /* This should be an Ethertype: */ + *(__be16 *)(skb->data + 8) = skb->protocol; + } + + } else if (skb->dev->type == ARPHRD_ETHER) { + if (skb_headroom(skb) < 10) { + struct sk_buff *skb2 = skb_realloc_headroom(skb, 10); if (!skb2) return -ENOBUFS; dev_kfree_skb(skb); skb = *skb_p = skb2; } - skb_push(skb, head_len); + skb_push(skb, 10); skb->data[3] = FR_PAD; skb->data[4] = NLPID_SNAP; - skb->data[5] = FR_PAD; + /* OUI 00-80-C2 stands for the 802.1 organization */ + skb->data[5] = 0x00; skb->data[6] = 0x80; skb->data[7] = 0xC2; + /* PID 00-07 stands for Ethernet frames without FCS */ skb->data[8] = 0x00; - skb->data[9] = 0x07; /* bridged Ethernet frame w/out FCS */ - break; + skb->data[9] = 0x07; - default: - head_len = 10; - skb_push(skb, head_len); - skb->data[3] = FR_PAD; - skb->data[4] = NLPID_SNAP; - skb->data[5] = FR_PAD; - skb->data[6] = FR_PAD; - skb->data[7] = FR_PAD; - *(__be16*)(skb->data + 8) = skb->protocol; + } else { + return -EINVAL; } dlci_to_q922(skb->data, dlci); @@ -425,8 +431,8 @@ static netdev_tx_t pvc_xmit(struct sk_buff *skb, struct net_device *dev) skb_put(skb, pad); memset(skb->data + len, 0, pad); } - skb->protocol = cpu_to_be16(ETH_P_802_3); } + skb->dev = dev; if (!fr_hard_header(&skb, pvc->dlci)) { dev->stats.tx_bytes += skb->len; dev->stats.tx_packets++; @@ -494,10 +500,8 @@ static void fr_lmi_send(struct net_device *dev, int fullrep) memset(skb->data, 0, len); skb_reserve(skb, 4); if (lmi == LMI_CISCO) { - skb->protocol = cpu_to_be16(NLPID_CISCO_LMI); fr_hard_header(&skb, LMI_CISCO_DLCI); } else { - skb->protocol = cpu_to_be16(NLPID_CCITT_ANSI_LMI); fr_hard_header(&skb, LMI_CCITT_ANSI_DLCI); } data = skb_tail_pointer(skb); diff --git a/drivers/net/wan/lmc/lmc_debug.c b/drivers/net/wan/lmc/lmc_debug.c index f999db788506..2b6051bda3fb 100644 --- a/drivers/net/wan/lmc/lmc_debug.c +++ b/drivers/net/wan/lmc/lmc_debug.c @@ -62,22 +62,4 @@ void lmcEventLog(u32 EventNum, u32 arg2, u32 arg3) } #endif /* DEBUG */ -void lmc_trace(struct net_device *dev, char *msg){ -#ifdef LMC_TRACE - unsigned long j = jiffies + 3; /* Wait for 50 ms */ - - if(in_interrupt()){ - printk("%s: * %s\n", dev->name, msg); -// while(time_before(jiffies, j+10)) -// ; - } - else { - printk("%s: %s\n", dev->name, msg); - while(time_before(jiffies, j)) - schedule(); - } -#endif -} - - /* --------------------------- end if_lmc_linux.c ------------------------ */ diff --git a/drivers/net/wan/lmc/lmc_debug.h b/drivers/net/wan/lmc/lmc_debug.h index 820adcae5d67..cfae9eddf003 100644 --- a/drivers/net/wan/lmc/lmc_debug.h +++ b/drivers/net/wan/lmc/lmc_debug.h @@ -48,6 +48,5 @@ extern u32 lmcEventLogBuf[LMC_EVENTLOGSIZE * LMC_EVENTLOGARGS]; void lmcConsoleLog(char *type, unsigned char *ucData, int iLen); void lmcEventLog(u32 EventNum, u32 arg2, u32 arg3); -void lmc_trace(struct net_device *dev, char *msg); #endif diff --git a/drivers/net/wan/lmc/lmc_main.c b/drivers/net/wan/lmc/lmc_main.c index 842def21e089..36600b0a0ab0 100644 --- a/drivers/net/wan/lmc/lmc_main.c +++ b/drivers/net/wan/lmc/lmc_main.c @@ -113,8 +113,6 @@ int lmc_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) /*fold00*/ u16 regVal; unsigned long flags; - lmc_trace(dev, "lmc_ioctl in"); - /* * Most functions mess with the structure * Disable interrupts while we do the polling @@ -619,8 +617,6 @@ int lmc_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) /*fold00*/ break; } - lmc_trace(dev, "lmc_ioctl out"); - return ret; } @@ -634,8 +630,6 @@ static void lmc_watchdog(struct timer_list *t) /*fold00*/ u32 ticks; unsigned long flags; - lmc_trace(dev, "lmc_watchdog in"); - spin_lock_irqsave(&sc->lmc_lock, flags); if(sc->check != 0xBEAFCAFE){ @@ -782,9 +776,6 @@ kick_timer: add_timer (&sc->timer); spin_unlock_irqrestore(&sc->lmc_lock, flags); - - lmc_trace(dev, "lmc_watchdog out"); - } static int lmc_attach(struct net_device *dev, unsigned short encoding, @@ -813,8 +804,6 @@ static int lmc_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) int err; static int cards_found; - /* lmc_trace(dev, "lmc_init_one in"); */ - err = pcim_enable_device(pdev); if (err) { printk(KERN_ERR "lmc: pci enable failed: %d\n", err); @@ -955,7 +944,6 @@ static int lmc_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) sc->lmc_ok = 0; sc->last_link_status = 0; - lmc_trace(dev, "lmc_init_one out"); return 0; } @@ -981,8 +969,6 @@ static int lmc_open(struct net_device *dev) lmc_softc_t *sc = dev_to_sc(dev); int err; - lmc_trace(dev, "lmc_open in"); - lmc_led_on(sc, LMC_DS3_LED0); lmc_dec_reset(sc); @@ -992,17 +978,14 @@ static int lmc_open(struct net_device *dev) LMC_EVENT_LOG(LMC_EVENT_RESET2, lmc_mii_readreg(sc, 0, 16), lmc_mii_readreg(sc, 0, 17)); - if (sc->lmc_ok){ - lmc_trace(dev, "lmc_open lmc_ok out"); + if (sc->lmc_ok) return 0; - } lmc_softreset (sc); /* Since we have to use PCI bus, this should work on x86,alpha,ppc */ if (request_irq (dev->irq, lmc_interrupt, IRQF_SHARED, dev->name, dev)){ printk(KERN_WARNING "%s: could not get irq: %d\n", dev->name, dev->irq); - lmc_trace(dev, "lmc_open irq failed out"); return -EAGAIN; } sc->got_irq = 1; @@ -1078,8 +1061,6 @@ static int lmc_open(struct net_device *dev) sc->timer.expires = jiffies + HZ; add_timer (&sc->timer); - lmc_trace(dev, "lmc_open out"); - return 0; } @@ -1091,8 +1072,6 @@ static void lmc_running_reset (struct net_device *dev) /*fold00*/ { lmc_softc_t *sc = dev_to_sc(dev); - lmc_trace(dev, "lmc_running_reset in"); - /* stop interrupts */ /* Clear the interrupt mask */ LMC_CSR_WRITE (sc, csr_intr, 0x00000000); @@ -1114,8 +1093,6 @@ static void lmc_running_reset (struct net_device *dev) /*fold00*/ sc->lmc_cmdmode |= (TULIP_CMD_TXRUN | TULIP_CMD_RXRUN); LMC_CSR_WRITE (sc, csr_command, sc->lmc_cmdmode); - - lmc_trace(dev, "lmc_running_reset_out"); } @@ -1128,16 +1105,12 @@ static int lmc_close(struct net_device *dev) /* not calling release_region() as we should */ lmc_softc_t *sc = dev_to_sc(dev); - lmc_trace(dev, "lmc_close in"); - sc->lmc_ok = 0; sc->lmc_media->set_link_status (sc, 0); del_timer (&sc->timer); lmc_proto_close(sc); lmc_ifdown (dev); - lmc_trace(dev, "lmc_close out"); - return 0; } @@ -1149,8 +1122,6 @@ static int lmc_ifdown (struct net_device *dev) /*fold00*/ u32 csr6; int i; - lmc_trace(dev, "lmc_ifdown in"); - /* Don't let anything else go on right now */ // dev->start = 0; netif_stop_queue(dev); @@ -1200,8 +1171,6 @@ static int lmc_ifdown (struct net_device *dev) /*fold00*/ netif_wake_queue(dev); sc->extra_stats.tx_tbusy0++; - lmc_trace(dev, "lmc_ifdown out"); - return 0; } @@ -1220,8 +1189,6 @@ static irqreturn_t lmc_interrupt (int irq, void *dev_instance) /*fold00*/ int max_work = LMC_RXDESCS; int handled = 0; - lmc_trace(dev, "lmc_interrupt in"); - spin_lock(&sc->lmc_lock); /* @@ -1264,12 +1231,10 @@ static irqreturn_t lmc_interrupt (int irq, void *dev_instance) /*fold00*/ lmc_running_reset (dev); break; } - - if (csr & TULIP_STS_RXINTR){ - lmc_trace(dev, "rx interrupt"); + + if (csr & TULIP_STS_RXINTR) lmc_rx (dev); - - } + if (csr & (TULIP_STS_TXINTR | TULIP_STS_TXNOBUF | TULIP_STS_TXSTOPPED)) { int n_compl = 0 ; @@ -1389,7 +1354,6 @@ lmc_int_fail_out: spin_unlock(&sc->lmc_lock); - lmc_trace(dev, "lmc_interrupt out"); return IRQ_RETVAL(handled); } @@ -1401,8 +1365,6 @@ static netdev_tx_t lmc_start_xmit(struct sk_buff *skb, int entry; unsigned long flags; - lmc_trace(dev, "lmc_start_xmit in"); - spin_lock_irqsave(&sc->lmc_lock, flags); /* normal path, tbusy known to be zero */ @@ -1477,7 +1439,6 @@ static netdev_tx_t lmc_start_xmit(struct sk_buff *skb, spin_unlock_irqrestore(&sc->lmc_lock, flags); - lmc_trace(dev, "lmc_start_xmit_out"); return NETDEV_TX_OK; } @@ -1493,8 +1454,6 @@ static int lmc_rx(struct net_device *dev) struct sk_buff *skb, *nsb; u16 len; - lmc_trace(dev, "lmc_rx in"); - lmc_led_on(sc, LMC_DS3_LED3); rxIntLoopCnt = 0; /* debug -baz */ @@ -1673,9 +1632,6 @@ static int lmc_rx(struct net_device *dev) lmc_led_off(sc, LMC_DS3_LED3); skip_out_of_mem: - - lmc_trace(dev, "lmc_rx out"); - return 0; } @@ -1684,16 +1640,12 @@ static struct net_device_stats *lmc_get_stats(struct net_device *dev) lmc_softc_t *sc = dev_to_sc(dev); unsigned long flags; - lmc_trace(dev, "lmc_get_stats in"); - spin_lock_irqsave(&sc->lmc_lock, flags); sc->lmc_device->stats.rx_missed_errors += LMC_CSR_READ(sc, csr_missed_frames) & 0xffff; spin_unlock_irqrestore(&sc->lmc_lock, flags); - lmc_trace(dev, "lmc_get_stats out"); - return &sc->lmc_device->stats; } @@ -1712,12 +1664,8 @@ unsigned lmc_mii_readreg (lmc_softc_t * const sc, unsigned devaddr, unsigned reg int command = (0xf6 << 10) | (devaddr << 5) | regno; int retval = 0; - lmc_trace(sc->lmc_device, "lmc_mii_readreg in"); - LMC_MII_SYNC (sc); - lmc_trace(sc->lmc_device, "lmc_mii_readreg: done sync"); - for (i = 15; i >= 0; i--) { int dataval = (command & (1 << i)) ? 0x20000 : 0; @@ -1730,8 +1678,6 @@ unsigned lmc_mii_readreg (lmc_softc_t * const sc, unsigned devaddr, unsigned reg /* __SLOW_DOWN_IO; */ } - lmc_trace(sc->lmc_device, "lmc_mii_readreg: done1"); - for (i = 19; i > 0; i--) { LMC_CSR_WRITE (sc, csr_9, 0x40000); @@ -1743,8 +1689,6 @@ unsigned lmc_mii_readreg (lmc_softc_t * const sc, unsigned devaddr, unsigned reg /* __SLOW_DOWN_IO; */ } - lmc_trace(sc->lmc_device, "lmc_mii_readreg out"); - return (retval >> 1) & 0xffff; } @@ -1753,8 +1697,6 @@ void lmc_mii_writereg (lmc_softc_t * const sc, unsigned devaddr, unsigned regno, int i = 32; int command = (0x5002 << 16) | (devaddr << 23) | (regno << 18) | data; - lmc_trace(sc->lmc_device, "lmc_mii_writereg in"); - LMC_MII_SYNC (sc); i = 31; @@ -1787,16 +1729,12 @@ void lmc_mii_writereg (lmc_softc_t * const sc, unsigned devaddr, unsigned regno, /* __SLOW_DOWN_IO; */ i--; } - - lmc_trace(sc->lmc_device, "lmc_mii_writereg out"); } static void lmc_softreset (lmc_softc_t * const sc) /*fold00*/ { int i; - lmc_trace(sc->lmc_device, "lmc_softreset in"); - /* Initialize the receive rings and buffers. */ sc->lmc_txfull = 0; sc->lmc_next_rx = 0; @@ -1871,55 +1809,40 @@ static void lmc_softreset (lmc_softc_t * const sc) /*fold00*/ } sc->lmc_txring[i - 1].buffer2 = virt_to_bus (&sc->lmc_txring[0]); LMC_CSR_WRITE (sc, csr_txlist, virt_to_bus (sc->lmc_txring)); - - lmc_trace(sc->lmc_device, "lmc_softreset out"); } void lmc_gpio_mkinput(lmc_softc_t * const sc, u32 bits) /*fold00*/ { - lmc_trace(sc->lmc_device, "lmc_gpio_mkinput in"); sc->lmc_gpio_io &= ~bits; LMC_CSR_WRITE(sc, csr_gp, TULIP_GP_PINSET | (sc->lmc_gpio_io)); - lmc_trace(sc->lmc_device, "lmc_gpio_mkinput out"); } void lmc_gpio_mkoutput(lmc_softc_t * const sc, u32 bits) /*fold00*/ { - lmc_trace(sc->lmc_device, "lmc_gpio_mkoutput in"); sc->lmc_gpio_io |= bits; LMC_CSR_WRITE(sc, csr_gp, TULIP_GP_PINSET | (sc->lmc_gpio_io)); - lmc_trace(sc->lmc_device, "lmc_gpio_mkoutput out"); } void lmc_led_on(lmc_softc_t * const sc, u32 led) /*fold00*/ { - lmc_trace(sc->lmc_device, "lmc_led_on in"); - if((~sc->lmc_miireg16) & led){ /* Already on! */ - lmc_trace(sc->lmc_device, "lmc_led_on aon out"); + if ((~sc->lmc_miireg16) & led) /* Already on! */ return; - } - + sc->lmc_miireg16 &= ~led; lmc_mii_writereg(sc, 0, 16, sc->lmc_miireg16); - lmc_trace(sc->lmc_device, "lmc_led_on out"); } void lmc_led_off(lmc_softc_t * const sc, u32 led) /*fold00*/ { - lmc_trace(sc->lmc_device, "lmc_led_off in"); - if(sc->lmc_miireg16 & led){ /* Already set don't do anything */ - lmc_trace(sc->lmc_device, "lmc_led_off aoff out"); + if (sc->lmc_miireg16 & led) /* Already set don't do anything */ return; - } - + sc->lmc_miireg16 |= led; lmc_mii_writereg(sc, 0, 16, sc->lmc_miireg16); - lmc_trace(sc->lmc_device, "lmc_led_off out"); } static void lmc_reset(lmc_softc_t * const sc) /*fold00*/ { - lmc_trace(sc->lmc_device, "lmc_reset in"); sc->lmc_miireg16 |= LMC_MII16_FIFO_RESET; lmc_mii_writereg(sc, 0, 16, sc->lmc_miireg16); @@ -1955,13 +1878,11 @@ static void lmc_reset(lmc_softc_t * const sc) /*fold00*/ sc->lmc_media->init(sc); sc->extra_stats.resetCount++; - lmc_trace(sc->lmc_device, "lmc_reset out"); } static void lmc_dec_reset(lmc_softc_t * const sc) /*fold00*/ { u32 val; - lmc_trace(sc->lmc_device, "lmc_dec_reset in"); /* * disable all interrupts @@ -2017,14 +1938,11 @@ static void lmc_dec_reset(lmc_softc_t * const sc) /*fold00*/ val = LMC_CSR_READ(sc, csr_sia_general); val |= (TULIP_WATCHDOG_TXDISABLE | TULIP_WATCHDOG_RXDISABLE); LMC_CSR_WRITE(sc, csr_sia_general, val); - - lmc_trace(sc->lmc_device, "lmc_dec_reset out"); } static void lmc_initcsrs(lmc_softc_t * const sc, lmc_csrptr_t csr_base, /*fold00*/ size_t csr_size) { - lmc_trace(sc->lmc_device, "lmc_initcsrs in"); sc->lmc_csrs.csr_busmode = csr_base + 0 * csr_size; sc->lmc_csrs.csr_txpoll = csr_base + 1 * csr_size; sc->lmc_csrs.csr_rxpoll = csr_base + 2 * csr_size; @@ -2041,7 +1959,6 @@ static void lmc_initcsrs(lmc_softc_t * const sc, lmc_csrptr_t csr_base, /*fold00 sc->lmc_csrs.csr_13 = csr_base + 13 * csr_size; sc->lmc_csrs.csr_14 = csr_base + 14 * csr_size; sc->lmc_csrs.csr_15 = csr_base + 15 * csr_size; - lmc_trace(sc->lmc_device, "lmc_initcsrs out"); } static void lmc_driver_timeout(struct net_device *dev, unsigned int txqueue) @@ -2050,8 +1967,6 @@ static void lmc_driver_timeout(struct net_device *dev, unsigned int txqueue) u32 csr6; unsigned long flags; - lmc_trace(dev, "lmc_driver_timeout in"); - spin_lock_irqsave(&sc->lmc_lock, flags); printk("%s: Xmitter busy|\n", dev->name); @@ -2094,8 +2009,4 @@ static void lmc_driver_timeout(struct net_device *dev, unsigned int txqueue) bug_out: spin_unlock_irqrestore(&sc->lmc_lock, flags); - - lmc_trace(dev, "lmc_driver_timeout out"); - - } diff --git a/drivers/net/wan/lmc/lmc_media.c b/drivers/net/wan/lmc/lmc_media.c index 23ca4a62d6f5..ec1ac7b1f3fd 100644 --- a/drivers/net/wan/lmc/lmc_media.c +++ b/drivers/net/wan/lmc/lmc_media.c @@ -1026,7 +1026,6 @@ lmc_t1_get_link_status (lmc_softc_t * const sc) * led3 red = Loss of Signal (LOS) or out of frame (OOF) * conditions detected on T3 receive signal */ - lmc_trace(sc->lmc_device, "lmc_t1_get_link_status in"); lmc_led_on(sc, LMC_DS3_LED2); lmc_mii_writereg (sc, 0, 17, T1FRAMER_ALARM1_STATUS); @@ -1120,9 +1119,6 @@ lmc_t1_get_link_status (lmc_softc_t * const sc) lmc_mii_writereg (sc, 0, 17, T1FRAMER_ALARM2_STATUS); sc->lmc_xinfo.t1_alarm2_status = lmc_mii_readreg (sc, 0, 18); - - lmc_trace(sc->lmc_device, "lmc_t1_get_link_status out"); - return ret; } diff --git a/drivers/net/wan/lmc/lmc_proto.c b/drivers/net/wan/lmc/lmc_proto.c index a58301dd0c1f..e8b0b902b424 100644 --- a/drivers/net/wan/lmc/lmc_proto.c +++ b/drivers/net/wan/lmc/lmc_proto.c @@ -47,7 +47,6 @@ // attach void lmc_proto_attach(lmc_softc_t *sc) /*FOLD00*/ { - lmc_trace(sc->lmc_device, "lmc_proto_attach in"); if (sc->if_type == LMC_NET) { struct net_device *dev = sc->lmc_device; /* @@ -57,12 +56,10 @@ void lmc_proto_attach(lmc_softc_t *sc) /*FOLD00*/ dev->hard_header_len = 0; dev->addr_len = 0; } - lmc_trace(sc->lmc_device, "lmc_proto_attach out"); } int lmc_proto_ioctl(lmc_softc_t *sc, struct ifreq *ifr, int cmd) { - lmc_trace(sc->lmc_device, "lmc_proto_ioctl"); if (sc->if_type == LMC_PPP) return hdlc_ioctl(sc->lmc_device, ifr, cmd); return -EOPNOTSUPP; @@ -72,32 +69,23 @@ int lmc_proto_open(lmc_softc_t *sc) { int ret = 0; - lmc_trace(sc->lmc_device, "lmc_proto_open in"); - if (sc->if_type == LMC_PPP) { ret = hdlc_open(sc->lmc_device); if (ret < 0) printk(KERN_WARNING "%s: HDLC open failed: %d\n", sc->name, ret); } - - lmc_trace(sc->lmc_device, "lmc_proto_open out"); return ret; } void lmc_proto_close(lmc_softc_t *sc) { - lmc_trace(sc->lmc_device, "lmc_proto_close in"); - if (sc->if_type == LMC_PPP) hdlc_close(sc->lmc_device); - - lmc_trace(sc->lmc_device, "lmc_proto_close out"); } __be16 lmc_proto_type(lmc_softc_t *sc, struct sk_buff *skb) /*FOLD00*/ { - lmc_trace(sc->lmc_device, "lmc_proto_type in"); switch(sc->if_type){ case LMC_PPP: return hdlc_type_trans(skb, sc->lmc_device); @@ -113,13 +101,10 @@ __be16 lmc_proto_type(lmc_softc_t *sc, struct sk_buff *skb) /*FOLD00*/ return htons(ETH_P_802_2); break; } - lmc_trace(sc->lmc_device, "lmc_proto_tye out"); - } void lmc_proto_netif(lmc_softc_t *sc, struct sk_buff *skb) /*FOLD00*/ { - lmc_trace(sc->lmc_device, "lmc_proto_netif in"); switch(sc->if_type){ case LMC_PPP: case LMC_NET: @@ -129,5 +114,4 @@ void lmc_proto_netif(lmc_softc_t *sc, struct sk_buff *skb) /*FOLD00*/ case LMC_RAW: break; } - lmc_trace(sc->lmc_device, "lmc_proto_netif out"); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c index 2c95a08a5871..3984fd7d918e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c @@ -397,9 +397,9 @@ brcmf_proto_bcdc_add_tdls_peer(struct brcmf_pub *drvr, int ifidx, } static void brcmf_proto_bcdc_rxreorder(struct brcmf_if *ifp, - struct sk_buff *skb) + struct sk_buff *skb, bool inirq) { - brcmf_fws_rxreorder(ifp, skb); + brcmf_fws_rxreorder(ifp, skb, inirq); } static void diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 318bd00bf94f..f9ebb98b0e3c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -74,7 +74,7 @@ static irqreturn_t brcmf_sdiod_oob_irqhandler(int irq, void *dev_id) sdiodev->irq_en = false; } - brcmf_sdio_isr(sdiodev->bus); + brcmf_sdio_isr(sdiodev->bus, true); return IRQ_HANDLED; } @@ -86,7 +86,7 @@ static void brcmf_sdiod_ib_irqhandler(struct sdio_func *func) brcmf_dbg(INTR, "IB intr triggered\n"); - brcmf_sdio_isr(sdiodev->bus); + brcmf_sdio_isr(sdiodev->bus, false); } /* dummy handler for SDIO function 2 interrupt */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index 623c0168da79..08f9d47f2e5c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -249,7 +249,8 @@ int brcmf_bus_reset(struct brcmf_bus *bus) */ /* Receive frame for delivery to OS. Callee disposes of rxp. */ -void brcmf_rx_frame(struct device *dev, struct sk_buff *rxp, bool handle_event); +void brcmf_rx_frame(struct device *dev, struct sk_buff *rxp, bool handle_event, + bool inirq); /* Receive async event packet from firmware. Callee disposes of rxp. */ void brcmf_rx_event(struct device *dev, struct sk_buff *rxp); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index b0465f81f9e6..3dd28f5fef19 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -395,7 +395,7 @@ void brcmf_txflowblock_if(struct brcmf_if *ifp, spin_unlock_irqrestore(&ifp->netif_stop_lock, flags); } -void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb) +void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb, bool inirq) { /* Most of Broadcom's firmwares send 802.11f ADD frame every time a new * STA connects to the AP interface. This is an obsoleted standard most @@ -418,14 +418,15 @@ void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb) ifp->ndev->stats.rx_packets++; brcmf_dbg(DATA, "rx proto=0x%X\n", ntohs(skb->protocol)); - if (in_interrupt()) + if (inirq) { netif_rx(skb); - else + } else { /* If the receive is not processed inside an ISR, * the softirqd must be woken explicitly to service * the NET_RX_SOFTIRQ. This is handled by netif_rx_ni(). */ netif_rx_ni(skb); + } } void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb) @@ -474,7 +475,7 @@ void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb) skb->pkt_type = PACKET_OTHERHOST; skb->protocol = htons(ETH_P_802_2); - brcmf_netif_rx(ifp, skb); + brcmf_netif_rx(ifp, skb, false); } static int brcmf_rx_hdrpull(struct brcmf_pub *drvr, struct sk_buff *skb, @@ -496,7 +497,8 @@ static int brcmf_rx_hdrpull(struct brcmf_pub *drvr, struct sk_buff *skb, return 0; } -void brcmf_rx_frame(struct device *dev, struct sk_buff *skb, bool handle_event) +void brcmf_rx_frame(struct device *dev, struct sk_buff *skb, bool handle_event, + bool inirq) { struct brcmf_if *ifp; struct brcmf_bus *bus_if = dev_get_drvdata(dev); @@ -508,14 +510,16 @@ void brcmf_rx_frame(struct device *dev, struct sk_buff *skb, bool handle_event) return; if (brcmf_proto_is_reorder_skb(skb)) { - brcmf_proto_rxreorder(ifp, skb); + brcmf_proto_rxreorder(ifp, skb, inirq); } else { /* Process special event packets */ - if (handle_event) - brcmf_fweh_process_skb(ifp->drvr, skb, - BCMILCP_SUBTYPE_VENDOR_LONG); + if (handle_event) { + gfp_t gfp = inirq ? GFP_ATOMIC : GFP_KERNEL; - brcmf_netif_rx(ifp, skb); + brcmf_fweh_process_skb(ifp->drvr, skb, + BCMILCP_SUBTYPE_VENDOR_LONG, gfp); + } + brcmf_netif_rx(ifp, skb, inirq); } } @@ -530,7 +534,7 @@ void brcmf_rx_event(struct device *dev, struct sk_buff *skb) if (brcmf_rx_hdrpull(drvr, skb, &ifp)) return; - brcmf_fweh_process_skb(ifp->drvr, skb, 0); + brcmf_fweh_process_skb(ifp->drvr, skb, 0, GFP_KERNEL); brcmu_pkt_buf_free_skb(skb); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 33b2ab3b54b0..5767d665cee5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -208,7 +208,7 @@ void brcmf_remove_interface(struct brcmf_if *ifp, bool rtnl_locked); void brcmf_txflowblock_if(struct brcmf_if *ifp, enum brcmf_netif_stop_reason reason, bool state); void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success); -void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb); +void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb, bool inirq); void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_net_detach(struct net_device *ndev, bool rtnl_locked); int brcmf_net_mon_attach(struct brcmf_if *ifp); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index baa98a202892..430d2cca98b3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -390,12 +390,11 @@ int brcmf_fweh_activate_events(struct brcmf_if *ifp) */ void brcmf_fweh_process_event(struct brcmf_pub *drvr, struct brcmf_event *event_packet, - u32 packet_len) + u32 packet_len, gfp_t gfp) { enum brcmf_fweh_event_code code; struct brcmf_fweh_info *fweh = &drvr->fweh; struct brcmf_fweh_queue_item *event; - gfp_t alloc_flag = GFP_KERNEL; void *data; u32 datalen; @@ -414,10 +413,7 @@ void brcmf_fweh_process_event(struct brcmf_pub *drvr, datalen + sizeof(*event_packet) > packet_len) return; - if (in_interrupt()) - alloc_flag = GFP_ATOMIC; - - event = kzalloc(sizeof(*event) + datalen, alloc_flag); + event = kzalloc(sizeof(*event) + datalen, gfp); if (!event) return; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index a82f51bc1e69..48414e8b9389 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -319,11 +319,12 @@ void brcmf_fweh_unregister(struct brcmf_pub *drvr, int brcmf_fweh_activate_events(struct brcmf_if *ifp); void brcmf_fweh_process_event(struct brcmf_pub *drvr, struct brcmf_event *event_packet, - u32 packet_len); + u32 packet_len, gfp_t gfp); void brcmf_fweh_p2pdev_setup(struct brcmf_if *ifp, bool ongoing); static inline void brcmf_fweh_process_skb(struct brcmf_pub *drvr, - struct sk_buff *skb, u16 stype) + struct sk_buff *skb, u16 stype, + gfp_t gfp) { struct brcmf_event *event_packet; u16 subtype, usr_stype; @@ -354,7 +355,7 @@ static inline void brcmf_fweh_process_skb(struct brcmf_pub *drvr, if (usr_stype != BCMILCP_BCM_SUBTYPE_EVENT) return; - brcmf_fweh_process_event(drvr, event_packet, skb->len + ETH_HLEN); + brcmf_fweh_process_event(drvr, event_packet, skb->len + ETH_HLEN, gfp); } #endif /* FWEH_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 775e0612fa17..437e83ea8902 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -1664,7 +1664,7 @@ static void brcmf_rxreorder_get_skb_list(struct brcmf_ampdu_rx_reorder *rfi, rfi->pend_pkts -= skb_queue_len(skb_list); } -void brcmf_fws_rxreorder(struct brcmf_if *ifp, struct sk_buff *pkt) +void brcmf_fws_rxreorder(struct brcmf_if *ifp, struct sk_buff *pkt, bool inirq) { struct brcmf_pub *drvr = ifp->drvr; u8 *reorder_data; @@ -1682,7 +1682,7 @@ void brcmf_fws_rxreorder(struct brcmf_if *ifp, struct sk_buff *pkt) /* validate flags and flow id */ if (flags == 0xFF) { bphy_err(drvr, "invalid flags...so ignore this packet\n"); - brcmf_netif_rx(ifp, pkt); + brcmf_netif_rx(ifp, pkt, inirq); return; } @@ -1694,7 +1694,7 @@ void brcmf_fws_rxreorder(struct brcmf_if *ifp, struct sk_buff *pkt) if (rfi == NULL) { brcmf_dbg(INFO, "received flags to cleanup, but no flow (%d) yet\n", flow_id); - brcmf_netif_rx(ifp, pkt); + brcmf_netif_rx(ifp, pkt, inirq); return; } @@ -1719,7 +1719,7 @@ void brcmf_fws_rxreorder(struct brcmf_if *ifp, struct sk_buff *pkt) rfi = kzalloc(buf_size, GFP_ATOMIC); if (rfi == NULL) { bphy_err(drvr, "failed to alloc buffer\n"); - brcmf_netif_rx(ifp, pkt); + brcmf_netif_rx(ifp, pkt, inirq); return; } @@ -1833,7 +1833,7 @@ void brcmf_fws_rxreorder(struct brcmf_if *ifp, struct sk_buff *pkt) netif_rx: skb_queue_walk_safe(&reorder_list, pkt, pnext) { __skb_unlink(pkt, &reorder_list); - brcmf_netif_rx(ifp, pkt); + brcmf_netif_rx(ifp, pkt, inirq); } } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h index b16a9d1c0508..50e424b5880d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h @@ -42,6 +42,6 @@ void brcmf_fws_add_interface(struct brcmf_if *ifp); void brcmf_fws_del_interface(struct brcmf_if *ifp); void brcmf_fws_bustxfail(struct brcmf_fws_info *fws, struct sk_buff *skb); void brcmf_fws_bus_blocked(struct brcmf_pub *drvr, bool flow_blocked); -void brcmf_fws_rxreorder(struct brcmf_if *ifp, struct sk_buff *skb); +void brcmf_fws_rxreorder(struct brcmf_if *ifp, struct sk_buff *skb, bool inirq); #endif /* FWSIGNAL_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index bfddb851e386..7c8e08ee8f0f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -536,7 +536,8 @@ static int brcmf_msgbuf_hdrpull(struct brcmf_pub *drvr, bool do_fws, return -ENODEV; } -static void brcmf_msgbuf_rxreorder(struct brcmf_if *ifp, struct sk_buff *skb) +static void brcmf_msgbuf_rxreorder(struct brcmf_if *ifp, struct sk_buff *skb, + bool inirq) { } @@ -1128,7 +1129,7 @@ static void brcmf_msgbuf_process_event(struct brcmf_msgbuf *msgbuf, void *buf) skb->protocol = eth_type_trans(skb, ifp->ndev); - brcmf_fweh_process_skb(ifp->drvr, skb, 0); + brcmf_fweh_process_skb(ifp->drvr, skb, 0, GFP_KERNEL); exit: brcmu_pkt_buf_free_skb(skb); @@ -1190,7 +1191,7 @@ brcmf_msgbuf_process_rx_complete(struct brcmf_msgbuf *msgbuf, void *buf) } skb->protocol = eth_type_trans(skb, ifp->ndev); - brcmf_netif_rx(ifp, skb); + brcmf_netif_rx(ifp, skb, false); } static void brcmf_msgbuf_process_gen_status(struct brcmf_msgbuf *msgbuf, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h index bd08d3aaa8f4..f4a79e217da5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h @@ -32,7 +32,7 @@ struct brcmf_proto { u8 peer[ETH_ALEN]); void (*add_tdls_peer)(struct brcmf_pub *drvr, int ifidx, u8 peer[ETH_ALEN]); - void (*rxreorder)(struct brcmf_if *ifp, struct sk_buff *skb); + void (*rxreorder)(struct brcmf_if *ifp, struct sk_buff *skb, bool inirq); void (*add_if)(struct brcmf_if *ifp); void (*del_if)(struct brcmf_if *ifp); void (*reset_if)(struct brcmf_if *ifp); @@ -109,9 +109,9 @@ static inline bool brcmf_proto_is_reorder_skb(struct sk_buff *skb) } static inline void -brcmf_proto_rxreorder(struct brcmf_if *ifp, struct sk_buff *skb) +brcmf_proto_rxreorder(struct brcmf_if *ifp, struct sk_buff *skb, bool inirq) { - ifp->drvr->proto->rxreorder(ifp, skb); + ifp->drvr->proto->rxreorder(ifp, skb, inirq); } static inline void diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index d4989e0cd7be..99987a789e7e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -1704,7 +1704,7 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq) brcmf_rx_event(bus->sdiodev->dev, pfirst); else brcmf_rx_frame(bus->sdiodev->dev, pfirst, - false); + false, false); bus->sdcnt.rxglompkts++; } @@ -2038,7 +2038,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) brcmf_rx_event(bus->sdiodev->dev, pkt); else brcmf_rx_frame(bus->sdiodev->dev, pkt, - false); + false, false); /* prepare the descriptor for the next read */ rd->len = rd->len_nxtfrm << 4; @@ -3625,7 +3625,7 @@ void brcmf_sdio_trigger_dpc(struct brcmf_sdio *bus) } } -void brcmf_sdio_isr(struct brcmf_sdio *bus) +void brcmf_sdio_isr(struct brcmf_sdio *bus, bool in_isr) { brcmf_dbg(TRACE, "Enter\n"); @@ -3636,7 +3636,7 @@ void brcmf_sdio_isr(struct brcmf_sdio *bus) /* Count the interrupt call */ bus->sdcnt.intrcount++; - if (in_interrupt()) + if (in_isr) atomic_set(&bus->ipend, 1); else if (brcmf_sdio_intr_rstatus(bus)) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index 12108927fb50..15d2c02fa3ec 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -372,7 +372,7 @@ int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev); struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); void brcmf_sdio_remove(struct brcmf_sdio *bus); -void brcmf_sdio_isr(struct brcmf_sdio *bus); +void brcmf_sdio_isr(struct brcmf_sdio *bus, bool in_isr); void brcmf_sdio_wd_timer(struct brcmf_sdio *bus, bool active); void brcmf_sdio_wowl_config(struct device *dev, bool enabled); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 6f67fefe4b58..586f4dfc638b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -532,7 +532,7 @@ static void brcmf_usb_rx_complete(struct urb *urb) if (devinfo->bus_pub.state == BRCMFMAC_USB_STATE_UP || devinfo->bus_pub.state == BRCMFMAC_USB_STATE_SLEEP) { skb_put(skb, urb->actual_length); - brcmf_rx_frame(devinfo->dev, skb, true); + brcmf_rx_frame(devinfo->dev, skb, true, true); brcmf_usb_rx_refill(devinfo, req); usb_mark_last_busy(urb->dev); } else { diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c index b7fbfc77b612..23fbddd0c1f8 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c +++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c @@ -201,8 +201,7 @@ static u32 ipw2100_debug_level = IPW_DL_NONE; #define IPW_DEBUG(level, message...) \ do { \ if (ipw2100_debug_level & (level)) { \ - printk(KERN_DEBUG "ipw2100: %c %s ", \ - in_interrupt() ? 'I' : 'U', __func__); \ + printk(KERN_DEBUG "ipw2100: %s ", __func__); \ printk(message); \ } \ } while (0) diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.h b/drivers/net/wireless/intel/ipw2x00/ipw2200.h index e1ec1c96dcd8..98fe62737888 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2200.h +++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.h @@ -1382,14 +1382,12 @@ BIT_ARG16(x) #define IPW_DEBUG(level, fmt, args...) \ do { if (ipw_debug_level & (level)) \ - printk(KERN_DEBUG DRV_NAME": %c %s " fmt, \ - in_interrupt() ? 'I' : 'U', __func__ , ## args); } while (0) + printk(KERN_DEBUG DRV_NAME": %s " fmt, __func__ , ## args); } while (0) #ifdef CONFIG_IPW2200_DEBUG #define IPW_LL_DEBUG(level, fmt, args...) \ do { if (ipw_debug_level & (level)) \ - printk(KERN_DEBUG DRV_NAME": %c %s " fmt, \ - in_interrupt() ? 'I' : 'U', __func__ , ## args); } while (0) + printk(KERN_DEBUG DRV_NAME": %s " fmt, __func__ , ## args); } while (0) #else #define IPW_LL_DEBUG(level, fmt, args...) do {} while (0) #endif /* CONFIG_IPW2200_DEBUG */ diff --git a/drivers/net/wireless/intel/ipw2x00/libipw.h b/drivers/net/wireless/intel/ipw2x00/libipw.h index e87538a8b88b..7964ef7d15f0 100644 --- a/drivers/net/wireless/intel/ipw2x00/libipw.h +++ b/drivers/net/wireless/intel/ipw2x00/libipw.h @@ -60,8 +60,7 @@ extern u32 libipw_debug_level; #define LIBIPW_DEBUG(level, fmt, args...) \ do { if (libipw_debug_level & (level)) \ - printk(KERN_DEBUG "libipw: %c %s " fmt, \ - in_interrupt() ? 'I' : 'U', __func__ , ## args); } while (0) + printk(KERN_DEBUG "libipw: %s " fmt, __func__ , ## args); } while (0) #else #define LIBIPW_DEBUG(level, fmt, args...) do {} while (0) #endif /* CONFIG_LIBIPW_DEBUG */ diff --git a/drivers/net/wireless/intel/iwlegacy/common.h b/drivers/net/wireless/intel/iwlegacy/common.h index bc9cd7e5ccb8..ea1b1bb7ddcb 100644 --- a/drivers/net/wireless/intel/iwlegacy/common.h +++ b/drivers/net/wireless/intel/iwlegacy/common.h @@ -2925,8 +2925,8 @@ do { \ #define IL_DBG(level, fmt, args...) \ do { \ if (il_get_debug_level(il) & level) \ - dev_err(&il->hw->wiphy->dev, "%c %s " fmt, \ - in_interrupt() ? 'I' : 'U', __func__ , ##args); \ + dev_err(&il->hw->wiphy->dev, "%s " fmt, __func__, \ + ##args); \ } while (0) #define il_print_hex_dump(il, level, p, len) \ diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-debug.c b/drivers/net/wireless/intel/iwlwifi/iwl-debug.c index e1a41fd503a8..7df173cc9ddc 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-debug.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-debug.c @@ -121,10 +121,9 @@ void __iwl_dbg(struct device *dev, #ifdef CONFIG_IWLWIFI_DEBUG if (iwl_have_debug_level(level) && (!limit || net_ratelimit())) - dev_printk(KERN_DEBUG, dev, "%c %s %pV", - in_interrupt() ? 'I' : 'U', function, &vaf); + dev_printk(KERN_DEBUG, dev, "%s %pV", function, &vaf); #endif - trace_iwlwifi_dbg(level, in_interrupt(), function, &vaf); + trace_iwlwifi_dbg(level, function, &vaf); va_end(args); } IWL_EXPORT_SYMBOL(__iwl_dbg); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-msg.h b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-msg.h index 9ad93ef60890..d0467da5af03 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-msg.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-msg.h @@ -54,18 +54,16 @@ DEFINE_EVENT(iwlwifi_msg_event, iwlwifi_crit, ); TRACE_EVENT(iwlwifi_dbg, - TP_PROTO(u32 level, bool in_interrupt, const char *function, + TP_PROTO(u32 level, const char *function, struct va_format *vaf), - TP_ARGS(level, in_interrupt, function, vaf), + TP_ARGS(level, function, vaf), TP_STRUCT__entry( __field(u32, level) - __field(u8, in_interrupt) __string(function, function) __dynamic_array(char, msg, MAX_MSG_LEN) ), TP_fast_assign( __entry->level = level; - __entry->in_interrupt = in_interrupt; __assign_str(function, function); WARN_ON_ONCE(vsnprintf(__get_dynamic_array(msg), MAX_MSG_LEN, vaf->fmt, diff --git a/drivers/net/wireless/intersil/hostap/hostap_hw.c b/drivers/net/wireless/intersil/hostap/hostap_hw.c index 01a2dd0edd71..22cfb6452644 100644 --- a/drivers/net/wireless/intersil/hostap/hostap_hw.c +++ b/drivers/net/wireless/intersil/hostap/hostap_hw.c @@ -320,12 +320,6 @@ static int hfa384x_cmd(struct net_device *dev, u16 cmd, u16 param0, iface = netdev_priv(dev); local = iface->local; - if (in_interrupt()) { - printk(KERN_DEBUG "%s: hfa384x_cmd called from interrupt " - "context\n", dev->name); - return -1; - } - if (local->cmd_queue_len >= HOSTAP_CMD_QUEUE_MAX_LEN) { printk(KERN_DEBUG "%s: hfa384x_cmd: cmd_queue full\n", dev->name); @@ -1560,12 +1554,6 @@ static void prism2_hw_reset(struct net_device *dev) iface = netdev_priv(dev); local = iface->local; - if (in_interrupt()) { - printk(KERN_DEBUG "%s: driver bug - prism2_hw_reset() called " - "in interrupt context\n", dev->name); - return; - } - if (local->hw_downloading) return; diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c index dce3bc9c9f84..f8d15abd079b 100644 --- a/drivers/net/wireless/mac80211_hwsim.c +++ b/drivers/net/wireless/mac80211_hwsim.c @@ -99,7 +99,7 @@ MODULE_PARM_DESC(support_p2p_device, "Support P2P-Device interface type"); * domain requests. The first radio will adhere to the first custom world * regulatory domain, the second one to the second custom world regulatory * domain. All other devices will world roam. - * @HWSIM_REGTEST_STRICT_FOLLOW_: Used for testing strict regulatory domain + * @HWSIM_REGTEST_STRICT_FOLLOW: Used for testing strict regulatory domain * settings, only the first radio will send a regulatory domain request * and use strict settings. The rest of the radios are expected to follow. * @HWSIM_REGTEST_STRICT_ALL: Used for testing strict regulatory domain @@ -377,6 +377,49 @@ static const struct ieee80211_channel hwsim_channels_5ghz[] = { CHAN5G(5925), /* Channel 185 */ }; +#define NUM_S1G_CHANS_US 51 +static struct ieee80211_channel hwsim_channels_s1g[NUM_S1G_CHANS_US]; + +static const struct ieee80211_sta_s1g_cap hwsim_s1g_cap = { + .s1g = true, + .cap = { S1G_CAP0_SGI_1MHZ | S1G_CAP0_SGI_2MHZ, + 0, + 0, + S1G_CAP3_MAX_MPDU_LEN, + 0, + S1G_CAP5_AMPDU, + 0, + S1G_CAP7_DUP_1MHZ, + S1G_CAP8_TWT_RESPOND | S1G_CAP8_TWT_REQUEST, + 0}, + .nss_mcs = { 0xfc | 1, /* MCS 7 for 1 SS */ + /* RX Highest Supported Long GI Data Rate 0:7 */ + 0, + /* RX Highest Supported Long GI Data Rate 0:7 */ + /* TX S1G MCS Map 0:6 */ + 0xfa, + /* TX S1G MCS Map :7 */ + /* TX Highest Supported Long GI Data Rate 0:6 */ + 0x80, + /* TX Highest Supported Long GI Data Rate 7:8 */ + /* Rx Single spatial stream and S1G-MCS Map for 1MHz */ + /* Tx Single spatial stream and S1G-MCS Map for 1MHz */ + 0 }, +}; + +static void hwsim_init_s1g_channels(struct ieee80211_channel *channels) +{ + int ch, freq; + + for (ch = 0; ch < NUM_S1G_CHANS_US; ch++) { + freq = 902000 + (ch + 1) * 500; + channels[ch].band = NL80211_BAND_S1GHZ; + channels[ch].center_freq = KHZ_TO_MHZ(freq); + channels[ch].freq_offset = freq % 1000; + channels[ch].hw_value = ch + 1; + } +} + static const struct ieee80211_rate hwsim_rates[] = { { .bitrate = 10 }, { .bitrate = 20, .flags = IEEE80211_RATE_SHORT_PREAMBLE }, @@ -505,6 +548,7 @@ struct mac80211_hwsim_data { struct ieee80211_supported_band bands[NUM_NL80211_BANDS]; struct ieee80211_channel channels_2ghz[ARRAY_SIZE(hwsim_channels_2ghz)]; struct ieee80211_channel channels_5ghz[ARRAY_SIZE(hwsim_channels_5ghz)]; + struct ieee80211_channel channels_s1g[ARRAY_SIZE(hwsim_channels_s1g)]; struct ieee80211_rate rates[ARRAY_SIZE(hwsim_rates)]; struct ieee80211_iface_combination if_combination; struct ieee80211_iface_limit if_limits[3]; @@ -900,12 +944,14 @@ static void mac80211_hwsim_monitor_rx(struct ieee80211_hw *hw, struct mac80211_hwsim_data *data = hw->priv; struct sk_buff *skb; struct hwsim_radiotap_hdr *hdr; - u16 flags; + u16 flags, bitrate; struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_skb); struct ieee80211_rate *txrate = ieee80211_get_tx_rate(hw, info); - if (WARN_ON(!txrate)) - return; + if (!txrate) + bitrate = 0; + else + bitrate = txrate->bitrate; if (!netif_running(hwsim_mon)) return; @@ -924,10 +970,10 @@ static void mac80211_hwsim_monitor_rx(struct ieee80211_hw *hw, (1 << IEEE80211_RADIOTAP_CHANNEL)); hdr->rt_tsft = __mac80211_hwsim_get_tsf(data); hdr->rt_flags = 0; - hdr->rt_rate = txrate->bitrate / 5; + hdr->rt_rate = bitrate / 5; hdr->rt_channel = cpu_to_le16(chan->center_freq); flags = IEEE80211_CHAN_2GHZ; - if (txrate->flags & IEEE80211_RATE_ERP_G) + if (txrate && txrate->flags & IEEE80211_RATE_ERP_G) flags |= IEEE80211_CHAN_OFDM; else flags |= IEEE80211_CHAN_CCK; @@ -1341,6 +1387,7 @@ static bool mac80211_hwsim_tx_frame_no_nl(struct ieee80211_hw *hw, memset(&rx_status, 0, sizeof(rx_status)); rx_status.flag |= RX_FLAG_MACTIME_START; rx_status.freq = chan->center_freq; + rx_status.freq_offset = chan->freq_offset ? 1 : 0; rx_status.band = chan->band; if (info->control.rates[0].flags & IEEE80211_TX_RC_VHT_MCS) { rx_status.rate_idx = @@ -1522,14 +1569,18 @@ static void mac80211_hwsim_tx(struct ieee80211_hw *hw, /* fake header transmission time */ struct ieee80211_mgmt *mgmt; struct ieee80211_rate *txrate; + /* TODO: get MCS */ + int bitrate = 100; u64 ts; mgmt = (struct ieee80211_mgmt *)skb->data; txrate = ieee80211_get_tx_rate(hw, txi); + if (txrate) + bitrate = txrate->bitrate; ts = mac80211_hwsim_get_tsf_raw(); mgmt->u.probe_resp.timestamp = cpu_to_le64(ts + data->tsf_offset + - 24 * 8 * 10 / txrate->bitrate); + 24 * 8 * 10 / bitrate); } mac80211_hwsim_monitor_rx(hw, skb, channel); @@ -1664,6 +1715,8 @@ static void mac80211_hwsim_beacon_tx(void *arg, u8 *mac, struct ieee80211_rate *txrate; struct ieee80211_mgmt *mgmt; struct sk_buff *skb; + /* TODO: get MCS */ + int bitrate = 100; hwsim_check_magic(vif); @@ -1683,13 +1736,25 @@ static void mac80211_hwsim_beacon_tx(void *arg, u8 *mac, ARRAY_SIZE(info->control.rates)); txrate = ieee80211_get_tx_rate(hw, info); + if (txrate) + bitrate = txrate->bitrate; mgmt = (struct ieee80211_mgmt *) skb->data; /* fake header transmission time */ data->abs_bcn_ts = mac80211_hwsim_get_tsf_raw(); - mgmt->u.beacon.timestamp = cpu_to_le64(data->abs_bcn_ts + - data->tsf_offset + - 24 * 8 * 10 / txrate->bitrate); + if (ieee80211_is_s1g_beacon(mgmt->frame_control)) { + struct ieee80211_ext *ext = (void *) mgmt; + + ext->u.s1g_beacon.timestamp = cpu_to_le32(data->abs_bcn_ts + + data->tsf_offset + + 10 * 8 * 10 / + bitrate); + } else { + mgmt->u.beacon.timestamp = cpu_to_le64(data->abs_bcn_ts + + data->tsf_offset + + 24 * 8 * 10 / + bitrate); + } mac80211_hwsim_tx_frame(hw, skb, rcu_dereference(vif->chanctx_conf)->def.chan); @@ -1737,6 +1802,11 @@ static const char * const hwsim_chanwidths[] = { [NL80211_CHAN_WIDTH_80] = "vht80", [NL80211_CHAN_WIDTH_80P80] = "vht80p80", [NL80211_CHAN_WIDTH_160] = "vht160", + [NL80211_CHAN_WIDTH_1] = "1MHz", + [NL80211_CHAN_WIDTH_2] = "2MHz", + [NL80211_CHAN_WIDTH_4] = "4MHz", + [NL80211_CHAN_WIDTH_8] = "8MHz", + [NL80211_CHAN_WIDTH_16] = "16MHz", }; static int mac80211_hwsim_config(struct ieee80211_hw *hw, u32 changed) @@ -3079,6 +3149,8 @@ static int mac80211_hwsim_new_radio(struct genl_info *info, sizeof(hwsim_channels_2ghz)); memcpy(data->channels_5ghz, hwsim_channels_5ghz, sizeof(hwsim_channels_5ghz)); + memcpy(data->channels_s1g, hwsim_channels_s1g, + sizeof(hwsim_channels_s1g)); memcpy(data->rates, hwsim_rates, sizeof(hwsim_rates)); for (band = NL80211_BAND_2GHZ; band < NUM_NL80211_BANDS; band++) { @@ -3121,6 +3193,12 @@ static int mac80211_hwsim_new_radio(struct genl_info *info, sband->vht_cap.vht_mcs.tx_mcs_map = sband->vht_cap.vht_mcs.rx_mcs_map; break; + case NL80211_BAND_S1GHZ: + memcpy(&sband->s1g_cap, &hwsim_s1g_cap, + sizeof(sband->s1g_cap)); + sband->channels = data->channels_s1g; + sband->n_channels = ARRAY_SIZE(hwsim_channels_s1g); + break; default: continue; } @@ -4318,6 +4396,8 @@ static int __init init_mac80211_hwsim(void) goto out_exit_virtio; } + hwsim_init_s1g_channels(hwsim_channels_s1g); + for (i = 0; i < radios; i++) { struct hwsim_new_radio_params param = { 0 }; diff --git a/drivers/net/wireless/marvell/libertas/defs.h b/drivers/net/wireless/marvell/libertas/defs.h index 58e2ead7b0cc..f7e7bf56f924 100644 --- a/drivers/net/wireless/marvell/libertas/defs.h +++ b/drivers/net/wireless/marvell/libertas/defs.h @@ -50,8 +50,7 @@ extern unsigned int lbs_debug; #ifdef DEBUG #define LBS_DEB_LL(grp, grpnam, fmt, args...) \ do { if ((lbs_debug & (grp)) == (grp)) \ - printk(KERN_DEBUG DRV_NAME grpnam "%s: " fmt, \ - in_interrupt() ? " (INT)" : "", ## args); } while (0) + printk(KERN_DEBUG DRV_NAME grpnam ": " fmt, ## args); } while (0) #else #define LBS_DEB_LL(grp, grpnam, fmt, args...) do {} while (0) #endif diff --git a/drivers/net/wireless/marvell/libertas/rx.c b/drivers/net/wireless/marvell/libertas/rx.c index f28aa09d1f9e..9f24b0760e1f 100644 --- a/drivers/net/wireless/marvell/libertas/rx.c +++ b/drivers/net/wireless/marvell/libertas/rx.c @@ -147,10 +147,7 @@ int lbs_process_rxed_packet(struct lbs_private *priv, struct sk_buff *skb) dev->stats.rx_packets++; skb->protocol = eth_type_trans(skb, dev); - if (in_interrupt()) - netif_rx(skb); - else - netif_rx_ni(skb); + netif_rx_any_context(skb); ret = 0; done: @@ -265,11 +262,7 @@ static int process_rxed_802_11_packet(struct lbs_private *priv, dev->stats.rx_packets++; skb->protocol = eth_type_trans(skb, priv->dev); - - if (in_interrupt()) - netif_rx(skb); - else - netif_rx_ni(skb); + netif_rx_any_context(skb); ret = 0; diff --git a/drivers/net/wireless/marvell/libertas_tf/deb_defs.h b/drivers/net/wireless/marvell/libertas_tf/deb_defs.h index 37a98e228b46..0b520df62f1f 100644 --- a/drivers/net/wireless/marvell/libertas_tf/deb_defs.h +++ b/drivers/net/wireless/marvell/libertas_tf/deb_defs.h @@ -48,8 +48,7 @@ extern unsigned int lbtf_debug; #ifdef DEBUG #define LBTF_DEB_LL(grp, grpnam, fmt, args...) \ do { if ((lbtf_debug & (grp)) == (grp)) \ - printk(KERN_DEBUG DRV_NAME grpnam "%s: " fmt, \ - in_interrupt() ? " (INT)" : "", ## args); } while (0) + printk(KERN_DEBUG DRV_NAME grpnam ": " fmt, ## args); } while (0) #else #define LBTF_DEB_LL(grp, grpnam, fmt, args...) do {} while (0) #endif diff --git a/drivers/net/wireless/marvell/mwifiex/uap_txrx.c b/drivers/net/wireless/marvell/mwifiex/uap_txrx.c index 77c8595f84f8..9bbdb8dfce62 100644 --- a/drivers/net/wireless/marvell/mwifiex/uap_txrx.c +++ b/drivers/net/wireless/marvell/mwifiex/uap_txrx.c @@ -350,11 +350,7 @@ int mwifiex_uap_recv_packet(struct mwifiex_private *priv, skb->truesize += (skb->len - MWIFIEX_RX_DATA_BUF_SIZE); /* Forward multicast/broadcast packet to upper layer*/ - if (in_interrupt()) - netif_rx(skb); - else - netif_rx_ni(skb); - + netif_rx_any_context(skb); return 0; } diff --git a/drivers/net/wireless/marvell/mwifiex/util.c b/drivers/net/wireless/marvell/mwifiex/util.c index de89a1e710b1..d583fa600a29 100644 --- a/drivers/net/wireless/marvell/mwifiex/util.c +++ b/drivers/net/wireless/marvell/mwifiex/util.c @@ -488,11 +488,7 @@ int mwifiex_recv_packet(struct mwifiex_private *priv, struct sk_buff *skb) (skb->truesize > MWIFIEX_RX_DATA_BUF_SIZE)) skb->truesize += (skb->len - MWIFIEX_RX_DATA_BUF_SIZE); - if (in_interrupt()) - netif_rx(skb); - else - netif_rx_ni(skb); - + netif_rx_any_context(skb); return 0; } diff --git a/drivers/net/wireless/realtek/rtlwifi/base.c b/drivers/net/wireless/realtek/rtlwifi/base.c index 8e6f07b3c87e..6e8bd99e8911 100644 --- a/drivers/net/wireless/realtek/rtlwifi/base.c +++ b/drivers/net/wireless/realtek/rtlwifi/base.c @@ -436,6 +436,10 @@ static void _rtl_init_mac80211(struct ieee80211_hw *hw) } } +static void rtl_watchdog_wq_callback(struct work_struct *work); +static void rtl_fwevt_wq_callback(struct work_struct *work); +static void rtl_c2hcmd_wq_callback(struct work_struct *work); + static void _rtl_init_deferred_work(struct ieee80211_hw *hw) { struct rtl_priv *rtlpriv = rtl_priv(hw); @@ -454,17 +458,14 @@ static void _rtl_init_deferred_work(struct ieee80211_hw *hw) } INIT_DELAYED_WORK(&rtlpriv->works.watchdog_wq, - (void *)rtl_watchdog_wq_callback); + rtl_watchdog_wq_callback); INIT_DELAYED_WORK(&rtlpriv->works.ips_nic_off_wq, - (void *)rtl_ips_nic_off_wq_callback); - INIT_DELAYED_WORK(&rtlpriv->works.ps_work, - (void *)rtl_swlps_wq_callback); + rtl_ips_nic_off_wq_callback); + INIT_DELAYED_WORK(&rtlpriv->works.ps_work, rtl_swlps_wq_callback); INIT_DELAYED_WORK(&rtlpriv->works.ps_rfon_wq, - (void *)rtl_swlps_rfon_wq_callback); - INIT_DELAYED_WORK(&rtlpriv->works.fwevt_wq, - (void *)rtl_fwevt_wq_callback); - INIT_DELAYED_WORK(&rtlpriv->works.c2hcmd_wq, - (void *)rtl_c2hcmd_wq_callback); + rtl_swlps_rfon_wq_callback); + INIT_DELAYED_WORK(&rtlpriv->works.fwevt_wq, rtl_fwevt_wq_callback); + INIT_DELAYED_WORK(&rtlpriv->works.c2hcmd_wq, rtl_c2hcmd_wq_callback); } void rtl_deinit_deferred_work(struct ieee80211_hw *hw, bool ips_wq) @@ -1455,7 +1456,7 @@ static void setup_special_tx(struct rtl_priv *rtlpriv, struct rtl_ps_ctl *ppsc, if (rtlpriv->cfg->ops->get_btc_status()) rtlpriv->btcoexist.btc_ops->btc_special_packet_notify( rtlpriv, type); - rtl_lps_leave(hw); + rtl_lps_leave(hw, false); ppsc->last_delaylps_stamp_jiffies = jiffies; } @@ -1545,7 +1546,7 @@ u8 rtl_is_special_data(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx, if (is_tx) { rtlpriv->ra.is_special_data = true; - rtl_lps_leave(hw); + rtl_lps_leave(hw, false); ppsc->last_delaylps_stamp_jiffies = jiffies; setup_special_tx(rtlpriv, ppsc, PACKET_EAPOL); @@ -2042,11 +2043,10 @@ label_err: } EXPORT_SYMBOL(rtl_collect_scan_list); -void rtl_watchdog_wq_callback(void *data) +static void rtl_watchdog_wq_callback(struct work_struct *work) { - struct rtl_works *rtlworks = container_of_dwork_rtl(data, - struct rtl_works, - watchdog_wq); + struct rtl_works *rtlworks = container_of(work, struct rtl_works, + watchdog_wq.work); struct ieee80211_hw *hw = rtlworks->hw; struct rtl_priv *rtlpriv = rtl_priv(hw); struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); @@ -2147,9 +2147,9 @@ void rtl_watchdog_wq_callback(void *data) if (rtlpriv->link_info.num_rx_inperiod + rtlpriv->link_info.num_tx_inperiod > 8 || rtlpriv->link_info.num_rx_inperiod > 2) - rtl_lps_leave(hw); + rtl_lps_leave(hw, true); else - rtl_lps_enter(hw); + rtl_lps_enter(hw, true); label_lps_done: ; @@ -2239,10 +2239,10 @@ void rtl_watch_dog_timer_callback(struct timer_list *t) jiffies + MSECS(RTL_WATCH_DOG_TIME)); } -void rtl_fwevt_wq_callback(void *data) +static void rtl_fwevt_wq_callback(struct work_struct *work) { - struct rtl_works *rtlworks = - container_of_dwork_rtl(data, struct rtl_works, fwevt_wq); + struct rtl_works *rtlworks = container_of(work, struct rtl_works, + fwevt_wq.work); struct ieee80211_hw *hw = rtlworks->hw; struct rtl_priv *rtlpriv = rtl_priv(hw); @@ -2368,11 +2368,10 @@ void rtl_c2hcmd_launcher(struct ieee80211_hw *hw, int exec) } } -void rtl_c2hcmd_wq_callback(void *data) +static void rtl_c2hcmd_wq_callback(struct work_struct *work) { - struct rtl_works *rtlworks = container_of_dwork_rtl(data, - struct rtl_works, - c2hcmd_wq); + struct rtl_works *rtlworks = container_of(work, struct rtl_works, + c2hcmd_wq.work); struct ieee80211_hw *hw = rtlworks->hw; rtl_c2hcmd_launcher(hw, 1); diff --git a/drivers/net/wireless/realtek/rtlwifi/base.h b/drivers/net/wireless/realtek/rtlwifi/base.h index fa92e29fffda..0e4f8a8ae3a5 100644 --- a/drivers/net/wireless/realtek/rtlwifi/base.h +++ b/drivers/net/wireless/realtek/rtlwifi/base.h @@ -108,9 +108,6 @@ int rtl_rx_agg_start(struct ieee80211_hw *hw, int rtl_rx_agg_stop(struct ieee80211_hw *hw, struct ieee80211_sta *sta, u16 tid); void rtl_rx_ampdu_apply(struct rtl_priv *rtlpriv); -void rtl_watchdog_wq_callback(void *data); -void rtl_fwevt_wq_callback(void *data); -void rtl_c2hcmd_wq_callback(void *data); void rtl_c2hcmd_launcher(struct ieee80211_hw *hw, int exec); void rtl_c2hcmd_enqueue(struct ieee80211_hw *hw, struct sk_buff *skb); diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c index fa4486669d9a..2c05369b79e4 100644 --- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c +++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c @@ -285,7 +285,8 @@ static void halbtc_leave_lps(struct btc_coexist *btcoexist) btcoexist->bt_info.bt_ctrl_lps = true; btcoexist->bt_info.bt_lps_on = false; - rtl_lps_leave(rtlpriv->mac80211.hw); + /* FIXME: Context is unclear. Is it allowed to block? */ + rtl_lps_leave(rtlpriv->mac80211.hw, false); } static void halbtc_enter_lps(struct btc_coexist *btcoexist) @@ -306,7 +307,8 @@ static void halbtc_enter_lps(struct btc_coexist *btcoexist) btcoexist->bt_info.bt_ctrl_lps = true; btcoexist->bt_info.bt_lps_on = true; - rtl_lps_enter(rtlpriv->mac80211.hw); + /* FIXME: Context is unclear. Is it allowed to block? */ + rtl_lps_enter(rtlpriv->mac80211.hw, false); } static void halbtc_normal_lps(struct btc_coexist *btcoexist) @@ -317,7 +319,8 @@ static void halbtc_normal_lps(struct btc_coexist *btcoexist) if (btcoexist->bt_info.bt_ctrl_lps) { btcoexist->bt_info.bt_lps_on = false; - rtl_lps_leave(rtlpriv->mac80211.hw); + /* FIXME: Context is unclear. Is it allowed to block? */ + rtl_lps_leave(rtlpriv->mac80211.hw, false); btcoexist->bt_info.bt_ctrl_lps = false; } } @@ -328,7 +331,8 @@ static void halbtc_pre_normal_lps(struct btc_coexist *btcoexist) if (btcoexist->bt_info.bt_ctrl_lps) { btcoexist->bt_info.bt_lps_on = false; - rtl_lps_leave(rtlpriv->mac80211.hw); + /* FIXME: Context is unclear. Is it allowed to block? */ + rtl_lps_leave(rtlpriv->mac80211.hw, false); } } diff --git a/drivers/net/wireless/realtek/rtlwifi/core.c b/drivers/net/wireless/realtek/rtlwifi/core.c index 2529b6e0fd76..a7259dbc953d 100644 --- a/drivers/net/wireless/realtek/rtlwifi/core.c +++ b/drivers/net/wireless/realtek/rtlwifi/core.c @@ -544,7 +544,7 @@ static int rtl_op_suspend(struct ieee80211_hw *hw, rtlhal->driver_is_goingto_unload = true; rtlhal->enter_pnp_sleep = true; - rtl_lps_leave(hw); + rtl_lps_leave(hw, true); rtl_op_stop(hw); device_set_wakeup_enable(wiphy_dev(hw->wiphy), true); return 0; @@ -1151,7 +1151,7 @@ static void rtl_op_bss_info_changed(struct ieee80211_hw *hw, mstatus = RT_MEDIA_DISCONNECT; if (mac->link_state == MAC80211_LINKED) - rtl_lps_leave(hw); + rtl_lps_leave(hw, true); if (ppsc->p2p_ps_info.p2p_ps_mode > P2P_PS_NONE) rtl_p2p_ps_cmd(hw, P2P_PS_DISABLE); mac->link_state = MAC80211_NOLINK; @@ -1448,7 +1448,7 @@ static void rtl_op_sw_scan_start(struct ieee80211_hw *hw, } if (mac->link_state == MAC80211_LINKED) { - rtl_lps_leave(hw); + rtl_lps_leave(hw, true); mac->link_state = MAC80211_LINKED_SCANNING; } else { rtl_ips_nic_on(hw); diff --git a/drivers/net/wireless/realtek/rtlwifi/debug.c b/drivers/net/wireless/realtek/rtlwifi/debug.c index 455b87e7548b..901cdfe3723c 100644 --- a/drivers/net/wireless/realtek/rtlwifi/debug.c +++ b/drivers/net/wireless/realtek/rtlwifi/debug.c @@ -8,26 +8,6 @@ #include <linux/vmalloc.h> #ifdef CONFIG_RTLWIFI_DEBUG -void _rtl_dbg_out(struct rtl_priv *rtlpriv, u64 comp, int level, - const char *fmt, ...) -{ - if (unlikely((comp & rtlpriv->cfg->mod_params->debug_mask) && - level <= rtlpriv->cfg->mod_params->debug_level)) { - struct va_format vaf; - va_list args; - - va_start(args, fmt); - - vaf.fmt = fmt; - vaf.va = &args; - - pr_info(":<%lx> %pV", in_interrupt(), &vaf); - - va_end(args); - } -} -EXPORT_SYMBOL_GPL(_rtl_dbg_out); - void _rtl_dbg_print(struct rtl_priv *rtlpriv, u64 comp, int level, const char *fmt, ...) { diff --git a/drivers/net/wireless/realtek/rtlwifi/debug.h b/drivers/net/wireless/realtek/rtlwifi/debug.h index 6028f1ffa5da..1c0bcf8ec1a9 100644 --- a/drivers/net/wireless/realtek/rtlwifi/debug.h +++ b/drivers/net/wireless/realtek/rtlwifi/debug.h @@ -149,10 +149,6 @@ enum dbgp_flag_e { struct rtl_priv; __printf(4, 5) -void _rtl_dbg_out(struct rtl_priv *rtlpriv, u64 comp, int level, - const char *fmt, ...); - -__printf(4, 5) void _rtl_dbg_print(struct rtl_priv *rtlpriv, u64 comp, int level, const char *fmt, ...); @@ -160,8 +156,8 @@ void _rtl_dbg_print_data(struct rtl_priv *rtlpriv, u64 comp, int level, const char *titlestring, const void *hexdata, int hexdatalen); -#define rtl_dbg(rtlpriv, comp, level, fmt, ...) \ - _rtl_dbg_out(rtlpriv, comp, level, \ +#define rtl_dbg(rtlpriv, comp, level, fmt, ...) \ + _rtl_dbg_print(rtlpriv, comp, level, \ fmt, ##__VA_ARGS__) #define RTPRINT(rtlpriv, dbgtype, dbgflag, fmt, ...) \ diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c index d9f901111e58..3776495fd9d0 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.c +++ b/drivers/net/wireless/realtek/rtlwifi/pci.c @@ -621,7 +621,7 @@ tx_status_ok: if (((rtlpriv->link_info.num_rx_inperiod + rtlpriv->link_info.num_tx_inperiod) > 8) || rtlpriv->link_info.num_rx_inperiod > 2) - rtl_lps_leave(hw); + rtl_lps_leave(hw, false); } static int _rtl_pci_init_one_rxdesc(struct ieee80211_hw *hw, @@ -874,7 +874,7 @@ new_trx_end: if (((rtlpriv->link_info.num_rx_inperiod + rtlpriv->link_info.num_tx_inperiod) > 8) || rtlpriv->link_info.num_rx_inperiod > 2) - rtl_lps_leave(hw); + rtl_lps_leave(hw, false); skb = new_skb; no_new: if (rtlpriv->use_new_trx_flow) { diff --git a/drivers/net/wireless/realtek/rtlwifi/ps.c b/drivers/net/wireless/realtek/rtlwifi/ps.c index 38442a36f599..f99882255d48 100644 --- a/drivers/net/wireless/realtek/rtlwifi/ps.c +++ b/drivers/net/wireless/realtek/rtlwifi/ps.c @@ -179,10 +179,10 @@ static void _rtl_ps_inactive_ps(struct ieee80211_hw *hw) ppsc->swrf_processing = false; } -void rtl_ips_nic_off_wq_callback(void *data) +void rtl_ips_nic_off_wq_callback(struct work_struct *work) { - struct rtl_works *rtlworks = - container_of_dwork_rtl(data, struct rtl_works, ips_nic_off_wq); + struct rtl_works *rtlworks = container_of(work, struct rtl_works, + ips_nic_off_wq.work); struct ieee80211_hw *hw = rtlworks->hw; struct rtl_priv *rtlpriv = rtl_priv(hw); struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); @@ -562,10 +562,10 @@ void rtl_swlps_rf_awake(struct ieee80211_hw *hw) mutex_unlock(&rtlpriv->locks.lps_mutex); } -void rtl_swlps_rfon_wq_callback(void *data) +void rtl_swlps_rfon_wq_callback(struct work_struct *work) { - struct rtl_works *rtlworks = - container_of_dwork_rtl(data, struct rtl_works, ps_rfon_wq); + struct rtl_works *rtlworks = container_of(work, struct rtl_works, + ps_rfon_wq.work); struct ieee80211_hw *hw = rtlworks->hw; rtl_swlps_rf_awake(hw); @@ -653,33 +653,32 @@ void rtl_lps_change_work_callback(struct work_struct *work) } EXPORT_SYMBOL_GPL(rtl_lps_change_work_callback); -void rtl_lps_enter(struct ieee80211_hw *hw) +void rtl_lps_enter(struct ieee80211_hw *hw, bool may_block) { struct rtl_priv *rtlpriv = rtl_priv(hw); - if (!in_interrupt()) + if (may_block) return rtl_lps_enter_core(hw); rtlpriv->enter_ps = true; schedule_work(&rtlpriv->works.lps_change_work); } EXPORT_SYMBOL_GPL(rtl_lps_enter); -void rtl_lps_leave(struct ieee80211_hw *hw) +void rtl_lps_leave(struct ieee80211_hw *hw, bool may_block) { struct rtl_priv *rtlpriv = rtl_priv(hw); - if (!in_interrupt()) + if (may_block) return rtl_lps_leave_core(hw); rtlpriv->enter_ps = false; schedule_work(&rtlpriv->works.lps_change_work); } EXPORT_SYMBOL_GPL(rtl_lps_leave); -void rtl_swlps_wq_callback(void *data) +void rtl_swlps_wq_callback(struct work_struct *work) { - struct rtl_works *rtlworks = container_of_dwork_rtl(data, - struct rtl_works, - ps_work); + struct rtl_works *rtlworks = container_of(work, struct rtl_works, + ps_work.work); struct ieee80211_hw *hw = rtlworks->hw; struct rtl_priv *rtlpriv = rtl_priv(hw); bool ps = false; diff --git a/drivers/net/wireless/realtek/rtlwifi/ps.h b/drivers/net/wireless/realtek/rtlwifi/ps.h index aaa2ed2bbe16..b37a929def82 100644 --- a/drivers/net/wireless/realtek/rtlwifi/ps.h +++ b/drivers/net/wireless/realtek/rtlwifi/ps.h @@ -10,15 +10,15 @@ bool rtl_ps_enable_nic(struct ieee80211_hw *hw); bool rtl_ps_disable_nic(struct ieee80211_hw *hw); void rtl_ips_nic_off(struct ieee80211_hw *hw); void rtl_ips_nic_on(struct ieee80211_hw *hw); -void rtl_ips_nic_off_wq_callback(void *data); -void rtl_lps_enter(struct ieee80211_hw *hw); -void rtl_lps_leave(struct ieee80211_hw *hw); +void rtl_ips_nic_off_wq_callback(struct work_struct *work); +void rtl_lps_enter(struct ieee80211_hw *hw, bool may_block); +void rtl_lps_leave(struct ieee80211_hw *hw, bool may_block); void rtl_lps_set_psmode(struct ieee80211_hw *hw, u8 rt_psmode); void rtl_swlps_beacon(struct ieee80211_hw *hw, void *data, unsigned int len); -void rtl_swlps_wq_callback(void *data); -void rtl_swlps_rfon_wq_callback(void *data); +void rtl_swlps_wq_callback(struct work_struct *work); +void rtl_swlps_rfon_wq_callback(struct work_struct *work); void rtl_swlps_rf_awake(struct ieee80211_hw *hw); void rtl_swlps_rf_sleep(struct ieee80211_hw *hw); void rtl_p2p_ps_cmd(struct ieee80211_hw *hw , u8 p2p_ps_state); diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h index 13421cf2d201..7e80fd829014 100644 --- a/drivers/net/wireless/realtek/rtlwifi/wifi.h +++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h @@ -2936,9 +2936,6 @@ enum bt_radio_shared { #define RT_SET_PS_LEVEL(ppsc, _ps_flg) \ (ppsc->cur_ps_level |= _ps_flg) -#define container_of_dwork_rtl(x, y, z) \ - container_of(to_delayed_work(x), y, z) - #define FILL_OCTET_STRING(_os, _octet, _len) \ (_os).octet = (u8 *)(_octet); \ (_os).length = (_len); diff --git a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c index 71ec3d4231ef..66367ab7e4c1 100644 --- a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c +++ b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c @@ -378,7 +378,6 @@ static inline void handle_regs_int(struct urb *urb) int len; u16 int_num; - ZD_ASSERT(in_interrupt()); spin_lock_irqsave(&intr->lock, flags); int_num = le16_to_cpu(*(__le16 *)(urb->transfer_buffer+2)); diff --git a/drivers/s390/net/ism.h b/drivers/s390/net/ism.h index 1901e9c80ed8..38fe90c2597d 100644 --- a/drivers/s390/net/ism.h +++ b/drivers/s390/net/ism.h @@ -16,6 +16,7 @@ #define ISM_DMB_WORD_OFFSET 1 #define ISM_DMB_BIT_OFFSET (ISM_DMB_WORD_OFFSET * 32) #define ISM_NR_DMBS 1920 +#define ISM_IDENT_MASK 0x00FFFF #define ISM_REG_SBA 0x1 #define ISM_REG_IEQ 0x2 @@ -206,6 +207,12 @@ struct ism_dev { #define ISM_CREATE_REQ(dmb, idx, sf, offset) \ ((dmb) | (idx) << 24 | (sf) << 23 | (offset)) +struct ism_systemeid { + u8 seid_string[24]; + u8 serial_number[4]; + u8 type[4]; +}; + static inline void __ism_read_cmd(struct ism_dev *ism, void *data, unsigned long offset, unsigned long len) { diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c index 5fbe9eae84d1..fe96ca3c88a5 100644 --- a/drivers/s390/net/ism_drv.c +++ b/drivers/s390/net/ism_drv.c @@ -13,6 +13,8 @@ #include <linux/device.h> #include <linux/pci.h> #include <linux/err.h> +#include <linux/ctype.h> +#include <linux/processor.h> #include <net/smc.h> #include <asm/debug.h> @@ -387,6 +389,42 @@ static int ism_move(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx, return 0; } +static struct ism_systemeid SYSTEM_EID = { + .seid_string = "IBM-SYSZ-IBMSEID00000000", + .serial_number = "0000", + .type = "0000", +}; + +static void ism_create_system_eid(void) +{ + struct cpuid id; + u16 ident_tail; + char tmp[5]; + + get_cpu_id(&id); + ident_tail = (u16)(id.ident & ISM_IDENT_MASK); + snprintf(tmp, 5, "%04X", ident_tail); + memcpy(&SYSTEM_EID.serial_number, tmp, 4); + snprintf(tmp, 5, "%04X", id.machine); + memcpy(&SYSTEM_EID.type, tmp, 4); +} + +static void ism_get_system_eid(struct smcd_dev *smcd, u8 **eid) +{ + *eid = &SYSTEM_EID.seid_string[0]; +} + +static u16 ism_get_chid(struct smcd_dev *smcd) +{ + struct ism_dev *ismdev; + + ismdev = (struct ism_dev *)smcd->priv; + if (!ismdev || !ismdev->pdev) + return 0; + + return to_zpci(ismdev->pdev)->pchid; +} + static void ism_handle_event(struct ism_dev *ism) { struct smcd_event *entry; @@ -443,6 +481,8 @@ static const struct smcd_ops ism_ops = { .reset_vlan_required = ism_reset_vlan_required, .signal_event = ism_signal_ieq, .move_data = ism_move, + .get_system_eid = ism_get_system_eid, + .get_chid = ism_get_chid, }; static int ism_dev_init(struct ism_dev *ism) @@ -471,6 +511,10 @@ static int ism_dev_init(struct ism_dev *ism) if (ret) goto unreg_ieq; + if (!ism_add_vlan_id(ism->smcd, ISM_RESERVED_VLANID)) + /* hardware is V2 capable */ + ism_create_system_eid(); + ret = smcd_register_dev(ism->smcd); if (ret) goto unreg_ieq; @@ -550,6 +594,9 @@ static void ism_dev_exit(struct ism_dev *ism) struct pci_dev *pdev = ism->pdev; smcd_unregister_dev(ism->smcd); + if (SYSTEM_EID.serial_number[0] != '0' || + SYSTEM_EID.type[0] != '0') + ism_del_vlan_id(ism->smcd, ISM_RESERVED_VLANID); unregister_ieq(ism); unregister_sba(ism); free_irq(pci_irq_vector(pdev, 0), ism); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index fc5c901c7542..50e5c4b52bd1 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -292,6 +292,7 @@ enum bpf_arg_type { ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */ ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */ ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ + ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */ __BPF_ARG_TYPE_MAX, }; @@ -382,8 +383,22 @@ enum bpf_reg_type { PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ - PTR_TO_BTF_ID, /* reg points to kernel struct */ - PTR_TO_BTF_ID_OR_NULL, /* reg points to kernel struct or NULL */ + /* PTR_TO_BTF_ID points to a kernel struct that does not need + * to be null checked by the BPF program. This does not imply the + * pointer is _not_ null and in practice this can easily be a null + * pointer when reading pointer chains. The assumption is program + * context will handle null pointer dereference typically via fault + * handling. The verifier must keep this in mind and can make no + * assumptions about null or non-null when doing branch analysis. + * Further, when passed into helpers the helpers can not, without + * additional context, assume the value is non-null. + */ + PTR_TO_BTF_ID, + /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not + * been checked for null. Used primarily to inform the verifier + * an explicit null check is required for this struct. + */ + PTR_TO_BTF_ID_OR_NULL, PTR_TO_MEM, /* reg points to valid memory region */ PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */ PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */ @@ -591,6 +606,13 @@ struct bpf_trampoline { struct bpf_ksym ksym; }; +struct bpf_attach_target_info { + struct btf_func_model fmodel; + long tgt_addr; + const char *tgt_name; + const struct btf_type *tgt_type; +}; + #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */ struct bpf_dispatcher_prog { @@ -618,9 +640,10 @@ static __always_inline unsigned int bpf_dispatcher_nop_func( return bpf_func(ctx, insnsi); } #ifdef CONFIG_BPF_JIT -struct bpf_trampoline *bpf_trampoline_lookup(u64 key); -int bpf_trampoline_link_prog(struct bpf_prog *prog); -int bpf_trampoline_unlink_prog(struct bpf_prog *prog); +int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr); +int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr); +struct bpf_trampoline *bpf_trampoline_get(u64 key, + struct bpf_attach_target_info *tgt_info); void bpf_trampoline_put(struct bpf_trampoline *tr); #define BPF_DISPATCHER_INIT(_name) { \ .mutex = __MUTEX_INITIALIZER(_name.mutex), \ @@ -665,17 +688,20 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym); void bpf_ksym_add(struct bpf_ksym *ksym); void bpf_ksym_del(struct bpf_ksym *ksym); #else -static inline struct bpf_trampoline *bpf_trampoline_lookup(u64 key) +static inline int bpf_trampoline_link_prog(struct bpf_prog *prog, + struct bpf_trampoline *tr) { - return NULL; + return -ENOTSUPP; } -static inline int bpf_trampoline_link_prog(struct bpf_prog *prog) +static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog, + struct bpf_trampoline *tr) { return -ENOTSUPP; } -static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog) +static inline struct bpf_trampoline *bpf_trampoline_get(u64 key, + struct bpf_attach_target_info *tgt_info) { - return -ENOTSUPP; + return ERR_PTR(-EOPNOTSUPP); } static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {} #define DEFINE_BPF_DISPATCHER(name) @@ -739,7 +765,11 @@ struct bpf_prog_aux { u32 max_rdonly_access; u32 max_rdwr_access; const struct bpf_ctx_arg_aux *ctx_arg_info; - struct bpf_prog *linked_prog; + struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */ + struct bpf_prog *dst_prog; + struct bpf_trampoline *dst_trampoline; + enum bpf_prog_type saved_dst_prog_type; + enum bpf_attach_type saved_dst_attach_type; bool verifier_zext; /* Zero extensions has been inserted by verifier. */ bool offload_requested; bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ @@ -747,7 +777,6 @@ struct bpf_prog_aux { bool sleepable; bool tail_call_reachable; enum bpf_tramp_prog_type trampoline_prog_type; - struct bpf_trampoline *trampoline; struct hlist_node tramp_hlist; /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ const struct btf_type *attach_func_proto; @@ -1340,6 +1369,8 @@ int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, union bpf_attr __user *uattr); void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); +struct btf *bpf_get_btf_vmlinux(void); + /* Map specifics */ struct xdp_buff; struct sk_buff; @@ -1381,6 +1412,9 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog, int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); +int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, + const union bpf_attr *kattr, + union bpf_attr __user *uattr); bool btf_ctx_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info); @@ -1402,7 +1436,7 @@ int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs); int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *reg); -int btf_check_type_match(struct bpf_verifier_env *env, struct bpf_prog *prog, +int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, struct btf *btf, const struct btf_type *t); struct bpf_prog *bpf_prog_by_id(u32 id); @@ -1793,6 +1827,7 @@ extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto; extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto; extern const struct bpf_func_proto bpf_copy_from_user_proto; +extern const struct bpf_func_proto bpf_snprintf_btf_proto; const struct bpf_func_proto *bpf_tracing_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 2bb48a2c4d08..363b4f1c562a 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -347,8 +347,9 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log) static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) { - return (log->level && log->ubuf && !bpf_verifier_log_full(log)) || - log->level == BPF_LOG_KERNEL; + return log && + ((log->level && log->ubuf && !bpf_verifier_log_full(log)) || + log->level == BPF_LOG_KERNEL); } #define BPF_MAX_SUBPROGS 256 @@ -449,4 +450,17 @@ bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); int check_ctx_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int regno); +/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */ +static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog, + u32 btf_id) +{ + return tgt_prog ? (((u64)tgt_prog->aux->id) << 32 | btf_id) : btf_id; +} + +int bpf_check_attach_target(struct bpf_verifier_log *log, + const struct bpf_prog *prog, + const struct bpf_prog *tgt_prog, + u32 btf_id, + struct bpf_attach_target_info *tgt_info); + #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/include/linux/btf.h b/include/linux/btf.h index a9af5e7a7ece..024e16ff7dcc 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -6,6 +6,7 @@ #include <linux/types.h> #include <uapi/linux/btf.h> +#include <uapi/linux/bpf.h> #define BTF_TYPE_EMIT(type) ((void)(type *)0) @@ -13,6 +14,7 @@ struct btf; struct btf_member; struct btf_type; union bpf_attr; +struct btf_show; extern const struct file_operations btf_fops; @@ -46,8 +48,45 @@ int btf_get_info_by_fd(const struct btf *btf, const struct btf_type *btf_type_id_size(const struct btf *btf, u32 *type_id, u32 *ret_size); + +/* + * Options to control show behaviour. + * - BTF_SHOW_COMPACT: no formatting around type information + * - BTF_SHOW_NONAME: no struct/union member names/types + * - BTF_SHOW_PTR_RAW: show raw (unobfuscated) pointer values; + * equivalent to %px. + * - BTF_SHOW_ZERO: show zero-valued struct/union members; they + * are not displayed by default + * - BTF_SHOW_UNSAFE: skip use of bpf_probe_read() to safely read + * data before displaying it. + */ +#define BTF_SHOW_COMPACT BTF_F_COMPACT +#define BTF_SHOW_NONAME BTF_F_NONAME +#define BTF_SHOW_PTR_RAW BTF_F_PTR_RAW +#define BTF_SHOW_ZERO BTF_F_ZERO +#define BTF_SHOW_UNSAFE (1ULL << 4) + void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, struct seq_file *m); +int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, void *obj, + struct seq_file *m, u64 flags); + +/* + * Copy len bytes of string representation of obj of BTF type_id into buf. + * + * @btf: struct btf object + * @type_id: type id of type obj points to + * @obj: pointer to typed data + * @buf: buffer to write to + * @len: maximum length to write to buf + * @flags: show options (see above) + * + * Return: length that would have been/was copied as per snprintf, or + * negative error. + */ +int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, + char *buf, int len, u64 flags); + int btf_get_fd_by_id(u32 id); u32 btf_id(const struct btf *btf); bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, diff --git a/include/linux/cookie.h b/include/linux/cookie.h new file mode 100644 index 000000000000..0c159f585109 --- /dev/null +++ b/include/linux/cookie.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_COOKIE_H +#define __LINUX_COOKIE_H + +#include <linux/atomic.h> +#include <linux/percpu.h> +#include <asm/local.h> + +struct pcpu_gen_cookie { + local_t nesting; + u64 last; +} __aligned(16); + +struct gen_cookie { + struct pcpu_gen_cookie __percpu *local; + atomic64_t forward_last ____cacheline_aligned_in_smp; + atomic64_t reverse_last; +}; + +#define COOKIE_LOCAL_BATCH 4096 + +#define DEFINE_COOKIE(name) \ + static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name); \ + static struct gen_cookie name = { \ + .local = &__##name, \ + .forward_last = ATOMIC64_INIT(0), \ + .reverse_last = ATOMIC64_INIT(0), \ + } + +static __always_inline u64 gen_cookie_next(struct gen_cookie *gc) +{ + struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local); + u64 val; + + if (likely(local_inc_return(&local->nesting) == 1)) { + val = local->last; + if (__is_defined(CONFIG_SMP) && + unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) { + s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH, + &gc->forward_last); + val = next - COOKIE_LOCAL_BATCH; + } + local->last = ++val; + } else { + val = atomic64_dec_return(&gc->reverse_last); + } + local_dec(&local->nesting); + return val; +} + +#endif /* __LINUX_COOKIE_H */ diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 53fba39d4ba6..770408b2fdaf 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -151,6 +151,9 @@ #define IEEE80211_ANO_NETTYPE_WILD 15 +/* bits unique to S1G beacon */ +#define IEEE80211_S1G_BCN_NEXT_TBTT 0x100 + /* control extension - for IEEE80211_FTYPE_CTL | IEEE80211_STYPE_CTL_EXT */ #define IEEE80211_CTL_EXT_POLL 0x2000 #define IEEE80211_CTL_EXT_SPR 0x3000 @@ -554,6 +557,28 @@ static inline bool ieee80211_is_s1g_beacon(__le16 fc) } /** + * ieee80211_next_tbtt_present - check if IEEE80211_FTYPE_EXT && + * IEEE80211_STYPE_S1G_BEACON && IEEE80211_S1G_BCN_NEXT_TBTT + * @fc: frame control bytes in little-endian byteorder + */ +static inline bool ieee80211_next_tbtt_present(__le16 fc) +{ + return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) == + cpu_to_le16(IEEE80211_FTYPE_EXT | IEEE80211_STYPE_S1G_BEACON) && + fc & cpu_to_le16(IEEE80211_S1G_BCN_NEXT_TBTT); +} + +/** + * ieee80211_is_s1g_short_beacon - check if next tbtt present bit is set. Only + * true for S1G beacons when they're short. + * @fc: frame control bytes in little-endian byteorder + */ +static inline bool ieee80211_is_s1g_short_beacon(__le16 fc) +{ + return ieee80211_is_s1g_beacon(fc) && ieee80211_next_tbtt_present(fc); +} + +/** * ieee80211_is_atim - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_ATIM * @fc: frame control bytes in little-endian byteorder */ @@ -962,6 +987,25 @@ enum ieee80211_vht_opmode_bits { IEEE80211_OPMODE_NOTIF_RX_NSS_TYPE_BF = 0x80, }; +/** + * enum ieee80211_s1g_chanwidth + * These are defined in IEEE802.11-2016ah Table 10-20 + * as BSS Channel Width + * + * @IEEE80211_S1G_CHANWIDTH_1MHZ: 1MHz operating channel + * @IEEE80211_S1G_CHANWIDTH_2MHZ: 2MHz operating channel + * @IEEE80211_S1G_CHANWIDTH_4MHZ: 4MHz operating channel + * @IEEE80211_S1G_CHANWIDTH_8MHZ: 8MHz operating channel + * @IEEE80211_S1G_CHANWIDTH_16MHZ: 16MHz operating channel + */ +enum ieee80211_s1g_chanwidth { + IEEE80211_S1G_CHANWIDTH_1MHZ = 0, + IEEE80211_S1G_CHANWIDTH_2MHZ = 1, + IEEE80211_S1G_CHANWIDTH_4MHZ = 3, + IEEE80211_S1G_CHANWIDTH_8MHZ = 7, + IEEE80211_S1G_CHANWIDTH_16MHZ = 15, +}; + #define WLAN_SA_QUERY_TR_ID_LEN 2 #define WLAN_MEMBERSHIP_LEN 8 #define WLAN_USER_POSITION_LEN 16 @@ -1034,6 +1078,13 @@ struct ieee80211_ext { u8 change_seq; u8 variable[0]; } __packed s1g_beacon; + struct { + u8 sa[ETH_ALEN]; + __le32 timestamp; + u8 change_seq; + u8 next_tbtt[3]; + u8 variable[0]; + } __packed s1g_short_beacon; } u; } __packed __aligned(2); @@ -1070,6 +1121,11 @@ struct ieee80211_mgmt { } __packed assoc_resp, reassoc_resp; struct { __le16 capab_info; + __le16 status_code; + u8 variable[0]; + } __packed s1g_assoc_resp, s1g_reassoc_resp; + struct { + __le16 capab_info; __le16 listen_interval; u8 current_ap[ETH_ALEN]; /* followed by SSID and Supported rates */ @@ -2294,8 +2350,11 @@ ieee80211_he_6ghz_oper(const struct ieee80211_he_operation *he_oper) } /* HE Spatial Reuse defines */ -#define IEEE80211_HE_SPR_NON_SRG_OFFSET_PRESENT 0x4 -#define IEEE80211_HE_SPR_SRG_INFORMATION_PRESENT 0x8 +#define IEEE80211_HE_SPR_PSR_DISALLOWED BIT(0) +#define IEEE80211_HE_SPR_NON_SRG_OBSS_PD_SR_DISALLOWED BIT(1) +#define IEEE80211_HE_SPR_NON_SRG_OFFSET_PRESENT BIT(2) +#define IEEE80211_HE_SPR_SRG_INFORMATION_PRESENT BIT(3) +#define IEEE80211_HE_SPR_HESIGA_SR_VAL15_ALLOWED BIT(4) /* * ieee80211_he_spr_size - calculate 802.11ax HE Spatial Reuse IE size @@ -2330,6 +2389,8 @@ ieee80211_he_spr_size(const u8 *he_spr_ie) } /* S1G Capabilities Information field */ +#define IEEE80211_S1G_CAPABILITY_LEN 15 + #define S1G_CAP0_S1G_LONG BIT(0) #define S1G_CAP0_SGI_1MHZ BIT(1) #define S1G_CAP0_SGI_2MHZ BIT(2) @@ -2409,6 +2470,13 @@ ieee80211_he_spr_size(const u8 *he_spr_ie) #define S1G_OPER_CH_WIDTH_PRIMARY_1MHZ BIT(0) #define S1G_OPER_CH_WIDTH_OPER GENMASK(4, 1) + +#define LISTEN_INT_USF GENMASK(15, 14) +#define LISTEN_INT_UI GENMASK(13, 0) + +#define IEEE80211_MAX_USF FIELD_MAX(LISTEN_INT_USF) +#define IEEE80211_MAX_UI FIELD_MAX(LISTEN_INT_UI) + /* Authentication algorithms */ #define WLAN_AUTH_OPEN 0 #define WLAN_AUTH_SHARED_KEY 1 @@ -2808,6 +2876,8 @@ enum ieee80211_eid { WLAN_EID_REDUCED_NEIGHBOR_REPORT = 201, + WLAN_EID_AID_REQUEST = 210, + WLAN_EID_AID_RESPONSE = 211, WLAN_EID_S1G_BCN_COMPAT = 213, WLAN_EID_S1G_SHORT_BCN_INTERVAL = 214, WLAN_EID_S1G_CAPABILITIES = 217, diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h index c16827eeba9c..b0ae8020f13e 100644 --- a/include/linux/mlx5/eswitch.h +++ b/include/linux/mlx5/eswitch.h @@ -74,15 +74,16 @@ bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw); bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw); /* Reg C0 usage: - * Reg C0 = < ESW_VHCA_ID_BITS(8) | ESW_VPORT BITS(8) | ESW_CHAIN_TAG(16) > + * Reg C0 = < ESW_PFNUM_BITS(4) | ESW_VPORT BITS(12) | ESW_CHAIN_TAG(16) > * - * Highest 8 bits of the reg c0 is the vhca_id, next 8 bits is vport_num, - * the rest (lowest 16 bits) is left for tc chain tag restoration. - * VHCA_ID + VPORT comprise the SOURCE_PORT matching. + * Highest 4 bits of the reg c0 is the PF_NUM (range 0-15), 12 bits of + * unique non-zero vport id (range 1-4095). The rest (lowest 16 bits) is left + * for tc chain tag restoration. + * PFNUM + VPORT comprise the SOURCE_PORT matching. */ -#define ESW_VHCA_ID_BITS 8 -#define ESW_VPORT_BITS 8 -#define ESW_SOURCE_PORT_METADATA_BITS (ESW_VHCA_ID_BITS + ESW_VPORT_BITS) +#define ESW_VPORT_BITS 12 +#define ESW_PFNUM_BITS 4 +#define ESW_SOURCE_PORT_METADATA_BITS (ESW_PFNUM_BITS + ESW_VPORT_BITS) #define ESW_SOURCE_PORT_METADATA_OFFSET (32 - ESW_SOURCE_PORT_METADATA_BITS) #define ESW_CHAIN_TAG_METADATA_BITS (32 - ESW_SOURCE_PORT_METADATA_BITS) #define ESW_CHAIN_TAG_METADATA_MASK GENMASK(ESW_CHAIN_TAG_METADATA_BITS - 1,\ diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index a431c3229cbf..28cfa53daf72 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3785,6 +3785,7 @@ void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog); int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb); int netif_rx(struct sk_buff *skb); int netif_rx_ni(struct sk_buff *skb); +int netif_rx_any_context(struct sk_buff *skb); int netif_receive_skb(struct sk_buff *skb); int netif_receive_skb_core(struct sk_buff *skb); void netif_receive_skb_list(struct list_head *head); diff --git a/include/linux/platform_data/macb.h b/include/linux/platform_data/macb.h deleted file mode 100644 index aa5b5562d6f7..000000000000 --- a/include/linux/platform_data/macb.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2004-2006 Atmel Corporation - */ -#ifndef __MACB_PDATA_H__ -#define __MACB_PDATA_H__ - -#include <linux/clk.h> - -/** - * struct macb_platform_data - platform data for MACB Ethernet - * @pclk: platform clock - * @hclk: AHB clock - */ -struct macb_platform_data { - struct clk *pclk; - struct clk *hclk; -}; - -#endif /* __MACB_PDATA_H__ */ diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index aaaac8ac927c..3e7919fc5f34 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -50,6 +50,7 @@ static inline void rcu_read_lock_trace(void) struct task_struct *t = current; WRITE_ONCE(t->trc_reader_nesting, READ_ONCE(t->trc_reader_nesting) + 1); + barrier(); if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && t->trc_reader_special.b.need_mb) smp_mb(); // Pairs with update-side barriers @@ -72,6 +73,9 @@ static inline void rcu_read_unlock_trace(void) rcu_lock_release(&rcu_trace_lock_map); nesting = READ_ONCE(t->trc_reader_nesting) - 1; + barrier(); // Critical section before disabling. + // Disable IPI-based setting of .need_qs. + WRITE_ONCE(t->trc_reader_nesting, INT_MIN); if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) { WRITE_ONCE(t->trc_reader_nesting, nesting); return; // We assume shallow reader nesting. diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 04a18e01b362..3d0cf3722bb4 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2548,6 +2548,11 @@ static inline int skb_mac_header_was_set(const struct sk_buff *skb) return skb->mac_header != (typeof(skb->mac_header))~0U; } +static inline void skb_unset_mac_header(struct sk_buff *skb) +{ + skb->mac_header = (typeof(skb->mac_header))~0U; +} + static inline void skb_reset_mac_header(struct sk_buff *skb) { skb->mac_header = skb->data - skb->head; diff --git a/include/linux/sock_diag.h b/include/linux/sock_diag.h index 15fe980a27ea..0b9ecd8cf979 100644 --- a/include/linux/sock_diag.h +++ b/include/linux/sock_diag.h @@ -25,7 +25,19 @@ void sock_diag_unregister(const struct sock_diag_handler *h); void sock_diag_register_inet_compat(int (*fn)(struct sk_buff *skb, struct nlmsghdr *nlh)); void sock_diag_unregister_inet_compat(int (*fn)(struct sk_buff *skb, struct nlmsghdr *nlh)); -u64 sock_gen_cookie(struct sock *sk); +u64 __sock_gen_cookie(struct sock *sk); + +static inline u64 sock_gen_cookie(struct sock *sk) +{ + u64 cookie; + + preempt_disable(); + cookie = __sock_gen_cookie(sk); + preempt_enable(); + + return cookie; +} + int sock_diag_check_cookie(struct sock *sk, const __u32 *cookie); void sock_diag_save_cookie(struct sock *sk, __u32 *cookie); diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index 00e83c877496..628e28903b8b 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -200,5 +200,6 @@ struct plat_stmmacenet_data { int has_xgmac; bool vlan_fail_q_en; u8 vlan_fail_q; + unsigned int eee_usecs_rate; }; #endif diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 8caac20556b4..9873e1c8cd16 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -484,6 +484,9 @@ struct hci_dev { enum suspended_state suspend_state; bool scanning_paused; bool suspended; + u8 wake_reason; + bdaddr_t wake_addr; + u8 wake_addr_type; wait_queue_head_t suspend_wait_q; DECLARE_BITMAP(suspend_tasks, __SUSPEND_NUM_TASKS); @@ -1750,6 +1753,9 @@ void mgmt_device_found(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 link_type, void mgmt_remote_name(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 link_type, u8 addr_type, s8 rssi, u8 *name, u8 name_len); void mgmt_discovering(struct hci_dev *hdev, u8 discovering); +void mgmt_suspending(struct hci_dev *hdev, u8 state); +void mgmt_resuming(struct hci_dev *hdev, u8 reason, bdaddr_t *bdaddr, + u8 addr_type); bool mgmt_powering_down(struct hci_dev *hdev); void mgmt_new_ltk(struct hci_dev *hdev, struct smp_ltk *key, bool persistent); void mgmt_new_irk(struct hci_dev *hdev, struct smp_irk *irk, bool persistent); diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 8f1e6a7a2df8..1d1232917de7 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -665,6 +665,8 @@ struct l2cap_ops { struct sk_buff *(*alloc_skb) (struct l2cap_chan *chan, unsigned long hdr_len, unsigned long len, int nb); + int (*filter) (struct l2cap_chan * chan, + struct sk_buff *skb); }; struct l2cap_conn { diff --git a/include/net/bluetooth/mgmt.h b/include/net/bluetooth/mgmt.h index beae5c3980f0..6b55155e05e9 100644 --- a/include/net/bluetooth/mgmt.h +++ b/include/net/bluetooth/mgmt.h @@ -572,6 +572,8 @@ struct mgmt_rp_add_advertising { #define MGMT_ADV_FLAG_SEC_1M BIT(7) #define MGMT_ADV_FLAG_SEC_2M BIT(8) #define MGMT_ADV_FLAG_SEC_CODED BIT(9) +#define MGMT_ADV_FLAG_CAN_SET_TX_POWER BIT(10) +#define MGMT_ADV_FLAG_HW_OFFLOAD BIT(11) #define MGMT_ADV_FLAG_SEC_MASK (MGMT_ADV_FLAG_SEC_1M | MGMT_ADV_FLAG_SEC_2M | \ MGMT_ADV_FLAG_SEC_CODED) @@ -840,6 +842,7 @@ struct mgmt_ev_device_connected { #define MGMT_DEV_DISCONN_LOCAL_HOST 0x02 #define MGMT_DEV_DISCONN_REMOTE 0x03 #define MGMT_DEV_DISCONN_AUTH_FAILURE 0x04 +#define MGMT_DEV_DISCONN_LOCAL_HOST_SUSPEND 0x05 #define MGMT_EV_DEVICE_DISCONNECTED 0x000C struct mgmt_ev_device_disconnected { @@ -1028,3 +1031,18 @@ struct mgmt_ev_adv_monitor_added { struct mgmt_ev_adv_monitor_removed { __le16 monitor_handle; } __packed; + +#define MGMT_EV_CONTROLLER_SUSPEND 0x002d +struct mgmt_ev_controller_suspend { + __u8 suspend_state; +} __packed; + +#define MGMT_EV_CONTROLLER_RESUME 0x002e +struct mgmt_ev_controller_resume { + __u8 wake_reason; + struct mgmt_addr_info addr; +} __packed; + +#define MGMT_WAKE_REASON_NON_BT_WAKE 0x0 +#define MGMT_WAKE_REASON_UNEXPECTED 0x1 +#define MGMT_WAKE_REASON_REMOTE_WAKE 0x2 diff --git a/include/net/bpf_sk_storage.h b/include/net/bpf_sk_storage.h index 119f4c9c3a9c..3c516dd07caf 100644 --- a/include/net/bpf_sk_storage.h +++ b/include/net/bpf_sk_storage.h @@ -20,8 +20,6 @@ void bpf_sk_storage_free(struct sock *sk); extern const struct bpf_func_proto bpf_sk_storage_get_proto; extern const struct bpf_func_proto bpf_sk_storage_delete_proto; -extern const struct bpf_func_proto sk_storage_get_btf_proto; -extern const struct bpf_func_proto sk_storage_delete_btf_proto; struct bpf_local_storage_elem; struct bpf_sk_storage_diag; diff --git a/include/net/caif/caif_spi.h b/include/net/caif/caif_spi.h deleted file mode 100644 index a0bf4cbce71b..000000000000 --- a/include/net/caif/caif_spi.h +++ /dev/null @@ -1,155 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Daniel Martensson / Daniel.Martensson@stericsson.com - */ - -#ifndef CAIF_SPI_H_ -#define CAIF_SPI_H_ - -#include <net/caif/caif_device.h> - -#define SPI_CMD_WR 0x00 -#define SPI_CMD_RD 0x01 -#define SPI_CMD_EOT 0x02 -#define SPI_CMD_IND 0x04 - -#define SPI_DMA_BUF_LEN 8192 - -#define WL_SZ 2 /* 16 bits. */ -#define SPI_CMD_SZ 4 /* 32 bits. */ -#define SPI_IND_SZ 4 /* 32 bits. */ - -#define SPI_XFER 0 -#define SPI_SS_ON 1 -#define SPI_SS_OFF 2 -#define SPI_TERMINATE 3 - -/* Minimum time between different levels is 50 microseconds. */ -#define MIN_TRANSITION_TIME_USEC 50 - -/* Defines for calculating duration of SPI transfers for a particular - * number of bytes. - */ -#define SPI_MASTER_CLK_MHZ 13 -#define SPI_XFER_TIME_USEC(bytes, clk) (((bytes) * 8) / clk) - -/* Normally this should be aligned on the modem in order to benefit from full - * duplex transfers. However a size of 8188 provokes errors when running with - * the modem. These errors occur when packet sizes approaches 4 kB of data. - */ -#define CAIF_MAX_SPI_FRAME 4092 - -/* Maximum number of uplink CAIF frames that can reside in the same SPI frame. - * This number should correspond with the modem setting. The application side - * CAIF accepts any number of embedded downlink CAIF frames. - */ -#define CAIF_MAX_SPI_PKTS 9 - -/* Decides if SPI buffers should be prefilled with 0xFF pattern for easier - * debugging. Both TX and RX buffers will be filled before the transfer. - */ -#define CFSPI_DBG_PREFILL 0 - -/* Structure describing a SPI transfer. */ -struct cfspi_xfer { - u16 tx_dma_len; - u16 rx_dma_len; - void *va_tx[2]; - dma_addr_t pa_tx[2]; - void *va_rx; - dma_addr_t pa_rx; -}; - -/* Structure implemented by the SPI interface. */ -struct cfspi_ifc { - void (*ss_cb) (bool assert, struct cfspi_ifc *ifc); - void (*xfer_done_cb) (struct cfspi_ifc *ifc); - void *priv; -}; - -/* Structure implemented by SPI clients. */ -struct cfspi_dev { - int (*init_xfer) (struct cfspi_xfer *xfer, struct cfspi_dev *dev); - void (*sig_xfer) (bool xfer, struct cfspi_dev *dev); - struct cfspi_ifc *ifc; - char *name; - u32 clk_mhz; - void *priv; -}; - -/* Enumeration describing the CAIF SPI state. */ -enum cfspi_state { - CFSPI_STATE_WAITING = 0, - CFSPI_STATE_AWAKE, - CFSPI_STATE_FETCH_PKT, - CFSPI_STATE_GET_NEXT, - CFSPI_STATE_INIT_XFER, - CFSPI_STATE_WAIT_ACTIVE, - CFSPI_STATE_SIG_ACTIVE, - CFSPI_STATE_WAIT_XFER_DONE, - CFSPI_STATE_XFER_DONE, - CFSPI_STATE_WAIT_INACTIVE, - CFSPI_STATE_SIG_INACTIVE, - CFSPI_STATE_DELIVER_PKT, - CFSPI_STATE_MAX, -}; - -/* Structure implemented by SPI physical interfaces. */ -struct cfspi { - struct caif_dev_common cfdev; - struct net_device *ndev; - struct platform_device *pdev; - struct sk_buff_head qhead; - struct sk_buff_head chead; - u16 cmd; - u16 tx_cpck_len; - u16 tx_npck_len; - u16 rx_cpck_len; - u16 rx_npck_len; - struct cfspi_ifc ifc; - struct cfspi_xfer xfer; - struct cfspi_dev *dev; - unsigned long state; - struct work_struct work; - struct workqueue_struct *wq; - struct list_head list; - int flow_off_sent; - u32 qd_low_mark; - u32 qd_high_mark; - struct completion comp; - wait_queue_head_t wait; - spinlock_t lock; - bool flow_stop; - bool slave; - bool slave_talked; -#ifdef CONFIG_DEBUG_FS - enum cfspi_state dbg_state; - u16 pcmd; - u16 tx_ppck_len; - u16 rx_ppck_len; - struct dentry *dbgfs_dir; - struct dentry *dbgfs_state; - struct dentry *dbgfs_frame; -#endif /* CONFIG_DEBUG_FS */ -}; - -extern int spi_frm_align; -extern int spi_up_head_align; -extern int spi_up_tail_align; -extern int spi_down_head_align; -extern int spi_down_tail_align; -extern struct platform_driver cfspi_spi_driver; - -void cfspi_dbg_state(struct cfspi *cfspi, int state); -int cfspi_xmitfrm(struct cfspi *cfspi, u8 *buf, size_t len); -int cfspi_xmitlen(struct cfspi *cfspi); -int cfspi_rxfrm(struct cfspi *cfspi, u8 *buf, size_t len); -int cfspi_spi_remove(struct platform_device *pdev); -int cfspi_spi_probe(struct platform_device *pdev); -int cfspi_xmitfrm(struct cfspi *cfspi, u8 *buf, size_t len); -int cfspi_xmitlen(struct cfspi *cfspi); -int cfspi_rxfrm(struct cfspi *cfspi, u8 *buf, size_t len); -void cfspi_xfer(struct work_struct *work); - -#endif /* CAIF_SPI_H_ */ diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 10c2cc8f0efc..aee47f2b5709 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -269,13 +269,23 @@ struct ieee80211_rate { * struct ieee80211_he_obss_pd - AP settings for spatial reuse * * @enable: is the feature enabled. + * @sr_ctrl: The SR Control field of SRP element. + * @non_srg_max_offset: non-SRG maximum tx power offset * @min_offset: minimal tx power offset an associated station shall use * @max_offset: maximum tx power offset an associated station shall use + * @bss_color_bitmap: bitmap that indicates the BSS color values used by + * members of the SRG + * @partial_bssid_bitmap: bitmap that indicates the partial BSSID values + * used by members of the SRG */ struct ieee80211_he_obss_pd { bool enable; + u8 sr_ctrl; + u8 non_srg_max_offset; u8 min_offset; u8 max_offset; + u8 bss_color_bitmap[8]; + u8 partial_bssid_bitmap[8]; }; /** @@ -2096,6 +2106,27 @@ struct cfg80211_scan_info { }; /** + * struct cfg80211_scan_6ghz_params - relevant for 6 GHz only + * + * @short_bssid: short ssid to scan for + * @bssid: bssid to scan for + * @channel_idx: idx of the channel in the channel array in the scan request + * which the above info relvant to + * @unsolicited_probe: the AP transmits unsolicited probe response every 20 TU + * @short_ssid_valid: short_ssid is valid and can be used + * @psc_no_listen: when set, and the channel is a PSC channel, no need to wait + * 20 TUs before starting to send probe requests. + */ +struct cfg80211_scan_6ghz_params { + u32 short_ssid; + u32 channel_idx; + u8 bssid[ETH_ALEN]; + bool unsolicited_probe; + bool short_ssid_valid; + bool psc_no_listen; +}; + +/** * struct cfg80211_scan_request - scan request description * * @ssids: SSIDs to scan for (active scan only) @@ -2122,6 +2153,10 @@ struct cfg80211_scan_info { * @mac_addr_mask: MAC address mask used with randomisation, bits that * are 0 in the mask should be randomised, bits that are 1 should * be taken from the @mac_addr + * @scan_6ghz: relevant for split scan request only, + * true if this is the second scan request + * @n_6ghz_params: number of 6 GHz params + * @scan_6ghz_params: 6 GHz params * @bssid: BSSID to scan for (most commonly, the wildcard BSSID) */ struct cfg80211_scan_request { @@ -2149,6 +2184,9 @@ struct cfg80211_scan_request { struct cfg80211_scan_info info; bool notified; bool no_cck; + bool scan_6ghz; + u32 n_6ghz_params; + struct cfg80211_scan_6ghz_params *scan_6ghz_params; /* keep last */ struct ieee80211_channel *channels[]; @@ -2528,6 +2566,8 @@ enum cfg80211_assoc_req_flags { * @fils_nonces: FILS nonces (part of AAD) for protecting (Re)Association * Request/Response frame or %NULL if FILS is not used. This field starts * with 16 octets of STA Nonce followed by 16 octets of AP Nonce. + * @s1g_capa: S1G capability override + * @s1g_capa_mask: S1G capability override mask */ struct cfg80211_assoc_request { struct cfg80211_bss *bss; @@ -2542,6 +2582,7 @@ struct cfg80211_assoc_request { const u8 *fils_kek; size_t fils_kek_len; const u8 *fils_nonces; + struct ieee80211_s1g_cap s1g_capa, s1g_capa_mask; }; /** @@ -4217,6 +4258,8 @@ struct cfg80211_ops { /** * enum wiphy_flags - wiphy capability flags * + * @WIPHY_FLAG_SPLIT_SCAN_6GHZ: if set to true, the scan request will be split + * into two, first for legacy bands and second for UHB. * @WIPHY_FLAG_NETNS_OK: if not set, do not allow changing the netns of this * wiphy at all * @WIPHY_FLAG_PS_ON_BY_DEFAULT: if set to true, powersave will be enabled @@ -4260,7 +4303,7 @@ struct cfg80211_ops { enum wiphy_flags { WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK = BIT(0), /* use hole at 1 */ - /* use hole at 2 */ + WIPHY_FLAG_SPLIT_SCAN_6GHZ = BIT(2), WIPHY_FLAG_NETNS_OK = BIT(3), WIPHY_FLAG_PS_ON_BY_DEFAULT = BIT(4), WIPHY_FLAG_4ADDR_AP = BIT(5), diff --git a/include/net/devlink.h b/include/net/devlink.h index 7339bf9ba6b4..1c286e9a3590 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -625,6 +625,22 @@ struct devlink_health_reporter_ops { }; /** + * struct devlink_trap_metadata - Packet trap metadata. + * @trap_name: Trap name. + * @trap_group_name: Trap group name. + * @input_dev: Input netdevice. + * @fa_cookie: Flow action user cookie. + * @trap_type: Trap type. + */ +struct devlink_trap_metadata { + const char *trap_name; + const char *trap_group_name; + struct net_device *input_dev; + const struct flow_action_cookie *fa_cookie; + enum devlink_trap_type trap_type; +}; + +/** * struct devlink_trap_policer - Immutable packet trap policer attributes. * @id: Policer identifier. * @init_rate: Initial rate in packets / sec. diff --git a/include/net/drop_monitor.h b/include/net/drop_monitor.h deleted file mode 100644 index 3f5b6ddb3179..000000000000 --- a/include/net/drop_monitor.h +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ - -#ifndef _NET_DROP_MONITOR_H_ -#define _NET_DROP_MONITOR_H_ - -#include <linux/ktime.h> -#include <linux/netdevice.h> -#include <linux/skbuff.h> -#include <net/flow_offload.h> - -/** - * struct net_dm_hw_metadata - Hardware-supplied packet metadata. - * @trap_group_name: Hardware trap group name. - * @trap_name: Hardware trap name. - * @input_dev: Input netdevice. - * @fa_cookie: Flow action user cookie. - */ -struct net_dm_hw_metadata { - const char *trap_group_name; - const char *trap_name; - struct net_device *input_dev; - const struct flow_action_cookie *fa_cookie; -}; - -#if IS_REACHABLE(CONFIG_NET_DROP_MONITOR) -void net_dm_hw_report(struct sk_buff *skb, - const struct net_dm_hw_metadata *hw_metadata); -#else -static inline void -net_dm_hw_report(struct sk_buff *skb, - const struct net_dm_hw_metadata *hw_metadata) -{ -} -#endif - -#endif /* _NET_DROP_MONITOR_H_ */ diff --git a/include/net/dsa.h b/include/net/dsa.h index b502a63d196e..8b0696e08cac 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -308,6 +308,14 @@ struct dsa_switch { */ bool configure_vlan_while_not_filtering; + /* If the switch driver always programs the CPU port as egress tagged + * despite the VLAN configuration indicating otherwise, then setting + * @untag_bridge_pvid will force the DSA receive path to pop the bridge's + * default_pvid VLAN tagged frames to offer a consistent behavior + * between a vlan_filtering=0 and vlan_filtering=1 bridge device. + */ + bool untag_bridge_pvid; + /* In case vlan_filtering_is_global is set, the VLAN awareness state * should be retrieved from here and not from the per-port settings. */ diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index dc763ca9413c..7338b3865a2a 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -110,7 +110,7 @@ struct inet_connection_sock { __u8 pending; /* ACK is pending */ __u8 quick; /* Scheduled number of quick acks */ __u8 pingpong; /* The session is interactive */ - __u8 blocked; /* Delayed ACK was blocked by socket lock */ + __u8 retry; /* Number of attempts */ __u32 ato; /* Predicted tick of soft clock */ unsigned long timeout; /* Currently scheduled timeout */ __u32 lrcvtime; /* timestamp of last received data packet */ @@ -198,7 +198,8 @@ static inline void inet_csk_clear_xmit_timer(struct sock *sk, const int what) sk_stop_timer(sk, &icsk->icsk_retransmit_timer); #endif } else if (what == ICSK_TIME_DACK) { - icsk->icsk_ack.blocked = icsk->icsk_ack.pending = 0; + icsk->icsk_ack.pending = 0; + icsk->icsk_ack.retry = 0; #ifdef INET_CSK_CLEAR_TIMERS sk_stop_timer(sk, &icsk->icsk_delack_timer); #endif diff --git a/include/net/mac80211.h b/include/net/mac80211.h index e90089d104b0..4747d446179a 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -627,6 +627,7 @@ struct ieee80211_fils_discovery { * @fils_discovery: FILS discovery configuration * @unsol_bcast_probe_resp_interval: Unsolicited broadcast probe response * interval. + * @s1g: BSS is S1G BSS (affects Association Request format). */ struct ieee80211_bss_conf { const u8 *bssid; @@ -696,6 +697,7 @@ struct ieee80211_bss_conf { struct cfg80211_he_bss_color he_bss_color; struct ieee80211_fils_discovery fils_discovery; u32 unsol_bcast_probe_resp_interval; + bool s1g; }; /** @@ -833,6 +835,8 @@ enum mac80211_tx_info_flags { #define IEEE80211_TX_CTL_STBC_SHIFT 23 +#define IEEE80211_TX_RC_S1G_MCS IEEE80211_TX_RC_VHT_MCS + /** * enum mac80211_tx_control_flags - flags to describe transmit control * @@ -5403,11 +5407,15 @@ void ieee80211_sched_scan_stopped(struct ieee80211_hw *hw); * @IEEE80211_IFACE_ITER_RESUME_ALL: During resume, iterate over all * interfaces, even if they haven't been re-added to the driver yet. * @IEEE80211_IFACE_ITER_ACTIVE: Iterate only active interfaces (netdev is up). + * @IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER: Skip any interfaces where SDATA + * is not in the driver. This may fix crashes during firmware recovery + * for instance. */ enum ieee80211_interface_iteration_flags { IEEE80211_IFACE_ITER_NORMAL = 0, IEEE80211_IFACE_ITER_RESUME_ALL = BIT(0), IEEE80211_IFACE_ITER_ACTIVE = BIT(1), + IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER = BIT(2), }; /** diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h index 2ee5901bec7a..22bc07f4b043 100644 --- a/include/net/net_namespace.h +++ b/include/net/net_namespace.h @@ -230,7 +230,7 @@ extern struct list_head net_namespace_list; struct net *get_net_ns_by_pid(pid_t pid); struct net *get_net_ns_by_fd(int fd); -u64 net_gen_cookie(struct net *net); +u64 __net_gen_cookie(struct net *net); #ifdef CONFIG_SYSCTL void ipx_register_sysctl(void); diff --git a/include/net/netlink.h b/include/net/netlink.h index b2cf34f53e55..9e7eca961a98 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -1938,7 +1938,8 @@ void nla_get_range_signed(const struct nla_policy *pt, int netlink_policy_dump_start(const struct nla_policy *policy, unsigned int maxtype, unsigned long *state); -bool netlink_policy_dump_loop(unsigned long *state); +bool netlink_policy_dump_loop(unsigned long state); int netlink_policy_dump_write(struct sk_buff *skb, unsigned long state); +void netlink_policy_dump_free(unsigned long state); #endif diff --git a/include/net/smc.h b/include/net/smc.h index 646feb4bc75f..e441aa97ad61 100644 --- a/include/net/smc.h +++ b/include/net/smc.h @@ -37,6 +37,8 @@ struct smcd_dmb { #define ISM_EVENT_GID 1 #define ISM_EVENT_SWR 2 +#define ISM_RESERVED_VLANID 0x1FFF + #define ISM_ERROR 0xFFFF struct smcd_event { @@ -63,6 +65,8 @@ struct smcd_ops { int (*move_data)(struct smcd_dev *dev, u64 dmb_tok, unsigned int idx, bool sf, unsigned int offset, void *data, unsigned int size); + void (*get_system_eid)(struct smcd_dev *dev, u8 **eid); + u16 (*get_chid)(struct smcd_dev *dev); }; struct smcd_dev { diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h index 94bb7a882250..2ea453dac876 100644 --- a/include/net/udp_tunnel.h +++ b/include/net/udp_tunnel.h @@ -200,11 +200,27 @@ enum udp_tunnel_nic_info_flags { UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN = BIT(3), }; +struct udp_tunnel_nic; + +#define UDP_TUNNEL_NIC_MAX_SHARING_DEVICES (U16_MAX / 2) + +struct udp_tunnel_nic_shared { + struct udp_tunnel_nic *udp_tunnel_nic_info; + + struct list_head devices; +}; + +struct udp_tunnel_nic_shared_node { + struct net_device *dev; + struct list_head list; +}; + /** * struct udp_tunnel_nic_info - driver UDP tunnel offload information * @set_port: callback for adding a new port * @unset_port: callback for removing a port * @sync_table: callback for syncing the entire port table at once + * @shared: reference to device global state (optional) * @flags: device flags from enum udp_tunnel_nic_info_flags * @tables: UDP port tables this device has * @tables.n_entries: number of entries in this table @@ -213,6 +229,12 @@ enum udp_tunnel_nic_info_flags { * Drivers are expected to provide either @set_port and @unset_port callbacks * or the @sync_table callback. Callbacks are invoked with rtnl lock held. * + * Devices which (misguidedly) share the UDP tunnel port table across multiple + * netdevs should allocate an instance of struct udp_tunnel_nic_shared and + * point @shared at it. + * There must never be more than %UDP_TUNNEL_NIC_MAX_SHARING_DEVICES devices + * sharing a table. + * * Known limitations: * - UDP tunnel port notifications are fundamentally best-effort - * it is likely the driver will both see skbs which use a UDP tunnel port, @@ -234,6 +256,8 @@ struct udp_tunnel_nic_info { /* all at once */ int (*sync_table)(struct net_device *dev, unsigned int table); + struct udp_tunnel_nic_shared *shared; + unsigned int flags; struct udp_tunnel_nic_table_info { diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 2737d24ec244..53618a31634b 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -2000,6 +2000,39 @@ static inline int xfrm_tunnel_check(struct sk_buff *skb, struct xfrm_state *x, return 0; } +extern const int xfrm_msg_min[XFRM_NR_MSGTYPES]; +extern const struct nla_policy xfrma_policy[XFRMA_MAX+1]; + +struct xfrm_translator { + /* Allocate frag_list and put compat translation there */ + int (*alloc_compat)(struct sk_buff *skb, const struct nlmsghdr *src); + + /* Allocate nlmsg with 64-bit translaton of received 32-bit message */ + struct nlmsghdr *(*rcv_msg_compat)(const struct nlmsghdr *nlh, + int maxtype, const struct nla_policy *policy, + struct netlink_ext_ack *extack); + + /* Translate 32-bit user_policy from sockptr */ + int (*xlate_user_policy_sockptr)(u8 **pdata32, int optlen); + + struct module *owner; +}; + +#if IS_ENABLED(CONFIG_XFRM_USER_COMPAT) +extern int xfrm_register_translator(struct xfrm_translator *xtr); +extern int xfrm_unregister_translator(struct xfrm_translator *xtr); +extern struct xfrm_translator *xfrm_get_translator(void); +extern void xfrm_put_translator(struct xfrm_translator *xtr); +#else +static inline struct xfrm_translator *xfrm_get_translator(void) +{ + return NULL; +} +static inline void xfrm_put_translator(struct xfrm_translator *xtr) +{ +} +#endif + #if IS_ENABLED(CONFIG_IPV6) static inline bool xfrm6_local_dontfrag(const struct sock *sk) { diff --git a/include/soc/mscc/ocelot.h b/include/soc/mscc/ocelot.h index 3093385f6147..46608494616f 100644 --- a/include/soc/mscc/ocelot.h +++ b/include/soc/mscc/ocelot.h @@ -123,6 +123,8 @@ enum ocelot_target { QSYS, REW, SYS, + S0, + S1, S2, HSIO, PTP, @@ -393,13 +395,6 @@ enum ocelot_reg { SYS_CM_DATA_RD, SYS_CM_OP, SYS_CM_DATA, - S2_CORE_UPDATE_CTRL = S2 << TARGET_OFFSET, - S2_CORE_MV_CFG, - S2_CACHE_ENTRY_DAT, - S2_CACHE_MASK_DAT, - S2_CACHE_ACTION_DAT, - S2_CACHE_CNT_DAT, - S2_CACHE_TG_DAT, PTP_PIN_CFG = PTP << TARGET_OFFSET, PTP_PIN_TOD_SEC_MSB, PTP_PIN_TOD_SEC_LSB, @@ -518,6 +513,29 @@ enum ocelot_regfield { REGFIELD_MAX }; +enum { + /* VCAP_CORE_CFG */ + VCAP_CORE_UPDATE_CTRL, + VCAP_CORE_MV_CFG, + /* VCAP_CORE_CACHE */ + VCAP_CACHE_ENTRY_DAT, + VCAP_CACHE_MASK_DAT, + VCAP_CACHE_ACTION_DAT, + VCAP_CACHE_CNT_DAT, + VCAP_CACHE_TG_DAT, + /* VCAP_CONST */ + VCAP_CONST_VCAP_VER, + VCAP_CONST_ENTRY_WIDTH, + VCAP_CONST_ENTRY_CNT, + VCAP_CONST_ENTRY_SWCNT, + VCAP_CONST_ENTRY_TG_WIDTH, + VCAP_CONST_ACTION_DEF_CNT, + VCAP_CONST_ACTION_WIDTH, + VCAP_CONST_CNT_WIDTH, + VCAP_CONST_CORE_CNT, + VCAP_CONST_IF_CNT, +}; + enum ocelot_ptp_pins { PTP_PIN_0, PTP_PIN_1, @@ -541,6 +559,8 @@ enum ocelot_tag_prefix { struct ocelot; struct ocelot_ops { + struct net_device *(*port_to_netdev)(struct ocelot *ocelot, int port); + int (*netdev_to_port)(struct net_device *dev); int (*reset)(struct ocelot *ocelot); u16 (*wm_enc)(u16 value); }; @@ -613,11 +633,9 @@ struct ocelot { struct list_head multicast; - struct ocelot_vcap_block block; - - const struct vcap_field *vcap_is2_keys; - const struct vcap_field *vcap_is2_actions; - const struct vcap_props *vcap; + struct list_head dummy_rules; + struct ocelot_vcap_block block[3]; + struct vcap_props *vcap; /* Workqueue to check statistics for overflow with its lock */ struct mutex stats_lock; @@ -661,6 +679,24 @@ struct ocelot_policer { #define ocelot_fields_write(ocelot, id, reg, val) regmap_fields_write((ocelot)->regfields[(reg)], (id), (val)) #define ocelot_fields_read(ocelot, id, reg, val) regmap_fields_read((ocelot)->regfields[(reg)], (id), (val)) +#define ocelot_target_read_ix(ocelot, target, reg, gi, ri) \ + __ocelot_target_read_ix(ocelot, target, reg, reg##_GSZ * (gi) + reg##_RSZ * (ri)) +#define ocelot_target_read_gix(ocelot, target, reg, gi) \ + __ocelot_target_read_ix(ocelot, target, reg, reg##_GSZ * (gi)) +#define ocelot_target_read_rix(ocelot, target, reg, ri) \ + __ocelot_target_read_ix(ocelot, target, reg, reg##_RSZ * (ri)) +#define ocelot_target_read(ocelot, target, reg) \ + __ocelot_target_read_ix(ocelot, target, reg, 0) + +#define ocelot_target_write_ix(ocelot, target, val, reg, gi, ri) \ + __ocelot_target_write_ix(ocelot, target, val, reg, reg##_GSZ * (gi) + reg##_RSZ * (ri)) +#define ocelot_target_write_gix(ocelot, target, val, reg, gi) \ + __ocelot_target_write_ix(ocelot, target, val, reg, reg##_GSZ * (gi)) +#define ocelot_target_write_rix(ocelot, target, val, reg, ri) \ + __ocelot_target_write_ix(ocelot, target, val, reg, reg##_RSZ * (ri)) +#define ocelot_target_write(ocelot, target, val, reg) \ + __ocelot_target_write_ix(ocelot, target, val, reg, 0) + /* I/O */ u32 ocelot_port_readl(struct ocelot_port *port, u32 reg); void ocelot_port_writel(struct ocelot_port *port, u32 val, u32 reg); @@ -668,6 +704,10 @@ u32 __ocelot_read_ix(struct ocelot *ocelot, u32 reg, u32 offset); void __ocelot_write_ix(struct ocelot *ocelot, u32 val, u32 reg, u32 offset); void __ocelot_rmw_ix(struct ocelot *ocelot, u32 val, u32 mask, u32 reg, u32 offset); +u32 __ocelot_target_read_ix(struct ocelot *ocelot, enum ocelot_target target, + u32 reg, u32 offset); +void __ocelot_target_write_ix(struct ocelot *ocelot, enum ocelot_target target, + u32 val, u32 reg, u32 offset); /* Hardware initialization */ int ocelot_regfields_init(struct ocelot *ocelot, diff --git a/include/soc/mscc/ocelot_vcap.h b/include/soc/mscc/ocelot_vcap.h index 5748373ab4d3..96300adf3648 100644 --- a/include/soc/mscc/ocelot_vcap.h +++ b/include/soc/mscc/ocelot_vcap.h @@ -6,17 +6,22 @@ #ifndef _OCELOT_VCAP_H_ #define _OCELOT_VCAP_H_ +#include <soc/mscc/ocelot.h> + /* ================================================================= * VCAP Common * ================================================================= */ enum { - /* VCAP_IS1, */ + VCAP_ES0, + VCAP_IS1, VCAP_IS2, - /* VCAP_ES0, */ + __VCAP_COUNT, }; +#define OCELOT_NUM_VCAP_BLOCKS __VCAP_COUNT + struct vcap_props { u16 tg_width; /* Type-group width (in bits) */ u16 sw_count; /* Sub word count */ @@ -33,6 +38,11 @@ struct vcap_props { } action_table[2]; u16 counter_words; /* Number of counter words */ u16 counter_width; /* Counter width (in bits) */ + + enum ocelot_target target; + + const struct vcap_field *keys; + const struct vcap_field *actions; }; /* VCAP Type-Group values */ @@ -41,6 +51,61 @@ struct vcap_props { #define VCAP_TG_HALF 2 /* Half entry */ #define VCAP_TG_QUARTER 3 /* Quarter entry */ +#define VCAP_CORE_UPDATE_CTRL_UPDATE_CMD(x) (((x) << 22) & GENMASK(24, 22)) +#define VCAP_CORE_UPDATE_CTRL_UPDATE_CMD_M GENMASK(24, 22) +#define VCAP_CORE_UPDATE_CTRL_UPDATE_CMD_X(x) (((x) & GENMASK(24, 22)) >> 22) +#define VCAP_CORE_UPDATE_CTRL_UPDATE_ENTRY_DIS BIT(21) +#define VCAP_CORE_UPDATE_CTRL_UPDATE_ACTION_DIS BIT(20) +#define VCAP_CORE_UPDATE_CTRL_UPDATE_CNT_DIS BIT(19) +#define VCAP_CORE_UPDATE_CTRL_UPDATE_ADDR(x) (((x) << 3) & GENMASK(18, 3)) +#define VCAP_CORE_UPDATE_CTRL_UPDATE_ADDR_M GENMASK(18, 3) +#define VCAP_CORE_UPDATE_CTRL_UPDATE_ADDR_X(x) (((x) & GENMASK(18, 3)) >> 3) +#define VCAP_CORE_UPDATE_CTRL_UPDATE_SHOT BIT(2) +#define VCAP_CORE_UPDATE_CTRL_CLEAR_CACHE BIT(1) +#define VCAP_CORE_UPDATE_CTRL_MV_TRAFFIC_IGN BIT(0) + +#define VCAP_CORE_MV_CFG_MV_NUM_POS(x) (((x) << 16) & GENMASK(31, 16)) +#define VCAP_CORE_MV_CFG_MV_NUM_POS_M GENMASK(31, 16) +#define VCAP_CORE_MV_CFG_MV_NUM_POS_X(x) (((x) & GENMASK(31, 16)) >> 16) +#define VCAP_CORE_MV_CFG_MV_SIZE(x) ((x) & GENMASK(15, 0)) +#define VCAP_CORE_MV_CFG_MV_SIZE_M GENMASK(15, 0) + +#define VCAP_CACHE_ENTRY_DAT_RSZ 0x4 + +#define VCAP_CACHE_MASK_DAT_RSZ 0x4 + +#define VCAP_CACHE_ACTION_DAT_RSZ 0x4 + +#define VCAP_CACHE_CNT_DAT_RSZ 0x4 + +#define VCAP_STICKY_VCAP_ROW_DELETED_STICKY BIT(0) + +#define TCAM_BIST_CTRL_TCAM_BIST BIT(1) +#define TCAM_BIST_CTRL_TCAM_INIT BIT(0) + +#define TCAM_BIST_CFG_TCAM_BIST_SOE_ENA BIT(8) +#define TCAM_BIST_CFG_TCAM_HCG_DIS BIT(7) +#define TCAM_BIST_CFG_TCAM_CG_DIS BIT(6) +#define TCAM_BIST_CFG_TCAM_BIAS(x) ((x) & GENMASK(5, 0)) +#define TCAM_BIST_CFG_TCAM_BIAS_M GENMASK(5, 0) + +#define TCAM_BIST_STAT_BIST_RT_ERR BIT(15) +#define TCAM_BIST_STAT_BIST_PENC_ERR BIT(14) +#define TCAM_BIST_STAT_BIST_COMP_ERR BIT(13) +#define TCAM_BIST_STAT_BIST_ADDR_ERR BIT(12) +#define TCAM_BIST_STAT_BIST_BL1E_ERR BIT(11) +#define TCAM_BIST_STAT_BIST_BL1_ERR BIT(10) +#define TCAM_BIST_STAT_BIST_BL0E_ERR BIT(9) +#define TCAM_BIST_STAT_BIST_BL0_ERR BIT(8) +#define TCAM_BIST_STAT_BIST_PH1_ERR BIT(7) +#define TCAM_BIST_STAT_BIST_PH0_ERR BIT(6) +#define TCAM_BIST_STAT_BIST_PV1_ERR BIT(5) +#define TCAM_BIST_STAT_BIST_PV0_ERR BIT(4) +#define TCAM_BIST_STAT_BIST_RUN BIT(3) +#define TCAM_BIST_STAT_BIST_ERR BIT(2) +#define TCAM_BIST_STAT_BIST_BUSY BIT(1) +#define TCAM_BIST_STAT_TCAM_RDY BIT(0) + /* ================================================================= * VCAP IS2 * ================================================================= @@ -202,4 +267,137 @@ enum vcap_is2_action_field { VCAP_IS2_ACT_HIT_CNT, }; +/* ================================================================= + * VCAP IS1 + * ================================================================= + */ + +/* IS1 half key types */ +#define IS1_TYPE_S1_NORMAL 0 +#define IS1_TYPE_S1_5TUPLE_IP4 1 + +/* IS1 full key types */ +#define IS1_TYPE_S1_NORMAL_IP6 0 +#define IS1_TYPE_S1_7TUPLE 1 +#define IS2_TYPE_S1_5TUPLE_IP6 2 + +enum { + IS1_ACTION_TYPE_NORMAL, + IS1_ACTION_TYPE_MAX, +}; + +enum vcap_is1_half_key_field { + VCAP_IS1_HK_TYPE, + VCAP_IS1_HK_LOOKUP, + VCAP_IS1_HK_IGR_PORT_MASK, + VCAP_IS1_HK_RSV, + VCAP_IS1_HK_OAM_Y1731, + VCAP_IS1_HK_L2_MC, + VCAP_IS1_HK_L2_BC, + VCAP_IS1_HK_IP_MC, + VCAP_IS1_HK_VLAN_TAGGED, + VCAP_IS1_HK_VLAN_DBL_TAGGED, + VCAP_IS1_HK_TPID, + VCAP_IS1_HK_VID, + VCAP_IS1_HK_DEI, + VCAP_IS1_HK_PCP, + /* Specific Fields for IS1 Half Key S1_NORMAL */ + VCAP_IS1_HK_L2_SMAC, + VCAP_IS1_HK_ETYPE_LEN, + VCAP_IS1_HK_ETYPE, + VCAP_IS1_HK_IP_SNAP, + VCAP_IS1_HK_IP4, + VCAP_IS1_HK_L3_FRAGMENT, + VCAP_IS1_HK_L3_FRAG_OFS_GT0, + VCAP_IS1_HK_L3_OPTIONS, + VCAP_IS1_HK_L3_DSCP, + VCAP_IS1_HK_L3_IP4_SIP, + VCAP_IS1_HK_TCP_UDP, + VCAP_IS1_HK_TCP, + VCAP_IS1_HK_L4_SPORT, + VCAP_IS1_HK_L4_RNG, + /* Specific Fields for IS1 Half Key S1_5TUPLE_IP4 */ + VCAP_IS1_HK_IP4_INNER_TPID, + VCAP_IS1_HK_IP4_INNER_VID, + VCAP_IS1_HK_IP4_INNER_DEI, + VCAP_IS1_HK_IP4_INNER_PCP, + VCAP_IS1_HK_IP4_IP4, + VCAP_IS1_HK_IP4_L3_FRAGMENT, + VCAP_IS1_HK_IP4_L3_FRAG_OFS_GT0, + VCAP_IS1_HK_IP4_L3_OPTIONS, + VCAP_IS1_HK_IP4_L3_DSCP, + VCAP_IS1_HK_IP4_L3_IP4_DIP, + VCAP_IS1_HK_IP4_L3_IP4_SIP, + VCAP_IS1_HK_IP4_L3_PROTO, + VCAP_IS1_HK_IP4_TCP_UDP, + VCAP_IS1_HK_IP4_TCP, + VCAP_IS1_HK_IP4_L4_RNG, + VCAP_IS1_HK_IP4_IP_PAYLOAD_S1_5TUPLE, +}; + +enum vcap_is1_action_field { + VCAP_IS1_ACT_DSCP_ENA, + VCAP_IS1_ACT_DSCP_VAL, + VCAP_IS1_ACT_QOS_ENA, + VCAP_IS1_ACT_QOS_VAL, + VCAP_IS1_ACT_DP_ENA, + VCAP_IS1_ACT_DP_VAL, + VCAP_IS1_ACT_PAG_OVERRIDE_MASK, + VCAP_IS1_ACT_PAG_VAL, + VCAP_IS1_ACT_RSV, + VCAP_IS1_ACT_VID_REPLACE_ENA, + VCAP_IS1_ACT_VID_ADD_VAL, + VCAP_IS1_ACT_FID_SEL, + VCAP_IS1_ACT_FID_VAL, + VCAP_IS1_ACT_PCP_DEI_ENA, + VCAP_IS1_ACT_PCP_VAL, + VCAP_IS1_ACT_DEI_VAL, + VCAP_IS1_ACT_VLAN_POP_CNT_ENA, + VCAP_IS1_ACT_VLAN_POP_CNT, + VCAP_IS1_ACT_CUSTOM_ACE_TYPE_ENA, + VCAP_IS1_ACT_HIT_STICKY, +}; + +/* ================================================================= + * VCAP ES0 + * ================================================================= + */ + +enum { + ES0_ACTION_TYPE_NORMAL, + ES0_ACTION_TYPE_MAX, +}; + +enum vcap_es0_key_field { + VCAP_ES0_EGR_PORT, + VCAP_ES0_IGR_PORT, + VCAP_ES0_RSV, + VCAP_ES0_L2_MC, + VCAP_ES0_L2_BC, + VCAP_ES0_VID, + VCAP_ES0_DP, + VCAP_ES0_PCP, +}; + +enum vcap_es0_action_field { + VCAP_ES0_ACT_PUSH_OUTER_TAG, + VCAP_ES0_ACT_PUSH_INNER_TAG, + VCAP_ES0_ACT_TAG_A_TPID_SEL, + VCAP_ES0_ACT_TAG_A_VID_SEL, + VCAP_ES0_ACT_TAG_A_PCP_SEL, + VCAP_ES0_ACT_TAG_A_DEI_SEL, + VCAP_ES0_ACT_TAG_B_TPID_SEL, + VCAP_ES0_ACT_TAG_B_VID_SEL, + VCAP_ES0_ACT_TAG_B_PCP_SEL, + VCAP_ES0_ACT_TAG_B_DEI_SEL, + VCAP_ES0_ACT_VID_A_VAL, + VCAP_ES0_ACT_PCP_A_VAL, + VCAP_ES0_ACT_DEI_A_VAL, + VCAP_ES0_ACT_VID_B_VAL, + VCAP_ES0_ACT_PCP_B_VAL, + VCAP_ES0_ACT_DEI_B_VAL, + VCAP_ES0_ACT_RSV, + VCAP_ES0_ACT_HIT_STICKY, +}; + #endif /* _OCELOT_VCAP_H_ */ diff --git a/include/trace/events/devlink.h b/include/trace/events/devlink.h index 6f60a78d9a7e..44d8e2981065 100644 --- a/include/trace/events/devlink.h +++ b/include/trace/events/devlink.h @@ -171,6 +171,43 @@ TRACE_EVENT(devlink_health_reporter_state_update, __entry->new_state) ); +/* + * Tracepoint for devlink packet trap: + */ +TRACE_EVENT(devlink_trap_report, + TP_PROTO(const struct devlink *devlink, struct sk_buff *skb, + const struct devlink_trap_metadata *metadata), + + TP_ARGS(devlink, skb, metadata), + + TP_STRUCT__entry( + __string(bus_name, devlink->dev->bus->name) + __string(dev_name, dev_name(devlink->dev)) + __string(driver_name, devlink->dev->driver->name) + __string(trap_name, metadata->trap_name) + __string(trap_group_name, metadata->trap_group_name) + __dynamic_array(char, input_dev_name, IFNAMSIZ) + ), + + TP_fast_assign( + struct net_device *input_dev = metadata->input_dev; + + __assign_str(bus_name, devlink->dev->bus->name); + __assign_str(dev_name, dev_name(devlink->dev)); + __assign_str(driver_name, devlink->dev->driver->name); + __assign_str(trap_name, metadata->trap_name); + __assign_str(trap_group_name, metadata->trap_group_name); + __assign_str(input_dev_name, + (input_dev ? input_dev->name : "NULL")); + ), + + TP_printk("bus_name=%s dev_name=%s driver_name=%s trap_name=%s " + "trap_group_name=%s input_dev_name=%s", __get_str(bus_name), + __get_str(dev_name), __get_str(driver_name), + __get_str(trap_name), __get_str(trap_group_name), + __get_str(input_dev_name)) +); + #endif /* _TRACE_DEVLINK_H */ /* This part must be outside protection */ diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index a22812561064..4f556cfcbfbe 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -414,6 +414,9 @@ enum { /* Enable memory-mapping BPF map */ BPF_F_MMAPABLE = (1U << 10), + +/* Share perf_event among processes */ + BPF_F_PRESERVE_ELEMS = (1U << 11), }; /* Flags for BPF_PROG_QUERY. */ @@ -424,6 +427,11 @@ enum { */ #define BPF_F_QUERY_EFFECTIVE (1U << 0) +/* Flags for BPF_PROG_TEST_RUN */ + +/* If set, run the test on the cpu specified by bpf_attr.test.cpu */ +#define BPF_F_TEST_RUN_ON_CPU (1U << 0) + /* type for BPF_ENABLE_STATS */ enum bpf_stats_type { /* enabled run_time_ns and run_cnt */ @@ -566,6 +574,8 @@ union bpf_attr { */ __aligned_u64 ctx_in; __aligned_u64 ctx_out; + __u32 flags; + __u32 cpu; } test; struct { /* anonymous struct used by BPF_*_GET_*_ID */ @@ -632,8 +642,13 @@ union bpf_attr { }; __u32 attach_type; /* attach type */ __u32 flags; /* extra flags */ - __aligned_u64 iter_info; /* extra bpf_iter_link_info */ - __u32 iter_info_len; /* iter_info length */ + union { + __u32 target_btf_id; /* btf_id of target to attach to */ + struct { + __aligned_u64 iter_info; /* extra bpf_iter_link_info */ + __u32 iter_info_len; /* iter_info length */ + }; + }; } link_create; struct { /* struct used by BPF_LINK_UPDATE command */ @@ -2512,7 +2527,7 @@ union bpf_attr { * result is from *reuse*\ **->socks**\ [] using the hash of the * tuple. * - * long bpf_sk_release(struct bpf_sock *sock) + * long bpf_sk_release(void *sock) * Description * Release the reference held by *sock*. *sock* must be a * non-**NULL** pointer that was returned from @@ -2692,7 +2707,7 @@ union bpf_attr { * result is from *reuse*\ **->socks**\ [] using the hash of the * tuple. * - * long bpf_tcp_check_syncookie(struct bpf_sock *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) + * long bpf_tcp_check_syncookie(void *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) * Description * Check whether *iph* and *th* contain a valid SYN cookie ACK for * the listening socket in *sk*. @@ -2861,6 +2876,7 @@ union bpf_attr { * 0 on success. * * **-ENOENT** if the bpf-local-storage cannot be found. + * **-EINVAL** if sk is not a fullsock (e.g. a request_sock). * * long bpf_send_signal(u32 sig) * Description @@ -2877,7 +2893,7 @@ union bpf_attr { * * **-EAGAIN** if bpf program can try again. * - * s64 bpf_tcp_gen_syncookie(struct bpf_sock *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) + * s64 bpf_tcp_gen_syncookie(void *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) * Description * Try to issue a SYN cookie for the packet with corresponding * IP/TCP headers, *iph* and *th*, on the listening socket in *sk*. @@ -3106,7 +3122,7 @@ union bpf_attr { * Return * The id is returned or 0 in case the id could not be retrieved. * - * long bpf_sk_assign(struct sk_buff *skb, struct bpf_sock *sk, u64 flags) + * long bpf_sk_assign(struct sk_buff *skb, void *sk, u64 flags) * Description * Helper is overloaded depending on BPF program type. This * description applies to **BPF_PROG_TYPE_SCHED_CLS** and @@ -3234,11 +3250,11 @@ union bpf_attr { * * **-EOVERFLOW** if an overflow happened: The same object will be tried again. * - * u64 bpf_sk_cgroup_id(struct bpf_sock *sk) + * u64 bpf_sk_cgroup_id(void *sk) * Description * Return the cgroup v2 id of the socket *sk*. * - * *sk* must be a non-**NULL** pointer to a full socket, e.g. one + * *sk* must be a non-**NULL** pointer to a socket, e.g. one * returned from **bpf_sk_lookup_xxx**\ (), * **bpf_sk_fullsock**\ (), etc. The format of returned id is * same as in **bpf_skb_cgroup_id**\ (). @@ -3248,7 +3264,7 @@ union bpf_attr { * Return * The id is returned or 0 in case the id could not be retrieved. * - * u64 bpf_sk_ancestor_cgroup_id(struct bpf_sock *sk, int ancestor_level) + * u64 bpf_sk_ancestor_cgroup_id(void *sk, int ancestor_level) * Description * Return id of cgroup v2 that is ancestor of cgroup associated * with the *sk* at the *ancestor_level*. The root cgroup is at @@ -3586,6 +3602,72 @@ union bpf_attr { * the data in *dst*. This is a wrapper of **copy_from_user**\ (). * Return * 0 on success, or a negative error in case of failure. + * + * long bpf_snprintf_btf(char *str, u32 str_size, struct btf_ptr *ptr, u32 btf_ptr_size, u64 flags) + * Description + * Use BTF to store a string representation of *ptr*->ptr in *str*, + * using *ptr*->type_id. This value should specify the type + * that *ptr*->ptr points to. LLVM __builtin_btf_type_id(type, 1) + * can be used to look up vmlinux BTF type ids. Traversing the + * data structure using BTF, the type information and values are + * stored in the first *str_size* - 1 bytes of *str*. Safe copy of + * the pointer data is carried out to avoid kernel crashes during + * operation. Smaller types can use string space on the stack; + * larger programs can use map data to store the string + * representation. + * + * The string can be subsequently shared with userspace via + * bpf_perf_event_output() or ring buffer interfaces. + * bpf_trace_printk() is to be avoided as it places too small + * a limit on string size to be useful. + * + * *flags* is a combination of + * + * **BTF_F_COMPACT** + * no formatting around type information + * **BTF_F_NONAME** + * no struct/union member names/types + * **BTF_F_PTR_RAW** + * show raw (unobfuscated) pointer values; + * equivalent to printk specifier %px. + * **BTF_F_ZERO** + * show zero-valued struct/union members; they + * are not displayed by default + * + * Return + * The number of bytes that were written (or would have been + * written if output had to be truncated due to string size), + * or a negative error in cases of failure. + * + * long bpf_seq_printf_btf(struct seq_file *m, struct btf_ptr *ptr, u32 ptr_size, u64 flags) + * Description + * Use BTF to write to seq_write a string representation of + * *ptr*->ptr, using *ptr*->type_id as per bpf_snprintf_btf(). + * *flags* are identical to those used for bpf_snprintf_btf. + * Return + * 0 on success or a negative error in case of failure. + * + * u64 bpf_skb_cgroup_classid(struct sk_buff *skb) + * Description + * See **bpf_get_cgroup_classid**\ () for the main description. + * This helper differs from **bpf_get_cgroup_classid**\ () in that + * the cgroup v1 net_cls class is retrieved only from the *skb*'s + * associated socket instead of the current process. + * Return + * The id is returned or 0 in case the id could not be retrieved. + * + * long bpf_redirect_neigh(u32 ifindex, u64 flags) + * Description + * Redirect the packet to another net device of index *ifindex* + * and fill in L2 addresses from neighboring subsystem. This helper + * is somewhat similar to **bpf_redirect**\ (), except that it + * fills in e.g. MAC addresses based on the L3 information from + * the packet. This helper is supported for IPv4 and IPv6 protocols. + * The *flags* argument is reserved and must be 0. The helper is + * currently only supported for tc BPF program types. + * Return + * The helper returns **TC_ACT_REDIRECT** on success or + * **TC_ACT_SHOT** on error. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -3737,6 +3819,10 @@ union bpf_attr { FN(inode_storage_delete), \ FN(d_path), \ FN(copy_from_user), \ + FN(snprintf_btf), \ + FN(seq_printf_btf), \ + FN(skb_cgroup_classid), \ + FN(redirect_neigh), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper @@ -4845,4 +4931,34 @@ struct bpf_sk_lookup { __u32 local_port; /* Host byte order */ }; +/* + * struct btf_ptr is used for typed pointer representation; the + * type id is used to render the pointer data as the appropriate type + * via the bpf_snprintf_btf() helper described above. A flags field - + * potentially to specify additional details about the BTF pointer + * (rather than its mode of display) - is included for future use. + * Display flags - BTF_F_* - are passed to bpf_snprintf_btf separately. + */ +struct btf_ptr { + void *ptr; + __u32 type_id; + __u32 flags; /* BTF ptr flags; unused at present. */ +}; + +/* + * Flags to control bpf_snprintf_btf() behaviour. + * - BTF_F_COMPACT: no formatting around type information + * - BTF_F_NONAME: no struct/union member names/types + * - BTF_F_PTR_RAW: show raw (unobfuscated) pointer values; + * equivalent to %px. + * - BTF_F_ZERO: show zero-valued struct/union members; they + * are not displayed by default + */ +enum { + BTF_F_COMPACT = (1ULL << 0), + BTF_F_NONAME = (1ULL << 1), + BTF_F_PTR_RAW = (1ULL << 2), + BTF_F_ZERO = (1ULL << 3), +}; + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h index 7b0face1bad5..ba467dc07852 100644 --- a/include/uapi/linux/devlink.h +++ b/include/uapi/linux/devlink.h @@ -13,6 +13,8 @@ #ifndef _UAPI_LINUX_DEVLINK_H_ #define _UAPI_LINUX_DEVLINK_H_ +#include <linux/const.h> + #define DEVLINK_GENL_NAME "devlink" #define DEVLINK_GENL_VERSION 0x1 #define DEVLINK_GENL_MCGRP_CONFIG_NAME "config" diff --git a/include/uapi/linux/l2tp.h b/include/uapi/linux/l2tp.h index 88a0d32b8c07..30c80d5ba4bf 100644 --- a/include/uapi/linux/l2tp.h +++ b/include/uapi/linux/l2tp.h @@ -144,6 +144,7 @@ enum { L2TP_ATTR_RX_OOS_PACKETS, /* u64 */ L2TP_ATTR_RX_ERRORS, /* u64 */ L2TP_ATTR_STATS_PAD, + L2TP_ATTR_RX_COOKIE_DISCARDS, /* u64 */ __L2TP_ATTR_STATS_MAX, }; diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index bdc90b8dfd24..47700a2b9af9 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -2521,6 +2521,12 @@ enum nl80211_commands { * unsolicited broadcast probe response. It is a nested attribute, see * &enum nl80211_unsol_bcast_probe_resp_attributes. * + * @NL80211_ATTR_S1G_CAPABILITY: S1G Capability information element (from + * association request when used with NL80211_CMD_NEW_STATION) + * @NL80211_ATTR_S1G_CAPABILITY_MASK: S1G Capability Information element + * override mask. Used with NL80211_ATTR_S1G_CAPABILITY in + * NL80211_CMD_ASSOCIATE or NL80211_CMD_CONNECT. + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -3007,6 +3013,9 @@ enum nl80211_attrs { NL80211_ATTR_UNSOL_BCAST_PROBE_RESP, + NL80211_ATTR_S1G_CAPABILITY, + NL80211_ATTR_S1G_CAPABILITY_MASK, + /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, @@ -4088,6 +4097,7 @@ enum nl80211_user_reg_hint_type { * receiving frames destined to the local BSS * @NL80211_SURVEY_INFO_MAX: highest survey info attribute number * currently defined + * @NL80211_SURVEY_INFO_FREQUENCY_OFFSET: center frequency offset in KHz * @__NL80211_SURVEY_INFO_AFTER_LAST: internal use */ enum nl80211_survey_info { @@ -4103,6 +4113,7 @@ enum nl80211_survey_info { NL80211_SURVEY_INFO_TIME_SCAN, NL80211_SURVEY_INFO_PAD, NL80211_SURVEY_INFO_TIME_BSS_RX, + NL80211_SURVEY_INFO_FREQUENCY_OFFSET, /* keep last */ __NL80211_SURVEY_INFO_AFTER_LAST, @@ -6059,6 +6070,8 @@ enum nl80211_timeout_reason { * @NL80211_SCAN_FLAG_FREQ_KHZ: report scan results with * %NL80211_ATTR_SCAN_FREQ_KHZ. This also means * %NL80211_ATTR_SCAN_FREQUENCIES will not be included. + * @NL80211_SCAN_FLAG_COLOCATED_6GHZ: scan for colocated APs reported by + * 2.4/5 GHz APs */ enum nl80211_scan_flags { NL80211_SCAN_FLAG_LOW_PRIORITY = 1<<0, @@ -6075,6 +6088,7 @@ enum nl80211_scan_flags { NL80211_SCAN_FLAG_RANDOM_SN = 1<<11, NL80211_SCAN_FLAG_MIN_PREQ_CONTENT = 1<<12, NL80211_SCAN_FLAG_FREQ_KHZ = 1<<13, + NL80211_SCAN_FLAG_COLOCATED_6GHZ = 1<<14, }; /** @@ -6977,6 +6991,13 @@ enum nl80211_peer_measurement_ftm_resp { * * @NL80211_HE_OBSS_PD_ATTR_MIN_OFFSET: the OBSS PD minimum tx power offset. * @NL80211_HE_OBSS_PD_ATTR_MAX_OFFSET: the OBSS PD maximum tx power offset. + * @NL80211_HE_OBSS_PD_ATTR_NON_SRG_MAX_OFFSET: the non-SRG OBSS PD maximum + * tx power offset. + * @NL80211_HE_OBSS_PD_ATTR_BSS_COLOR_BITMAP: bitmap that indicates the BSS color + * values used by members of the SRG. + * @NL80211_HE_OBSS_PD_ATTR_PARTIAL_BSSID_BITMAP: bitmap that indicates the partial + * BSSID values used by members of the SRG. + * @NL80211_HE_OBSS_PD_ATTR_SR_CTRL: The SR Control field of SRP element. * * @__NL80211_HE_OBSS_PD_ATTR_LAST: Internal * @NL80211_HE_OBSS_PD_ATTR_MAX: highest OBSS PD attribute. @@ -6986,6 +7007,10 @@ enum nl80211_obss_pd_attributes { NL80211_HE_OBSS_PD_ATTR_MIN_OFFSET, NL80211_HE_OBSS_PD_ATTR_MAX_OFFSET, + NL80211_HE_OBSS_PD_ATTR_NON_SRG_MAX_OFFSET, + NL80211_HE_OBSS_PD_ATTR_BSS_COLOR_BITMAP, + NL80211_HE_OBSS_PD_ATTR_PARTIAL_BSSID_BITMAP, + NL80211_HE_OBSS_PD_ATTR_SR_CTRL, /* keep last */ __NL80211_HE_OBSS_PD_ATTR_LAST, diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index e5fd31268ae0..bd777dd6f967 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -15,7 +15,8 @@ #include "map_in_map.h" #define ARRAY_CREATE_FLAG_MASK \ - (BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK) + (BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK | \ + BPF_F_PRESERVE_ELEMS) static void bpf_array_free_percpu(struct bpf_array *array) { @@ -64,6 +65,10 @@ int array_map_alloc_check(union bpf_attr *attr) attr->map_flags & BPF_F_MMAPABLE) return -EINVAL; + if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY && + attr->map_flags & BPF_F_PRESERVE_ELEMS) + return -EINVAL; + if (attr->value_size > KMALLOC_MAX_SIZE) /* if value_size is bigger, the user space won't be able to * access the elements. @@ -1134,6 +1139,9 @@ static void perf_event_fd_array_release(struct bpf_map *map, struct bpf_event_entry *ee; int i; + if (map->map_flags & BPF_F_PRESERVE_ELEMS) + return; + rcu_read_lock(); for (i = 0; i < array->map.max_entries; i++) { ee = READ_ONCE(array->ptrs[i]); @@ -1143,12 +1151,19 @@ static void perf_event_fd_array_release(struct bpf_map *map, rcu_read_unlock(); } +static void perf_event_fd_array_map_free(struct bpf_map *map) +{ + if (map->map_flags & BPF_F_PRESERVE_ELEMS) + bpf_fd_array_map_clear(map); + fd_array_map_free(map); +} + static int perf_event_array_map_btf_id; const struct bpf_map_ops perf_event_array_map_ops = { .map_meta_equal = bpf_map_meta_equal, .map_alloc_check = fd_array_map_alloc_check, .map_alloc = array_map_alloc, - .map_free = fd_array_map_free, + .map_free = perf_event_fd_array_map_free, .map_get_next_key = array_map_get_next_key, .map_lookup_elem = fd_array_map_lookup_elem, .map_delete_elem = fd_array_map_delete_elem, diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index 30833bbf3019..8f10e30ea0b0 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -88,8 +88,8 @@ static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size, mutex_lock(&seq->lock); if (!seq->buf) { - seq->size = PAGE_SIZE; - seq->buf = kmalloc(seq->size, GFP_KERNEL); + seq->size = PAGE_SIZE << 3; + seq->buf = kvmalloc(seq->size, GFP_KERNEL); if (!seq->buf) { err = -ENOMEM; goto done; diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 9cd1428c7199..78ea8a7bd27f 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -56,9 +56,9 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_inode_storage_delete: return &bpf_inode_storage_delete_proto; case BPF_FUNC_sk_storage_get: - return &sk_storage_get_btf_proto; + return &bpf_sk_storage_get_proto; case BPF_FUNC_sk_storage_delete: - return &sk_storage_delete_btf_proto; + return &bpf_sk_storage_delete_proto; default: return tracing_prog_func_proto(func_id, prog); } diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 5d3c36e13139..4d0ee7839fdb 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -284,6 +284,91 @@ static const char *btf_type_str(const struct btf_type *t) return btf_kind_str[BTF_INFO_KIND(t->info)]; } +/* Chunk size we use in safe copy of data to be shown. */ +#define BTF_SHOW_OBJ_SAFE_SIZE 32 + +/* + * This is the maximum size of a base type value (equivalent to a + * 128-bit int); if we are at the end of our safe buffer and have + * less than 16 bytes space we can't be assured of being able + * to copy the next type safely, so in such cases we will initiate + * a new copy. + */ +#define BTF_SHOW_OBJ_BASE_TYPE_SIZE 16 + +/* Type name size */ +#define BTF_SHOW_NAME_SIZE 80 + +/* + * Common data to all BTF show operations. Private show functions can add + * their own data to a structure containing a struct btf_show and consult it + * in the show callback. See btf_type_show() below. + * + * One challenge with showing nested data is we want to skip 0-valued + * data, but in order to figure out whether a nested object is all zeros + * we need to walk through it. As a result, we need to make two passes + * when handling structs, unions and arrays; the first path simply looks + * for nonzero data, while the second actually does the display. The first + * pass is signalled by show->state.depth_check being set, and if we + * encounter a non-zero value we set show->state.depth_to_show to + * the depth at which we encountered it. When we have completed the + * first pass, we will know if anything needs to be displayed if + * depth_to_show > depth. See btf_[struct,array]_show() for the + * implementation of this. + * + * Another problem is we want to ensure the data for display is safe to + * access. To support this, the anonymous "struct {} obj" tracks the data + * object and our safe copy of it. We copy portions of the data needed + * to the object "copy" buffer, but because its size is limited to + * BTF_SHOW_OBJ_COPY_LEN bytes, multiple copies may be required as we + * traverse larger objects for display. + * + * The various data type show functions all start with a call to + * btf_show_start_type() which returns a pointer to the safe copy + * of the data needed (or if BTF_SHOW_UNSAFE is specified, to the + * raw data itself). btf_show_obj_safe() is responsible for + * using copy_from_kernel_nofault() to update the safe data if necessary + * as we traverse the object's data. skbuff-like semantics are + * used: + * + * - obj.head points to the start of the toplevel object for display + * - obj.size is the size of the toplevel object + * - obj.data points to the current point in the original data at + * which our safe data starts. obj.data will advance as we copy + * portions of the data. + * + * In most cases a single copy will suffice, but larger data structures + * such as "struct task_struct" will require many copies. The logic in + * btf_show_obj_safe() handles the logic that determines if a new + * copy_from_kernel_nofault() is needed. + */ +struct btf_show { + u64 flags; + void *target; /* target of show operation (seq file, buffer) */ + void (*showfn)(struct btf_show *show, const char *fmt, va_list args); + const struct btf *btf; + /* below are used during iteration */ + struct { + u8 depth; + u8 depth_to_show; + u8 depth_check; + u8 array_member:1, + array_terminated:1; + u16 array_encoding; + u32 type_id; + int status; /* non-zero for error */ + const struct btf_type *type; + const struct btf_member *member; + char name[BTF_SHOW_NAME_SIZE]; /* space for member name/type */ + } state; + struct { + u32 size; + void *head; + void *data; + u8 safe[BTF_SHOW_OBJ_SAFE_SIZE]; + } obj; +}; + struct btf_kind_operations { s32 (*check_meta)(struct btf_verifier_env *env, const struct btf_type *t, @@ -300,9 +385,9 @@ struct btf_kind_operations { const struct btf_type *member_type); void (*log_details)(struct btf_verifier_env *env, const struct btf_type *t); - void (*seq_show)(const struct btf *btf, const struct btf_type *t, + void (*show)(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offsets, - struct seq_file *m); + struct btf_show *show); }; static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS]; @@ -679,6 +764,488 @@ bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, return true; } +/* Similar to btf_type_skip_modifiers() but does not skip typedefs. */ +static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, + u32 id) +{ + const struct btf_type *t = btf_type_by_id(btf, id); + + while (btf_type_is_modifier(t) && + BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) { + id = t->type; + t = btf_type_by_id(btf, t->type); + } + + return t; +} + +#define BTF_SHOW_MAX_ITER 10 + +#define BTF_KIND_BIT(kind) (1ULL << kind) + +/* + * Populate show->state.name with type name information. + * Format of type name is + * + * [.member_name = ] (type_name) + */ +static const char *btf_show_name(struct btf_show *show) +{ + /* BTF_MAX_ITER array suffixes "[]" */ + const char *array_suffixes = "[][][][][][][][][][]"; + const char *array_suffix = &array_suffixes[strlen(array_suffixes)]; + /* BTF_MAX_ITER pointer suffixes "*" */ + const char *ptr_suffixes = "**********"; + const char *ptr_suffix = &ptr_suffixes[strlen(ptr_suffixes)]; + const char *name = NULL, *prefix = "", *parens = ""; + const struct btf_member *m = show->state.member; + const struct btf_type *t = show->state.type; + const struct btf_array *array; + u32 id = show->state.type_id; + const char *member = NULL; + bool show_member = false; + u64 kinds = 0; + int i; + + show->state.name[0] = '\0'; + + /* + * Don't show type name if we're showing an array member; + * in that case we show the array type so don't need to repeat + * ourselves for each member. + */ + if (show->state.array_member) + return ""; + + /* Retrieve member name, if any. */ + if (m) { + member = btf_name_by_offset(show->btf, m->name_off); + show_member = strlen(member) > 0; + id = m->type; + } + + /* + * Start with type_id, as we have resolved the struct btf_type * + * via btf_modifier_show() past the parent typedef to the child + * struct, int etc it is defined as. In such cases, the type_id + * still represents the starting type while the struct btf_type * + * in our show->state points at the resolved type of the typedef. + */ + t = btf_type_by_id(show->btf, id); + if (!t) + return ""; + + /* + * The goal here is to build up the right number of pointer and + * array suffixes while ensuring the type name for a typedef + * is represented. Along the way we accumulate a list of + * BTF kinds we have encountered, since these will inform later + * display; for example, pointer types will not require an + * opening "{" for struct, we will just display the pointer value. + * + * We also want to accumulate the right number of pointer or array + * indices in the format string while iterating until we get to + * the typedef/pointee/array member target type. + * + * We start by pointing at the end of pointer and array suffix + * strings; as we accumulate pointers and arrays we move the pointer + * or array string backwards so it will show the expected number of + * '*' or '[]' for the type. BTF_SHOW_MAX_ITER of nesting of pointers + * and/or arrays and typedefs are supported as a precaution. + * + * We also want to get typedef name while proceeding to resolve + * type it points to so that we can add parentheses if it is a + * "typedef struct" etc. + */ + for (i = 0; i < BTF_SHOW_MAX_ITER; i++) { + + switch (BTF_INFO_KIND(t->info)) { + case BTF_KIND_TYPEDEF: + if (!name) + name = btf_name_by_offset(show->btf, + t->name_off); + kinds |= BTF_KIND_BIT(BTF_KIND_TYPEDEF); + id = t->type; + break; + case BTF_KIND_ARRAY: + kinds |= BTF_KIND_BIT(BTF_KIND_ARRAY); + parens = "["; + if (!t) + return ""; + array = btf_type_array(t); + if (array_suffix > array_suffixes) + array_suffix -= 2; + id = array->type; + break; + case BTF_KIND_PTR: + kinds |= BTF_KIND_BIT(BTF_KIND_PTR); + if (ptr_suffix > ptr_suffixes) + ptr_suffix -= 1; + id = t->type; + break; + default: + id = 0; + break; + } + if (!id) + break; + t = btf_type_skip_qualifiers(show->btf, id); + } + /* We may not be able to represent this type; bail to be safe */ + if (i == BTF_SHOW_MAX_ITER) + return ""; + + if (!name) + name = btf_name_by_offset(show->btf, t->name_off); + + switch (BTF_INFO_KIND(t->info)) { + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: + prefix = BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT ? + "struct" : "union"; + /* if it's an array of struct/union, parens is already set */ + if (!(kinds & (BTF_KIND_BIT(BTF_KIND_ARRAY)))) + parens = "{"; + break; + case BTF_KIND_ENUM: + prefix = "enum"; + break; + default: + break; + } + + /* pointer does not require parens */ + if (kinds & BTF_KIND_BIT(BTF_KIND_PTR)) + parens = ""; + /* typedef does not require struct/union/enum prefix */ + if (kinds & BTF_KIND_BIT(BTF_KIND_TYPEDEF)) + prefix = ""; + + if (!name) + name = ""; + + /* Even if we don't want type name info, we want parentheses etc */ + if (show->flags & BTF_SHOW_NONAME) + snprintf(show->state.name, sizeof(show->state.name), "%s", + parens); + else + snprintf(show->state.name, sizeof(show->state.name), + "%s%s%s(%s%s%s%s%s%s)%s", + /* first 3 strings comprise ".member = " */ + show_member ? "." : "", + show_member ? member : "", + show_member ? " = " : "", + /* ...next is our prefix (struct, enum, etc) */ + prefix, + strlen(prefix) > 0 && strlen(name) > 0 ? " " : "", + /* ...this is the type name itself */ + name, + /* ...suffixed by the appropriate '*', '[]' suffixes */ + strlen(ptr_suffix) > 0 ? " " : "", ptr_suffix, + array_suffix, parens); + + return show->state.name; +} + +static const char *__btf_show_indent(struct btf_show *show) +{ + const char *indents = " "; + const char *indent = &indents[strlen(indents)]; + + if ((indent - show->state.depth) >= indents) + return indent - show->state.depth; + return indents; +} + +static const char *btf_show_indent(struct btf_show *show) +{ + return show->flags & BTF_SHOW_COMPACT ? "" : __btf_show_indent(show); +} + +static const char *btf_show_newline(struct btf_show *show) +{ + return show->flags & BTF_SHOW_COMPACT ? "" : "\n"; +} + +static const char *btf_show_delim(struct btf_show *show) +{ + if (show->state.depth == 0) + return ""; + + if ((show->flags & BTF_SHOW_COMPACT) && show->state.type && + BTF_INFO_KIND(show->state.type->info) == BTF_KIND_UNION) + return "|"; + + return ","; +} + +__printf(2, 3) static void btf_show(struct btf_show *show, const char *fmt, ...) +{ + va_list args; + + if (!show->state.depth_check) { + va_start(args, fmt); + show->showfn(show, fmt, args); + va_end(args); + } +} + +/* Macros are used here as btf_show_type_value[s]() prepends and appends + * format specifiers to the format specifier passed in; these do the work of + * adding indentation, delimiters etc while the caller simply has to specify + * the type value(s) in the format specifier + value(s). + */ +#define btf_show_type_value(show, fmt, value) \ + do { \ + if ((value) != 0 || (show->flags & BTF_SHOW_ZERO) || \ + show->state.depth == 0) { \ + btf_show(show, "%s%s" fmt "%s%s", \ + btf_show_indent(show), \ + btf_show_name(show), \ + value, btf_show_delim(show), \ + btf_show_newline(show)); \ + if (show->state.depth > show->state.depth_to_show) \ + show->state.depth_to_show = show->state.depth; \ + } \ + } while (0) + +#define btf_show_type_values(show, fmt, ...) \ + do { \ + btf_show(show, "%s%s" fmt "%s%s", btf_show_indent(show), \ + btf_show_name(show), \ + __VA_ARGS__, btf_show_delim(show), \ + btf_show_newline(show)); \ + if (show->state.depth > show->state.depth_to_show) \ + show->state.depth_to_show = show->state.depth; \ + } while (0) + +/* How much is left to copy to safe buffer after @data? */ +static int btf_show_obj_size_left(struct btf_show *show, void *data) +{ + return show->obj.head + show->obj.size - data; +} + +/* Is object pointed to by @data of @size already copied to our safe buffer? */ +static bool btf_show_obj_is_safe(struct btf_show *show, void *data, int size) +{ + return data >= show->obj.data && + (data + size) < (show->obj.data + BTF_SHOW_OBJ_SAFE_SIZE); +} + +/* + * If object pointed to by @data of @size falls within our safe buffer, return + * the equivalent pointer to the same safe data. Assumes + * copy_from_kernel_nofault() has already happened and our safe buffer is + * populated. + */ +static void *__btf_show_obj_safe(struct btf_show *show, void *data, int size) +{ + if (btf_show_obj_is_safe(show, data, size)) + return show->obj.safe + (data - show->obj.data); + return NULL; +} + +/* + * Return a safe-to-access version of data pointed to by @data. + * We do this by copying the relevant amount of information + * to the struct btf_show obj.safe buffer using copy_from_kernel_nofault(). + * + * If BTF_SHOW_UNSAFE is specified, just return data as-is; no + * safe copy is needed. + * + * Otherwise we need to determine if we have the required amount + * of data (determined by the @data pointer and the size of the + * largest base type we can encounter (represented by + * BTF_SHOW_OBJ_BASE_TYPE_SIZE). Having that much data ensures + * that we will be able to print some of the current object, + * and if more is needed a copy will be triggered. + * Some objects such as structs will not fit into the buffer; + * in such cases additional copies when we iterate over their + * members may be needed. + * + * btf_show_obj_safe() is used to return a safe buffer for + * btf_show_start_type(); this ensures that as we recurse into + * nested types we always have safe data for the given type. + * This approach is somewhat wasteful; it's possible for example + * that when iterating over a large union we'll end up copying the + * same data repeatedly, but the goal is safety not performance. + * We use stack data as opposed to per-CPU buffers because the + * iteration over a type can take some time, and preemption handling + * would greatly complicate use of the safe buffer. + */ +static void *btf_show_obj_safe(struct btf_show *show, + const struct btf_type *t, + void *data) +{ + const struct btf_type *rt; + int size_left, size; + void *safe = NULL; + + if (show->flags & BTF_SHOW_UNSAFE) + return data; + + rt = btf_resolve_size(show->btf, t, &size); + if (IS_ERR(rt)) { + show->state.status = PTR_ERR(rt); + return NULL; + } + + /* + * Is this toplevel object? If so, set total object size and + * initialize pointers. Otherwise check if we still fall within + * our safe object data. + */ + if (show->state.depth == 0) { + show->obj.size = size; + show->obj.head = data; + } else { + /* + * If the size of the current object is > our remaining + * safe buffer we _may_ need to do a new copy. However + * consider the case of a nested struct; it's size pushes + * us over the safe buffer limit, but showing any individual + * struct members does not. In such cases, we don't need + * to initiate a fresh copy yet; however we definitely need + * at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes left + * in our buffer, regardless of the current object size. + * The logic here is that as we resolve types we will + * hit a base type at some point, and we need to be sure + * the next chunk of data is safely available to display + * that type info safely. We cannot rely on the size of + * the current object here because it may be much larger + * than our current buffer (e.g. task_struct is 8k). + * All we want to do here is ensure that we can print the + * next basic type, which we can if either + * - the current type size is within the safe buffer; or + * - at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes are left in + * the safe buffer. + */ + safe = __btf_show_obj_safe(show, data, + min(size, + BTF_SHOW_OBJ_BASE_TYPE_SIZE)); + } + + /* + * We need a new copy to our safe object, either because we haven't + * yet copied and are intializing safe data, or because the data + * we want falls outside the boundaries of the safe object. + */ + if (!safe) { + size_left = btf_show_obj_size_left(show, data); + if (size_left > BTF_SHOW_OBJ_SAFE_SIZE) + size_left = BTF_SHOW_OBJ_SAFE_SIZE; + show->state.status = copy_from_kernel_nofault(show->obj.safe, + data, size_left); + if (!show->state.status) { + show->obj.data = data; + safe = show->obj.safe; + } + } + + return safe; +} + +/* + * Set the type we are starting to show and return a safe data pointer + * to be used for showing the associated data. + */ +static void *btf_show_start_type(struct btf_show *show, + const struct btf_type *t, + u32 type_id, void *data) +{ + show->state.type = t; + show->state.type_id = type_id; + show->state.name[0] = '\0'; + + return btf_show_obj_safe(show, t, data); +} + +static void btf_show_end_type(struct btf_show *show) +{ + show->state.type = NULL; + show->state.type_id = 0; + show->state.name[0] = '\0'; +} + +static void *btf_show_start_aggr_type(struct btf_show *show, + const struct btf_type *t, + u32 type_id, void *data) +{ + void *safe_data = btf_show_start_type(show, t, type_id, data); + + if (!safe_data) + return safe_data; + + btf_show(show, "%s%s%s", btf_show_indent(show), + btf_show_name(show), + btf_show_newline(show)); + show->state.depth++; + return safe_data; +} + +static void btf_show_end_aggr_type(struct btf_show *show, + const char *suffix) +{ + show->state.depth--; + btf_show(show, "%s%s%s%s", btf_show_indent(show), suffix, + btf_show_delim(show), btf_show_newline(show)); + btf_show_end_type(show); +} + +static void btf_show_start_member(struct btf_show *show, + const struct btf_member *m) +{ + show->state.member = m; +} + +static void btf_show_start_array_member(struct btf_show *show) +{ + show->state.array_member = 1; + btf_show_start_member(show, NULL); +} + +static void btf_show_end_member(struct btf_show *show) +{ + show->state.member = NULL; +} + +static void btf_show_end_array_member(struct btf_show *show) +{ + show->state.array_member = 0; + btf_show_end_member(show); +} + +static void *btf_show_start_array_type(struct btf_show *show, + const struct btf_type *t, + u32 type_id, + u16 array_encoding, + void *data) +{ + show->state.array_encoding = array_encoding; + show->state.array_terminated = 0; + return btf_show_start_aggr_type(show, t, type_id, data); +} + +static void btf_show_end_array_type(struct btf_show *show) +{ + show->state.array_encoding = 0; + show->state.array_terminated = 0; + btf_show_end_aggr_type(show, "]"); +} + +static void *btf_show_start_struct_type(struct btf_show *show, + const struct btf_type *t, + u32 type_id, + void *data) +{ + return btf_show_start_aggr_type(show, t, type_id, data); +} + +static void btf_show_end_struct_type(struct btf_show *show) +{ + btf_show_end_aggr_type(show, "}"); +} + __printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log, const char *fmt, ...) { @@ -1268,11 +1835,11 @@ static int btf_df_resolve(struct btf_verifier_env *env, return -EINVAL; } -static void btf_df_seq_show(const struct btf *btf, const struct btf_type *t, - u32 type_id, void *data, u8 bits_offsets, - struct seq_file *m) +static void btf_df_show(const struct btf *btf, const struct btf_type *t, + u32 type_id, void *data, u8 bits_offsets, + struct btf_show *show) { - seq_printf(m, "<unsupported kind:%u>", BTF_INFO_KIND(t->info)); + btf_show(show, "<unsupported kind:%u>", BTF_INFO_KIND(t->info)); } static int btf_int_check_member(struct btf_verifier_env *env, @@ -1445,7 +2012,7 @@ static void btf_int_log(struct btf_verifier_env *env, btf_int_encoding_str(BTF_INT_ENCODING(int_data))); } -static void btf_int128_print(struct seq_file *m, void *data) +static void btf_int128_print(struct btf_show *show, void *data) { /* data points to a __int128 number. * Suppose @@ -1464,9 +2031,10 @@ static void btf_int128_print(struct seq_file *m, void *data) lower_num = *(u64 *)data; #endif if (upper_num == 0) - seq_printf(m, "0x%llx", lower_num); + btf_show_type_value(show, "0x%llx", lower_num); else - seq_printf(m, "0x%llx%016llx", upper_num, lower_num); + btf_show_type_values(show, "0x%llx%016llx", upper_num, + lower_num); } static void btf_int128_shift(u64 *print_num, u16 left_shift_bits, @@ -1510,8 +2078,8 @@ static void btf_int128_shift(u64 *print_num, u16 left_shift_bits, #endif } -static void btf_bitfield_seq_show(void *data, u8 bits_offset, - u8 nr_bits, struct seq_file *m) +static void btf_bitfield_show(void *data, u8 bits_offset, + u8 nr_bits, struct btf_show *show) { u16 left_shift_bits, right_shift_bits; u8 nr_copy_bytes; @@ -1531,14 +2099,14 @@ static void btf_bitfield_seq_show(void *data, u8 bits_offset, right_shift_bits = BITS_PER_U128 - nr_bits; btf_int128_shift(print_num, left_shift_bits, right_shift_bits); - btf_int128_print(m, print_num); + btf_int128_print(show, print_num); } -static void btf_int_bits_seq_show(const struct btf *btf, - const struct btf_type *t, - void *data, u8 bits_offset, - struct seq_file *m) +static void btf_int_bits_show(const struct btf *btf, + const struct btf_type *t, + void *data, u8 bits_offset, + struct btf_show *show) { u32 int_data = btf_type_int(t); u8 nr_bits = BTF_INT_BITS(int_data); @@ -1551,55 +2119,77 @@ static void btf_int_bits_seq_show(const struct btf *btf, total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data); data += BITS_ROUNDDOWN_BYTES(total_bits_offset); bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset); - btf_bitfield_seq_show(data, bits_offset, nr_bits, m); + btf_bitfield_show(data, bits_offset, nr_bits, show); } -static void btf_int_seq_show(const struct btf *btf, const struct btf_type *t, - u32 type_id, void *data, u8 bits_offset, - struct seq_file *m) +static void btf_int_show(const struct btf *btf, const struct btf_type *t, + u32 type_id, void *data, u8 bits_offset, + struct btf_show *show) { u32 int_data = btf_type_int(t); u8 encoding = BTF_INT_ENCODING(int_data); bool sign = encoding & BTF_INT_SIGNED; u8 nr_bits = BTF_INT_BITS(int_data); + void *safe_data; + + safe_data = btf_show_start_type(show, t, type_id, data); + if (!safe_data) + return; if (bits_offset || BTF_INT_OFFSET(int_data) || BITS_PER_BYTE_MASKED(nr_bits)) { - btf_int_bits_seq_show(btf, t, data, bits_offset, m); - return; + btf_int_bits_show(btf, t, safe_data, bits_offset, show); + goto out; } switch (nr_bits) { case 128: - btf_int128_print(m, data); + btf_int128_print(show, safe_data); break; case 64: if (sign) - seq_printf(m, "%lld", *(s64 *)data); + btf_show_type_value(show, "%lld", *(s64 *)safe_data); else - seq_printf(m, "%llu", *(u64 *)data); + btf_show_type_value(show, "%llu", *(u64 *)safe_data); break; case 32: if (sign) - seq_printf(m, "%d", *(s32 *)data); + btf_show_type_value(show, "%d", *(s32 *)safe_data); else - seq_printf(m, "%u", *(u32 *)data); + btf_show_type_value(show, "%u", *(u32 *)safe_data); break; case 16: if (sign) - seq_printf(m, "%d", *(s16 *)data); + btf_show_type_value(show, "%d", *(s16 *)safe_data); else - seq_printf(m, "%u", *(u16 *)data); + btf_show_type_value(show, "%u", *(u16 *)safe_data); break; case 8: + if (show->state.array_encoding == BTF_INT_CHAR) { + /* check for null terminator */ + if (show->state.array_terminated) + break; + if (*(char *)data == '\0') { + show->state.array_terminated = 1; + break; + } + if (isprint(*(char *)data)) { + btf_show_type_value(show, "'%c'", + *(char *)safe_data); + break; + } + } if (sign) - seq_printf(m, "%d", *(s8 *)data); + btf_show_type_value(show, "%d", *(s8 *)safe_data); else - seq_printf(m, "%u", *(u8 *)data); + btf_show_type_value(show, "%u", *(u8 *)safe_data); break; default: - btf_int_bits_seq_show(btf, t, data, bits_offset, m); + btf_int_bits_show(btf, t, safe_data, bits_offset, show); + break; } +out: + btf_show_end_type(show); } static const struct btf_kind_operations int_ops = { @@ -1608,7 +2198,7 @@ static const struct btf_kind_operations int_ops = { .check_member = btf_int_check_member, .check_kflag_member = btf_int_check_kflag_member, .log_details = btf_int_log, - .seq_show = btf_int_seq_show, + .show = btf_int_show, }; static int btf_modifier_check_member(struct btf_verifier_env *env, @@ -1872,34 +2462,44 @@ static int btf_ptr_resolve(struct btf_verifier_env *env, return 0; } -static void btf_modifier_seq_show(const struct btf *btf, - const struct btf_type *t, - u32 type_id, void *data, - u8 bits_offset, struct seq_file *m) +static void btf_modifier_show(const struct btf *btf, + const struct btf_type *t, + u32 type_id, void *data, + u8 bits_offset, struct btf_show *show) { if (btf->resolved_ids) t = btf_type_id_resolve(btf, &type_id); else t = btf_type_skip_modifiers(btf, type_id, NULL); - btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m); + btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); } -static void btf_var_seq_show(const struct btf *btf, const struct btf_type *t, - u32 type_id, void *data, u8 bits_offset, - struct seq_file *m) +static void btf_var_show(const struct btf *btf, const struct btf_type *t, + u32 type_id, void *data, u8 bits_offset, + struct btf_show *show) { t = btf_type_id_resolve(btf, &type_id); - btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m); + btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); } -static void btf_ptr_seq_show(const struct btf *btf, const struct btf_type *t, - u32 type_id, void *data, u8 bits_offset, - struct seq_file *m) +static void btf_ptr_show(const struct btf *btf, const struct btf_type *t, + u32 type_id, void *data, u8 bits_offset, + struct btf_show *show) { - /* It is a hashed value */ - seq_printf(m, "%p", *(void **)data); + void *safe_data; + + safe_data = btf_show_start_type(show, t, type_id, data); + if (!safe_data) + return; + + /* It is a hashed value unless BTF_SHOW_PTR_RAW is specified */ + if (show->flags & BTF_SHOW_PTR_RAW) + btf_show_type_value(show, "0x%px", *(void **)safe_data); + else + btf_show_type_value(show, "0x%p", *(void **)safe_data); + btf_show_end_type(show); } static void btf_ref_type_log(struct btf_verifier_env *env, @@ -1914,7 +2514,7 @@ static struct btf_kind_operations modifier_ops = { .check_member = btf_modifier_check_member, .check_kflag_member = btf_modifier_check_kflag_member, .log_details = btf_ref_type_log, - .seq_show = btf_modifier_seq_show, + .show = btf_modifier_show, }; static struct btf_kind_operations ptr_ops = { @@ -1923,7 +2523,7 @@ static struct btf_kind_operations ptr_ops = { .check_member = btf_ptr_check_member, .check_kflag_member = btf_generic_check_kflag_member, .log_details = btf_ref_type_log, - .seq_show = btf_ptr_seq_show, + .show = btf_ptr_show, }; static s32 btf_fwd_check_meta(struct btf_verifier_env *env, @@ -1964,7 +2564,7 @@ static struct btf_kind_operations fwd_ops = { .check_member = btf_df_check_member, .check_kflag_member = btf_df_check_kflag_member, .log_details = btf_fwd_type_log, - .seq_show = btf_df_seq_show, + .show = btf_df_show, }; static int btf_array_check_member(struct btf_verifier_env *env, @@ -2123,28 +2723,90 @@ static void btf_array_log(struct btf_verifier_env *env, array->type, array->index_type, array->nelems); } -static void btf_array_seq_show(const struct btf *btf, const struct btf_type *t, - u32 type_id, void *data, u8 bits_offset, - struct seq_file *m) +static void __btf_array_show(const struct btf *btf, const struct btf_type *t, + u32 type_id, void *data, u8 bits_offset, + struct btf_show *show) { const struct btf_array *array = btf_type_array(t); const struct btf_kind_operations *elem_ops; const struct btf_type *elem_type; - u32 i, elem_size, elem_type_id; + u32 i, elem_size = 0, elem_type_id; + u16 encoding = 0; elem_type_id = array->type; - elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); + elem_type = btf_type_skip_modifiers(btf, elem_type_id, NULL); + if (elem_type && btf_type_has_size(elem_type)) + elem_size = elem_type->size; + + if (elem_type && btf_type_is_int(elem_type)) { + u32 int_type = btf_type_int(elem_type); + + encoding = BTF_INT_ENCODING(int_type); + + /* + * BTF_INT_CHAR encoding never seems to be set for + * char arrays, so if size is 1 and element is + * printable as a char, we'll do that. + */ + if (elem_size == 1) + encoding = BTF_INT_CHAR; + } + + if (!btf_show_start_array_type(show, t, type_id, encoding, data)) + return; + + if (!elem_type) + goto out; elem_ops = btf_type_ops(elem_type); - seq_puts(m, "["); + for (i = 0; i < array->nelems; i++) { - if (i) - seq_puts(m, ","); - elem_ops->seq_show(btf, elem_type, elem_type_id, data, - bits_offset, m); + btf_show_start_array_member(show); + + elem_ops->show(btf, elem_type, elem_type_id, data, + bits_offset, show); data += elem_size; + + btf_show_end_array_member(show); + + if (show->state.array_terminated) + break; + } +out: + btf_show_end_array_type(show); +} + +static void btf_array_show(const struct btf *btf, const struct btf_type *t, + u32 type_id, void *data, u8 bits_offset, + struct btf_show *show) +{ + const struct btf_member *m = show->state.member; + + /* + * First check if any members would be shown (are non-zero). + * See comments above "struct btf_show" definition for more + * details on how this works at a high-level. + */ + if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) { + if (!show->state.depth_check) { + show->state.depth_check = show->state.depth + 1; + show->state.depth_to_show = 0; + } + __btf_array_show(btf, t, type_id, data, bits_offset, show); + show->state.member = m; + + if (show->state.depth_check != show->state.depth + 1) + return; + show->state.depth_check = 0; + + if (show->state.depth_to_show <= show->state.depth) + return; + /* + * Reaching here indicates we have recursed and found + * non-zero array member(s). + */ } - seq_puts(m, "]"); + __btf_array_show(btf, t, type_id, data, bits_offset, show); } static struct btf_kind_operations array_ops = { @@ -2153,7 +2815,7 @@ static struct btf_kind_operations array_ops = { .check_member = btf_array_check_member, .check_kflag_member = btf_generic_check_kflag_member, .log_details = btf_array_log, - .seq_show = btf_array_seq_show, + .show = btf_array_show, }; static int btf_struct_check_member(struct btf_verifier_env *env, @@ -2376,15 +3038,18 @@ int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t) return off; } -static void btf_struct_seq_show(const struct btf *btf, const struct btf_type *t, - u32 type_id, void *data, u8 bits_offset, - struct seq_file *m) +static void __btf_struct_show(const struct btf *btf, const struct btf_type *t, + u32 type_id, void *data, u8 bits_offset, + struct btf_show *show) { - const char *seq = BTF_INFO_KIND(t->info) == BTF_KIND_UNION ? "|" : ","; const struct btf_member *member; + void *safe_data; u32 i; - seq_puts(m, "{"); + safe_data = btf_show_start_struct_type(show, t, type_id, data); + if (!safe_data) + return; + for_each_member(i, t, member) { const struct btf_type *member_type = btf_type_by_id(btf, member->type); @@ -2393,23 +3058,65 @@ static void btf_struct_seq_show(const struct btf *btf, const struct btf_type *t, u32 bytes_offset; u8 bits8_offset; - if (i) - seq_puts(m, seq); + btf_show_start_member(show, member); member_offset = btf_member_bit_offset(t, member); bitfield_size = btf_member_bitfield_size(t, member); bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset); bits8_offset = BITS_PER_BYTE_MASKED(member_offset); if (bitfield_size) { - btf_bitfield_seq_show(data + bytes_offset, bits8_offset, - bitfield_size, m); + safe_data = btf_show_start_type(show, member_type, + member->type, + data + bytes_offset); + if (safe_data) + btf_bitfield_show(safe_data, + bits8_offset, + bitfield_size, show); + btf_show_end_type(show); } else { ops = btf_type_ops(member_type); - ops->seq_show(btf, member_type, member->type, - data + bytes_offset, bits8_offset, m); + ops->show(btf, member_type, member->type, + data + bytes_offset, bits8_offset, show); } + + btf_show_end_member(show); } - seq_puts(m, "}"); + + btf_show_end_struct_type(show); +} + +static void btf_struct_show(const struct btf *btf, const struct btf_type *t, + u32 type_id, void *data, u8 bits_offset, + struct btf_show *show) +{ + const struct btf_member *m = show->state.member; + + /* + * First check if any members would be shown (are non-zero). + * See comments above "struct btf_show" definition for more + * details on how this works at a high-level. + */ + if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) { + if (!show->state.depth_check) { + show->state.depth_check = show->state.depth + 1; + show->state.depth_to_show = 0; + } + __btf_struct_show(btf, t, type_id, data, bits_offset, show); + /* Restore saved member data here */ + show->state.member = m; + if (show->state.depth_check != show->state.depth + 1) + return; + show->state.depth_check = 0; + + if (show->state.depth_to_show <= show->state.depth) + return; + /* + * Reaching here indicates we have recursed and found + * non-zero child values. + */ + } + + __btf_struct_show(btf, t, type_id, data, bits_offset, show); } static struct btf_kind_operations struct_ops = { @@ -2418,7 +3125,7 @@ static struct btf_kind_operations struct_ops = { .check_member = btf_struct_check_member, .check_kflag_member = btf_generic_check_kflag_member, .log_details = btf_struct_log, - .seq_show = btf_struct_seq_show, + .show = btf_struct_show, }; static int btf_enum_check_member(struct btf_verifier_env *env, @@ -2549,24 +3256,35 @@ static void btf_enum_log(struct btf_verifier_env *env, btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); } -static void btf_enum_seq_show(const struct btf *btf, const struct btf_type *t, - u32 type_id, void *data, u8 bits_offset, - struct seq_file *m) +static void btf_enum_show(const struct btf *btf, const struct btf_type *t, + u32 type_id, void *data, u8 bits_offset, + struct btf_show *show) { const struct btf_enum *enums = btf_type_enum(t); u32 i, nr_enums = btf_type_vlen(t); - int v = *(int *)data; + void *safe_data; + int v; + + safe_data = btf_show_start_type(show, t, type_id, data); + if (!safe_data) + return; + + v = *(int *)safe_data; for (i = 0; i < nr_enums; i++) { - if (v == enums[i].val) { - seq_printf(m, "%s", - __btf_name_by_offset(btf, - enums[i].name_off)); - return; - } + if (v != enums[i].val) + continue; + + btf_show_type_value(show, "%s", + __btf_name_by_offset(btf, + enums[i].name_off)); + + btf_show_end_type(show); + return; } - seq_printf(m, "%d", v); + btf_show_type_value(show, "%d", v); + btf_show_end_type(show); } static struct btf_kind_operations enum_ops = { @@ -2575,7 +3293,7 @@ static struct btf_kind_operations enum_ops = { .check_member = btf_enum_check_member, .check_kflag_member = btf_enum_check_kflag_member, .log_details = btf_enum_log, - .seq_show = btf_enum_seq_show, + .show = btf_enum_show, }; static s32 btf_func_proto_check_meta(struct btf_verifier_env *env, @@ -2662,7 +3380,7 @@ static struct btf_kind_operations func_proto_ops = { .check_member = btf_df_check_member, .check_kflag_member = btf_df_check_kflag_member, .log_details = btf_func_proto_log, - .seq_show = btf_df_seq_show, + .show = btf_df_show, }; static s32 btf_func_check_meta(struct btf_verifier_env *env, @@ -2696,7 +3414,7 @@ static struct btf_kind_operations func_ops = { .check_member = btf_df_check_member, .check_kflag_member = btf_df_check_kflag_member, .log_details = btf_ref_type_log, - .seq_show = btf_df_seq_show, + .show = btf_df_show, }; static s32 btf_var_check_meta(struct btf_verifier_env *env, @@ -2760,7 +3478,7 @@ static const struct btf_kind_operations var_ops = { .check_member = btf_df_check_member, .check_kflag_member = btf_df_check_kflag_member, .log_details = btf_var_log, - .seq_show = btf_var_seq_show, + .show = btf_var_show, }; static s32 btf_datasec_check_meta(struct btf_verifier_env *env, @@ -2886,24 +3604,28 @@ static void btf_datasec_log(struct btf_verifier_env *env, btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); } -static void btf_datasec_seq_show(const struct btf *btf, - const struct btf_type *t, u32 type_id, - void *data, u8 bits_offset, - struct seq_file *m) +static void btf_datasec_show(const struct btf *btf, + const struct btf_type *t, u32 type_id, + void *data, u8 bits_offset, + struct btf_show *show) { const struct btf_var_secinfo *vsi; const struct btf_type *var; u32 i; - seq_printf(m, "section (\"%s\") = {", __btf_name_by_offset(btf, t->name_off)); + if (!btf_show_start_type(show, t, type_id, data)) + return; + + btf_show_type_value(show, "section (\"%s\") = {", + __btf_name_by_offset(btf, t->name_off)); for_each_vsi(i, t, vsi) { var = btf_type_by_id(btf, vsi->type); if (i) - seq_puts(m, ","); - btf_type_ops(var)->seq_show(btf, var, vsi->type, - data + vsi->offset, bits_offset, m); + btf_show(show, ","); + btf_type_ops(var)->show(btf, var, vsi->type, + data + vsi->offset, bits_offset, show); } - seq_puts(m, "}"); + btf_show_end_type(show); } static const struct btf_kind_operations datasec_ops = { @@ -2912,7 +3634,7 @@ static const struct btf_kind_operations datasec_ops = { .check_member = btf_df_check_member, .check_kflag_member = btf_df_check_kflag_member, .log_details = btf_datasec_log, - .seq_show = btf_datasec_seq_show, + .show = btf_datasec_show, }; static int btf_func_proto_check(struct btf_verifier_env *env, @@ -3706,7 +4428,7 @@ errout: struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog) { - struct bpf_prog *tgt_prog = prog->aux->linked_prog; + struct bpf_prog *tgt_prog = prog->aux->dst_prog; if (tgt_prog) { return tgt_prog->aux->btf; @@ -3733,7 +4455,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, struct bpf_insn_access_aux *info) { const struct btf_type *t = prog->aux->attach_func_proto; - struct bpf_prog *tgt_prog = prog->aux->linked_prog; + struct bpf_prog *tgt_prog = prog->aux->dst_prog; struct btf *btf = bpf_prog_get_target_btf(prog); const char *tname = prog->aux->attach_func_name; struct bpf_verifier_log *log = info->log; @@ -3860,7 +4582,14 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, info->reg_type = PTR_TO_BTF_ID; if (tgt_prog) { - ret = btf_translate_to_vmlinux(log, btf, t, tgt_prog->type, arg); + enum bpf_prog_type tgt_type; + + if (tgt_prog->type == BPF_PROG_TYPE_EXT) + tgt_type = tgt_prog->aux->saved_dst_prog_type; + else + tgt_type = tgt_prog->type; + + ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg); if (ret > 0) { info->btf_id = ret; return true; @@ -4388,7 +5117,7 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, } /* Compare BTFs of given program with BTF of target program */ -int btf_check_type_match(struct bpf_verifier_env *env, struct bpf_prog *prog, +int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, struct btf *btf2, const struct btf_type *t2) { struct btf *btf1 = prog->aux->btf; @@ -4396,7 +5125,7 @@ int btf_check_type_match(struct bpf_verifier_env *env, struct bpf_prog *prog, u32 btf_id = 0; if (!prog->aux->func_info) { - bpf_log(&env->log, "Program extension requires BTF\n"); + bpf_log(log, "Program extension requires BTF\n"); return -EINVAL; } @@ -4408,7 +5137,7 @@ int btf_check_type_match(struct bpf_verifier_env *env, struct bpf_prog *prog, if (!t1 || !btf_type_is_func(t1)) return -EFAULT; - return btf_check_func_type_match(&env->log, btf1, t1, btf2, t2); + return btf_check_func_type_match(log, btf1, t1, btf2, t2); } /* Compare BTF of a function with given bpf_reg_state. @@ -4559,7 +5288,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, return -EFAULT; } if (prog_type == BPF_PROG_TYPE_EXT) - prog_type = prog->aux->linked_prog->type; + prog_type = prog->aux->dst_prog->type; t = btf_type_by_id(btf, t->type); if (!t || !btf_type_is_func_proto(t)) { @@ -4606,12 +5335,93 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, return 0; } +static void btf_type_show(const struct btf *btf, u32 type_id, void *obj, + struct btf_show *show) +{ + const struct btf_type *t = btf_type_by_id(btf, type_id); + + show->btf = btf; + memset(&show->state, 0, sizeof(show->state)); + memset(&show->obj, 0, sizeof(show->obj)); + + btf_type_ops(t)->show(btf, t, type_id, obj, 0, show); +} + +static void btf_seq_show(struct btf_show *show, const char *fmt, + va_list args) +{ + seq_vprintf((struct seq_file *)show->target, fmt, args); +} + +int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, + void *obj, struct seq_file *m, u64 flags) +{ + struct btf_show sseq; + + sseq.target = m; + sseq.showfn = btf_seq_show; + sseq.flags = flags; + + btf_type_show(btf, type_id, obj, &sseq); + + return sseq.state.status; +} + void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, struct seq_file *m) { - const struct btf_type *t = btf_type_by_id(btf, type_id); + (void) btf_type_seq_show_flags(btf, type_id, obj, m, + BTF_SHOW_NONAME | BTF_SHOW_COMPACT | + BTF_SHOW_ZERO | BTF_SHOW_UNSAFE); +} + +struct btf_show_snprintf { + struct btf_show show; + int len_left; /* space left in string */ + int len; /* length we would have written */ +}; + +static void btf_snprintf_show(struct btf_show *show, const char *fmt, + va_list args) +{ + struct btf_show_snprintf *ssnprintf = (struct btf_show_snprintf *)show; + int len; + + len = vsnprintf(show->target, ssnprintf->len_left, fmt, args); + + if (len < 0) { + ssnprintf->len_left = 0; + ssnprintf->len = len; + } else if (len > ssnprintf->len_left) { + /* no space, drive on to get length we would have written */ + ssnprintf->len_left = 0; + ssnprintf->len += len; + } else { + ssnprintf->len_left -= len; + ssnprintf->len += len; + show->target += len; + } +} + +int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, + char *buf, int len, u64 flags) +{ + struct btf_show_snprintf ssnprintf; + + ssnprintf.show.target = buf; + ssnprintf.show.flags = flags; + ssnprintf.show.showfn = btf_snprintf_show; + ssnprintf.len_left = len; + ssnprintf.len = 0; + + btf_type_show(btf, type_id, obj, (struct btf_show *)&ssnprintf); + + /* If we encontered an error, return it. */ + if (ssnprintf.show.state.status) + return ssnprintf.show.state.status; - btf_type_ops(t)->seq_show(btf, t, type_id, obj, 0, m); + /* Otherwise return length we would have written */ + return ssnprintf.len; } #ifdef CONFIG_PROC_FS diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index c4811b139caa..cda674f1392f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -99,6 +99,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode); mutex_init(&fp->aux->used_maps_mutex); + mutex_init(&fp->aux->dst_mutex); return fp; } @@ -255,6 +256,7 @@ void __bpf_prog_free(struct bpf_prog *fp) { if (fp->aux) { mutex_destroy(&fp->aux->used_maps_mutex); + mutex_destroy(&fp->aux->dst_mutex); free_percpu(fp->aux->stats); kfree(fp->aux->poke_tab); kfree(fp->aux); @@ -2138,7 +2140,8 @@ static void bpf_prog_free_deferred(struct work_struct *work) if (aux->prog->has_callchain_buf) put_callchain_buffers(); #endif - bpf_trampoline_put(aux->trampoline); + if (aux->dst_trampoline) + bpf_trampoline_put(aux->dst_trampoline); for (i = 0; i < aux->func_cnt; i++) bpf_jit_free(aux->func[i]); if (aux->func_cnt) { @@ -2154,8 +2157,8 @@ void bpf_prog_free(struct bpf_prog *fp) { struct bpf_prog_aux *aux = fp->aux; - if (aux->linked_prog) - bpf_prog_put(aux->linked_prog); + if (aux->dst_prog) + bpf_prog_put(aux->dst_prog); INIT_WORK(&aux->work, bpf_prog_free_deferred); schedule_work(&aux->work); } @@ -2216,6 +2219,8 @@ const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak; const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto __weak; const struct bpf_func_proto bpf_get_local_storage_proto __weak; const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak; +const struct bpf_func_proto bpf_snprintf_btf_proto __weak; +const struct bpf_func_proto bpf_seq_printf_btf_proto __weak; const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void) { diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 7e1a8ad0c32a..c61a23b564aa 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -155,8 +155,7 @@ static void cpu_map_kthread_stop(struct work_struct *work) kthread_stop(rcpu->kthread); } -static struct sk_buff *cpu_map_build_skb(struct bpf_cpu_map_entry *rcpu, - struct xdp_frame *xdpf, +static struct sk_buff *cpu_map_build_skb(struct xdp_frame *xdpf, struct sk_buff *skb) { unsigned int hard_start_headroom; @@ -365,7 +364,7 @@ static int cpu_map_kthread_run(void *data) struct sk_buff *skb = skbs[i]; int ret; - skb = cpu_map_build_skb(rcpu, xdpf, skb); + skb = cpu_map_build_skb(xdpf, skb); if (!skb) { xdp_return_frame(xdpf); continue; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 5cc7425ee476..e825441781ab 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -683,6 +683,10 @@ bpf_base_func_proto(enum bpf_func_id func_id) if (!perfmon_capable()) return NULL; return bpf_get_trace_printk_proto(); + case BPF_FUNC_snprintf_btf: + if (!perfmon_capable()) + return NULL; + return &bpf_snprintf_btf_proto; case BPF_FUNC_jiffies64: return &bpf_jiffies64_proto; default: diff --git a/kernel/bpf/preload/.gitignore b/kernel/bpf/preload/.gitignore new file mode 100644 index 000000000000..856a4c5ad0dd --- /dev/null +++ b/kernel/bpf/preload/.gitignore @@ -0,0 +1,4 @@ +/FEATURE-DUMP.libbpf +/bpf_helper_defs.h +/feature +/bpf_preload_umd diff --git a/kernel/bpf/preload/Makefile b/kernel/bpf/preload/Makefile index 12c7b62b9b6e..23ee310b6eb4 100644 --- a/kernel/bpf/preload/Makefile +++ b/kernel/bpf/preload/Makefile @@ -12,6 +12,8 @@ userccflags += -I $(srctree)/tools/include/ -I $(srctree)/tools/include/uapi \ userprogs := bpf_preload_umd +clean-files := $(userprogs) bpf_helper_defs.h FEATURE-DUMP.libbpf staticobjs/ feature/ + bpf_preload_umd-objs := iterators/iterators.o bpf_preload_umd-userldlibs := $(LIBBPF_A) -lelf -lz diff --git a/kernel/bpf/preload/iterators/iterators.bpf.c b/kernel/bpf/preload/iterators/iterators.bpf.c index 5ded550b2ed6..52aa7b38e8b8 100644 --- a/kernel/bpf/preload/iterators/iterators.bpf.c +++ b/kernel/bpf/preload/iterators/iterators.bpf.c @@ -42,7 +42,7 @@ struct bpf_prog_aux { __u32 id; char name[16]; const char *attach_func_name; - struct bpf_prog *linked_prog; + struct bpf_prog *dst_prog; struct bpf_func_info *func_info; struct btf *btf; }; @@ -108,7 +108,7 @@ int dump_bpf_prog(struct bpf_iter__bpf_prog *ctx) BPF_SEQ_PRINTF(seq, "%4u %-16s %s %s\n", aux->id, get_name(aux->btf, aux->func_info[0].type_id, aux->name), - aux->attach_func_name, aux->linked_prog->aux->name); + aux->attach_func_name, aux->dst_prog->aux->name); return 0; } char LICENSE[] SEC("license") = "GPL"; diff --git a/kernel/bpf/preload/iterators/iterators.skel.h b/kernel/bpf/preload/iterators/iterators.skel.h index c3171357dc4f..cf9a6a94b3a4 100644 --- a/kernel/bpf/preload/iterators/iterators.skel.h +++ b/kernel/bpf/preload/iterators/iterators.skel.h @@ -47,7 +47,7 @@ iterators_bpf__open_opts(const struct bpf_object_open_opts *opts) { struct iterators_bpf *obj; - obj = (typeof(obj))calloc(1, sizeof(*obj)); + obj = (struct iterators_bpf *)calloc(1, sizeof(*obj)); if (!obj) return NULL; if (iterators_bpf__create_skeleton(obj)) @@ -105,7 +105,7 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) { struct bpf_object_skeleton *s; - s = (typeof(s))calloc(1, sizeof(*s)); + s = (struct bpf_object_skeleton *)calloc(1, sizeof(*s)); if (!s) return -1; obj->skeleton = s; @@ -117,7 +117,7 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) /* maps */ s->map_cnt = 1; s->map_skel_sz = sizeof(*s->maps); - s->maps = (typeof(s->maps))calloc(s->map_cnt, s->map_skel_sz); + s->maps = (struct bpf_map_skeleton *)calloc(s->map_cnt, s->map_skel_sz); if (!s->maps) goto err; @@ -128,7 +128,7 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) /* programs */ s->prog_cnt = 2; s->prog_skel_sz = sizeof(*s->progs); - s->progs = (typeof(s->progs))calloc(s->prog_cnt, s->prog_skel_sz); + s->progs = (struct bpf_prog_skeleton *)calloc(s->prog_cnt, s->prog_skel_sz); if (!s->progs) goto err; @@ -140,10 +140,10 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) s->progs[1].prog = &obj->progs.dump_bpf_prog; s->progs[1].link = &obj->links.dump_bpf_prog; - s->data_sz = 7128; + s->data_sz = 7176; s->data = (void *)"\ \x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\x18\x18\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0f\0\ +\0\0\0\0\0\0\0\0\0\0\0\x48\x18\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0f\0\ \x0e\0\x79\x12\0\0\0\0\0\0\x79\x26\0\0\0\0\0\0\x79\x17\x08\0\0\0\0\0\x15\x07\ \x1a\0\0\0\0\0\x79\x21\x10\0\0\0\0\0\x55\x01\x08\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\ \x07\x04\0\0\xe8\xff\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\0\ @@ -164,7 +164,7 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) \x79\x86\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf8\xff\xff\xff\xb7\x02\0\ \0\x08\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x01\0\0\0\0\0\0\x79\xa3\xf8\xff\0\0\0\0\ \x0f\x13\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf4\xff\xff\xff\xb7\x02\0\ -\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\x03\0\0\x04\0\0\0\x61\xa1\xf4\xff\0\0\0\0\ +\0\x04\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x03\0\0\x04\0\0\0\x61\xa1\xf4\xff\0\0\0\0\ \x61\x82\x10\0\0\0\0\0\x3d\x21\x02\0\0\0\0\0\x0f\x16\0\0\0\0\0\0\xbf\x69\0\0\0\ \0\0\0\x7b\x9a\xd8\xff\0\0\0\0\x79\x71\x18\0\0\0\0\0\x7b\x1a\xe0\xff\0\0\0\0\ \x79\x71\x20\0\0\0\0\0\x79\x11\0\0\0\0\0\0\x0f\x31\0\0\0\0\0\0\x7b\x1a\xe8\xff\ @@ -176,230 +176,232 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) \x73\x25\x36\x64\x0a\0\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\ \x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x0a\0\x25\x34\ \x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x0a\0\x47\x50\x4c\0\x9f\ -\xeb\x01\0\x18\0\0\0\0\0\0\0\x1c\x04\0\0\x1c\x04\0\0\0\x05\0\0\0\0\0\0\0\0\0\ +\xeb\x01\0\x18\0\0\0\0\0\0\0\x1c\x04\0\0\x1c\x04\0\0\x09\x05\0\0\0\0\0\0\0\0\0\ \x02\x02\0\0\0\x01\0\0\0\x02\0\0\x04\x10\0\0\0\x13\0\0\0\x03\0\0\0\0\0\0\0\x18\ \0\0\0\x04\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\x02\x0d\0\ \0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x01\0\0\0\x20\0\0\0\0\0\0\x01\x04\ -\0\0\0\x20\0\0\x01\x24\0\0\0\x01\0\0\x0c\x05\0\0\0\xa3\0\0\0\x03\0\0\x04\x18\0\ -\0\0\xb1\0\0\0\x09\0\0\0\0\0\0\0\xb5\0\0\0\x0b\0\0\0\x40\0\0\0\xc0\0\0\0\x0b\0\ -\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\xc8\0\0\0\0\0\0\x07\0\0\0\0\xd1\0\0\ -\0\0\0\0\x08\x0c\0\0\0\xd7\0\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\x98\x01\0\0\x03\ -\0\0\x04\x18\0\0\0\xa0\x01\0\0\x0e\0\0\0\0\0\0\0\xa3\x01\0\0\x11\0\0\0\x20\0\0\ -\0\xa8\x01\0\0\x0e\0\0\0\xa0\0\0\0\xb4\x01\0\0\0\0\0\x08\x0f\0\0\0\xba\x01\0\0\ -\0\0\0\x01\x04\0\0\0\x20\0\0\0\xc7\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\x01\0\0\ -\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x10\0\0\0\xcc\x01\0\0\0\0\0\x01\x04\ -\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x14\0\0\0\x30\x02\0\0\x02\0\0\x04\x10\0\0\0\ -\x13\0\0\0\x03\0\0\0\0\0\0\0\x43\x02\0\0\x15\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\ -\x18\0\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x13\0\0\0\x48\x02\0\0\x01\0\ -\0\x0c\x16\0\0\0\x94\x02\0\0\x01\0\0\x04\x08\0\0\0\x9d\x02\0\0\x19\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\x02\x1a\0\0\0\xee\x02\0\0\x06\0\0\x04\x38\0\0\0\xa0\x01\0\0\ -\x0e\0\0\0\0\0\0\0\xa3\x01\0\0\x11\0\0\0\x20\0\0\0\xfb\x02\0\0\x1b\0\0\0\xc0\0\ -\0\0\x0c\x03\0\0\x15\0\0\0\0\x01\0\0\x18\x03\0\0\x1d\0\0\0\x40\x01\0\0\x22\x03\ +\0\0\0\x20\0\0\x01\x24\0\0\0\x01\0\0\x0c\x05\0\0\0\xaf\0\0\0\x03\0\0\x04\x18\0\ +\0\0\xbd\0\0\0\x09\0\0\0\0\0\0\0\xc1\0\0\0\x0b\0\0\0\x40\0\0\0\xcc\0\0\0\x0b\0\ +\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\xd4\0\0\0\0\0\0\x07\0\0\0\0\xdd\0\0\ +\0\0\0\0\x08\x0c\0\0\0\xe3\0\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\xa4\x01\0\0\x03\ +\0\0\x04\x18\0\0\0\xac\x01\0\0\x0e\0\0\0\0\0\0\0\xaf\x01\0\0\x11\0\0\0\x20\0\0\ +\0\xb4\x01\0\0\x0e\0\0\0\xa0\0\0\0\xc0\x01\0\0\0\0\0\x08\x0f\0\0\0\xc6\x01\0\0\ +\0\0\0\x01\x04\0\0\0\x20\0\0\0\xd3\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\x01\0\0\ +\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x10\0\0\0\xd8\x01\0\0\0\0\0\x01\x04\ +\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x14\0\0\0\x3c\x02\0\0\x02\0\0\x04\x10\0\0\0\ +\x13\0\0\0\x03\0\0\0\0\0\0\0\x4f\x02\0\0\x15\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\ +\x18\0\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x13\0\0\0\x54\x02\0\0\x01\0\ +\0\x0c\x16\0\0\0\xa0\x02\0\0\x01\0\0\x04\x08\0\0\0\xa9\x02\0\0\x19\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x02\x1a\0\0\0\xfa\x02\0\0\x06\0\0\x04\x38\0\0\0\xac\x01\0\0\ +\x0e\0\0\0\0\0\0\0\xaf\x01\0\0\x11\0\0\0\x20\0\0\0\x07\x03\0\0\x1b\0\0\0\xc0\0\ +\0\0\x18\x03\0\0\x15\0\0\0\0\x01\0\0\x21\x03\0\0\x1d\0\0\0\x40\x01\0\0\x2b\x03\ \0\0\x1e\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\x02\x1c\0\0\0\0\0\0\0\0\0\0\x0a\x10\0\ -\0\0\0\0\0\0\0\0\0\x02\x1f\0\0\0\0\0\0\0\0\0\0\x02\x20\0\0\0\x6c\x03\0\0\x02\0\ -\0\x04\x08\0\0\0\x7a\x03\0\0\x0e\0\0\0\0\0\0\0\x83\x03\0\0\x0e\0\0\0\x20\0\0\0\ -\x22\x03\0\0\x03\0\0\x04\x18\0\0\0\x8d\x03\0\0\x1b\0\0\0\0\0\0\0\x95\x03\0\0\ -\x21\0\0\0\x40\0\0\0\x9b\x03\0\0\x23\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x22\0\0\ -\0\0\0\0\0\0\0\0\x02\x24\0\0\0\x9f\x03\0\0\x01\0\0\x04\x04\0\0\0\xaa\x03\0\0\ -\x0e\0\0\0\0\0\0\0\x13\x04\0\0\x01\0\0\x04\x04\0\0\0\x1c\x04\0\0\x0e\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x23\0\0\0\x92\x04\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\x02\x1f\0\0\0\0\0\0\0\0\0\0\x02\x20\0\0\0\x75\x03\0\0\x02\0\ +\0\x04\x08\0\0\0\x83\x03\0\0\x0e\0\0\0\0\0\0\0\x8c\x03\0\0\x0e\0\0\0\x20\0\0\0\ +\x2b\x03\0\0\x03\0\0\x04\x18\0\0\0\x96\x03\0\0\x1b\0\0\0\0\0\0\0\x9e\x03\0\0\ +\x21\0\0\0\x40\0\0\0\xa4\x03\0\0\x23\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x22\0\0\ +\0\0\0\0\0\0\0\0\x02\x24\0\0\0\xa8\x03\0\0\x01\0\0\x04\x04\0\0\0\xb3\x03\0\0\ +\x0e\0\0\0\0\0\0\0\x1c\x04\0\0\x01\0\0\x04\x04\0\0\0\x25\x04\0\0\x0e\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x23\0\0\0\x9b\x04\0\0\0\0\0\ \x0e\x25\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x0e\0\0\0\ -\xa6\x04\0\0\0\0\0\x0e\x27\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\ -\x12\0\0\0\x20\0\0\0\xbc\x04\0\0\0\0\0\x0e\x29\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\ -\0\0\0\0\x1c\0\0\0\x12\0\0\0\x11\0\0\0\xd1\x04\0\0\0\0\0\x0e\x2b\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x04\0\0\0\xe8\x04\0\0\0\0\0\x0e\ -\x2d\0\0\0\x01\0\0\0\xf0\x04\0\0\x04\0\0\x0f\0\0\0\0\x26\0\0\0\0\0\0\0\x23\0\0\ +\xaf\x04\0\0\0\0\0\x0e\x27\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\ +\x12\0\0\0\x20\0\0\0\xc5\x04\0\0\0\0\0\x0e\x29\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\ +\0\0\0\0\x1c\0\0\0\x12\0\0\0\x11\0\0\0\xda\x04\0\0\0\0\0\x0e\x2b\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x04\0\0\0\xf1\x04\0\0\0\0\0\x0e\ +\x2d\0\0\0\x01\0\0\0\xf9\x04\0\0\x04\0\0\x0f\0\0\0\0\x26\0\0\0\0\0\0\0\x23\0\0\ \0\x28\0\0\0\x23\0\0\0\x0e\0\0\0\x2a\0\0\0\x31\0\0\0\x20\0\0\0\x2c\0\0\0\x51\0\ -\0\0\x11\0\0\0\xf8\x04\0\0\x01\0\0\x0f\0\0\0\0\x2e\0\0\0\0\0\0\0\x04\0\0\0\0\ +\0\0\x11\0\0\0\x01\x05\0\0\x01\0\0\x0f\0\0\0\0\x2e\0\0\0\0\0\0\0\x04\0\0\0\0\ \x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x6d\x65\ \x74\x61\0\x6d\x61\x70\0\x63\x74\x78\0\x69\x6e\x74\0\x64\x75\x6d\x70\x5f\x62\ \x70\x66\x5f\x6d\x61\x70\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\ -\x30\x3a\x30\0\x2f\x77\x2f\x6e\x65\x74\x2d\x6e\x65\x78\x74\x2f\x6b\x65\x72\x6e\ -\x65\x6c\x2f\x62\x70\x66\x2f\x70\x72\x65\x6c\x6f\x61\x64\x2f\x69\x74\x65\x72\ -\x61\x74\x6f\x72\x73\x2f\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\ -\x2e\x63\0\x09\x73\x74\x72\x75\x63\x74\x20\x73\x65\x71\x5f\x66\x69\x6c\x65\x20\ -\x2a\x73\x65\x71\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\ -\x65\x71\x3b\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x6d\x65\x74\x61\0\x73\x65\ -\x71\0\x73\x65\x73\x73\x69\x6f\x6e\x5f\x69\x64\0\x73\x65\x71\x5f\x6e\x75\x6d\0\ -\x73\x65\x71\x5f\x66\x69\x6c\x65\0\x5f\x5f\x75\x36\x34\0\x6c\x6f\x6e\x67\x20\ -\x6c\x6f\x6e\x67\x20\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x30\x3a\ -\x31\0\x09\x73\x74\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x6d\x61\x70\x20\x2a\x6d\ -\x61\x70\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x61\x70\x3b\0\x09\x69\x66\x20\x28\ -\x21\x6d\x61\x70\x29\0\x30\x3a\x32\0\x09\x5f\x5f\x75\x36\x34\x20\x73\x65\x71\ -\x5f\x6e\x75\x6d\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\ -\x65\x71\x5f\x6e\x75\x6d\x3b\0\x09\x69\x66\x20\x28\x73\x65\x71\x5f\x6e\x75\x6d\ -\x20\x3d\x3d\x20\x30\x29\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\ -\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\ -\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\ -\x74\x72\x69\x65\x73\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\x5f\x6d\x61\x70\0\x69\ -\x64\0\x6e\x61\x6d\x65\0\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x5f\x5f\ -\x75\x33\x32\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x63\x68\x61\ -\x72\0\x5f\x5f\x41\x52\x52\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\ -\x5f\0\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\ -\x71\x2c\x20\x22\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x25\x36\x64\x5c\x6e\x22\ -\x2c\x20\x6d\x61\x70\x2d\x3e\x69\x64\x2c\x20\x6d\x61\x70\x2d\x3e\x6e\x61\x6d\ -\x65\x2c\x20\x6d\x61\x70\x2d\x3e\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\ -\x29\x3b\0\x7d\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x70\ -\x72\x6f\x67\0\x70\x72\x6f\x67\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\ -\x6f\x67\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x09\x73\x74\ -\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x70\x72\x6f\x67\x20\x2a\x70\x72\x6f\x67\ -\x20\x3d\x20\x63\x74\x78\x2d\x3e\x70\x72\x6f\x67\x3b\0\x09\x69\x66\x20\x28\x21\ -\x70\x72\x6f\x67\x29\0\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x61\x75\x78\0\x09\x61\ -\x75\x78\x20\x3d\x20\x70\x72\x6f\x67\x2d\x3e\x61\x75\x78\x3b\0\x09\x09\x42\x50\ -\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\ -\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\ -\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\ -\x5f\x70\x72\x6f\x67\x5f\x61\x75\x78\0\x61\x74\x74\x61\x63\x68\x5f\x66\x75\x6e\ -\x63\x5f\x6e\x61\x6d\x65\0\x6c\x69\x6e\x6b\x65\x64\x5f\x70\x72\x6f\x67\0\x66\ -\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x62\x74\x66\0\x09\x42\x50\x46\x5f\x53\x45\ -\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\x34\x75\x20\ -\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x5c\x6e\x22\x2c\x20\x61\x75\x78\ -\x2d\x3e\x69\x64\x2c\0\x30\x3a\x34\0\x30\x3a\x35\0\x09\x69\x66\x20\x28\x21\x62\ -\x74\x66\x29\0\x62\x70\x66\x5f\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x69\x6e\ -\x73\x6e\x5f\x6f\x66\x66\0\x74\x79\x70\x65\x5f\x69\x64\0\x30\0\x73\x74\x72\x69\ -\x6e\x67\x73\0\x74\x79\x70\x65\x73\0\x68\x64\x72\0\x62\x74\x66\x5f\x68\x65\x61\ -\x64\x65\x72\0\x73\x74\x72\x5f\x6c\x65\x6e\0\x09\x74\x79\x70\x65\x73\x20\x3d\ -\x20\x62\x74\x66\x2d\x3e\x74\x79\x70\x65\x73\x3b\0\x09\x62\x70\x66\x5f\x70\x72\ -\x6f\x62\x65\x5f\x72\x65\x61\x64\x5f\x6b\x65\x72\x6e\x65\x6c\x28\x26\x74\x2c\ -\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\x29\x2c\x20\x74\x79\x70\x65\x73\x20\x2b\ -\x20\x62\x74\x66\x5f\x69\x64\x29\x3b\0\x09\x73\x74\x72\x20\x3d\x20\x62\x74\x66\ -\x2d\x3e\x73\x74\x72\x69\x6e\x67\x73\x3b\0\x62\x74\x66\x5f\x74\x79\x70\x65\0\ -\x6e\x61\x6d\x65\x5f\x6f\x66\x66\0\x09\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3d\ -\x20\x42\x50\x46\x5f\x43\x4f\x52\x45\x5f\x52\x45\x41\x44\x28\x74\x2c\x20\x6e\ -\x61\x6d\x65\x5f\x6f\x66\x66\x29\x3b\0\x30\x3a\x32\x3a\x30\0\x09\x69\x66\x20\ -\x28\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3e\x3d\x20\x62\x74\x66\x2d\x3e\x68\ -\x64\x72\x2e\x73\x74\x72\x5f\x6c\x65\x6e\x29\0\x09\x72\x65\x74\x75\x72\x6e\x20\ -\x73\x74\x72\x20\x2b\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x3b\0\x30\x3a\x33\0\ -\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\ -\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\ -\x2e\x31\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\ -\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\ -\x5f\x66\x6d\x74\x2e\x32\0\x4c\x49\x43\x45\x4e\x53\x45\0\x2e\x72\x6f\x64\x61\ -\x74\x61\0\x6c\x69\x63\x65\x6e\x73\x65\0\x9f\xeb\x01\0\x20\0\0\0\0\0\0\0\x24\0\ -\0\0\x24\0\0\0\x44\x02\0\0\x68\x02\0\0\xa4\x01\0\0\x08\0\0\0\x31\0\0\0\x01\0\0\ -\0\0\0\0\0\x07\0\0\0\x56\x02\0\0\x01\0\0\0\0\0\0\0\x17\0\0\0\x10\0\0\0\x31\0\0\ -\0\x09\0\0\0\0\0\0\0\x42\0\0\0\x7b\0\0\0\x1e\x40\x01\0\x08\0\0\0\x42\0\0\0\x7b\ -\0\0\0\x24\x40\x01\0\x10\0\0\0\x42\0\0\0\xf2\0\0\0\x1d\x48\x01\0\x18\0\0\0\x42\ -\0\0\0\x13\x01\0\0\x06\x50\x01\0\x20\0\0\0\x42\0\0\0\x22\x01\0\0\x1d\x44\x01\0\ -\x28\0\0\0\x42\0\0\0\x47\x01\0\0\x06\x5c\x01\0\x38\0\0\0\x42\0\0\0\x5a\x01\0\0\ -\x03\x60\x01\0\x70\0\0\0\x42\0\0\0\xe0\x01\0\0\x02\x68\x01\0\xf0\0\0\0\x42\0\0\ -\0\x2e\x02\0\0\x01\x70\x01\0\x56\x02\0\0\x1a\0\0\0\0\0\0\0\x42\0\0\0\x7b\0\0\0\ -\x1e\x84\x01\0\x08\0\0\0\x42\0\0\0\x7b\0\0\0\x24\x84\x01\0\x10\0\0\0\x42\0\0\0\ -\x64\x02\0\0\x1f\x8c\x01\0\x18\0\0\0\x42\0\0\0\x88\x02\0\0\x06\x98\x01\0\x20\0\ -\0\0\x42\0\0\0\xa1\x02\0\0\x0e\xa4\x01\0\x28\0\0\0\x42\0\0\0\x22\x01\0\0\x1d\ -\x88\x01\0\x30\0\0\0\x42\0\0\0\x47\x01\0\0\x06\xa8\x01\0\x40\0\0\0\x42\0\0\0\ -\xb3\x02\0\0\x03\xac\x01\0\x80\0\0\0\x42\0\0\0\x26\x03\0\0\x02\xb4\x01\0\xb8\0\ -\0\0\x42\0\0\0\x61\x03\0\0\x06\x08\x01\0\xd0\0\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\ -\xd8\0\0\0\x42\0\0\0\xb2\x03\0\0\x0f\x14\x01\0\xe0\0\0\0\x42\0\0\0\xc7\x03\0\0\ -\x2d\x18\x01\0\xf0\0\0\0\x42\0\0\0\xfe\x03\0\0\x0d\x10\x01\0\0\x01\0\0\x42\0\0\ -\0\0\0\0\0\0\0\0\0\x08\x01\0\0\x42\0\0\0\xc7\x03\0\0\x02\x18\x01\0\x20\x01\0\0\ -\x42\0\0\0\x25\x04\0\0\x0d\x1c\x01\0\x38\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x40\ -\x01\0\0\x42\0\0\0\x25\x04\0\0\x0d\x1c\x01\0\x58\x01\0\0\x42\0\0\0\x25\x04\0\0\ -\x0d\x1c\x01\0\x60\x01\0\0\x42\0\0\0\x53\x04\0\0\x1b\x20\x01\0\x68\x01\0\0\x42\ -\0\0\0\x53\x04\0\0\x06\x20\x01\0\x70\x01\0\0\x42\0\0\0\x76\x04\0\0\x0d\x28\x01\ -\0\x78\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x80\x01\0\0\x42\0\0\0\x26\x03\0\0\x02\ -\xb4\x01\0\xf8\x01\0\0\x42\0\0\0\x2e\x02\0\0\x01\xc4\x01\0\x10\0\0\0\x31\0\0\0\ -\x07\0\0\0\0\0\0\0\x02\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\ -\0\0\0\x10\0\0\0\x02\0\0\0\xee\0\0\0\0\0\0\0\x20\0\0\0\x08\0\0\0\x1e\x01\0\0\0\ -\0\0\0\x70\0\0\0\x0d\0\0\0\x3e\0\0\0\0\0\0\0\x80\0\0\0\x0d\0\0\0\xee\0\0\0\0\0\ -\0\0\xa0\0\0\0\x0d\0\0\0\x1e\x01\0\0\0\0\0\0\x56\x02\0\0\x12\0\0\0\0\0\0\0\x14\ -\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x14\0\ -\0\0\xee\0\0\0\0\0\0\0\x20\0\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\x28\0\0\0\x08\0\0\ -\0\x1e\x01\0\0\0\0\0\0\x80\0\0\0\x1a\0\0\0\x3e\0\0\0\0\0\0\0\x90\0\0\0\x1a\0\0\ -\0\xee\0\0\0\0\0\0\0\xa8\0\0\0\x1a\0\0\0\x59\x03\0\0\0\0\0\0\xb0\0\0\0\x1a\0\0\ -\0\x5d\x03\0\0\0\0\0\0\xc0\0\0\0\x1f\0\0\0\x8b\x03\0\0\0\0\0\0\xd8\0\0\0\x20\0\ -\0\0\xee\0\0\0\0\0\0\0\xf0\0\0\0\x20\0\0\0\x3e\0\0\0\0\0\0\0\x18\x01\0\0\x24\0\ -\0\0\x3e\0\0\0\0\0\0\0\x50\x01\0\0\x1a\0\0\0\xee\0\0\0\0\0\0\0\x60\x01\0\0\x20\ -\0\0\0\x4d\x04\0\0\0\0\0\0\x88\x01\0\0\x1a\0\0\0\x1e\x01\0\0\0\0\0\0\x98\x01\0\ -\0\x1a\0\0\0\x8e\x04\0\0\0\0\0\0\xa0\x01\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd6\0\0\0\0\0\x02\0\x70\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\xc8\0\0\0\0\0\x02\0\xf0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\xcf\0\0\0\0\0\x03\0\x78\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc1\0\0\0\0\0\x03\0\x80\ -\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xba\0\0\0\0\0\x03\0\xf8\x01\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\x14\0\0\0\x01\0\x04\0\0\0\0\0\0\0\0\0\x23\0\0\0\0\0\0\0\xf4\0\0\0\ -\x01\0\x04\0\x23\0\0\0\0\0\0\0\x0e\0\0\0\0\0\0\0\x28\0\0\0\x01\0\x04\0\x31\0\0\ -\0\0\0\0\0\x20\0\0\0\0\0\0\0\xdd\0\0\0\x01\0\x04\0\x51\0\0\0\0\0\0\0\x11\0\0\0\ -\0\0\0\0\0\0\0\0\x03\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x03\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\xb2\0\0\0\x11\0\x05\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\x3d\0\0\0\x12\ -\0\x02\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\x5b\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\ -\0\x08\x02\0\0\0\0\0\0\x48\0\0\0\0\0\0\0\x01\0\0\0\x0c\0\0\0\xc8\0\0\0\0\0\0\0\ -\x01\0\0\0\x0c\0\0\0\x50\0\0\0\0\0\0\0\x01\0\0\0\x0c\0\0\0\xd0\x01\0\0\0\0\0\0\ -\x01\0\0\0\x0c\0\0\0\xf0\x03\0\0\0\0\0\0\x0a\0\0\0\x0c\0\0\0\xfc\x03\0\0\0\0\0\ -\0\x0a\0\0\0\x0c\0\0\0\x08\x04\0\0\0\0\0\0\x0a\0\0\0\x0c\0\0\0\x14\x04\0\0\0\0\ -\0\0\x0a\0\0\0\x0c\0\0\0\x2c\x04\0\0\0\0\0\0\0\0\0\0\x0d\0\0\0\x2c\0\0\0\0\0\0\ -\0\0\0\0\0\x0a\0\0\0\x3c\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x50\0\0\0\0\0\0\0\0\0\ -\0\0\x0a\0\0\0\x60\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\x70\0\0\0\0\0\0\0\0\0\0\0\ -\x0a\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\x90\0\0\0\0\0\0\0\0\0\0\0\x0a\0\ -\0\0\xa0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xb0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\ -\xc0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xd0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xe8\0\ -\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x08\x01\0\0\ -\0\0\0\0\0\0\0\0\x0b\0\0\0\x18\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x28\x01\0\0\0\ -\0\0\0\0\0\0\0\x0b\0\0\0\x38\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x48\x01\0\0\0\0\ -\0\0\0\0\0\0\x0b\0\0\0\x58\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x68\x01\0\0\0\0\0\ -\0\0\0\0\0\x0b\0\0\0\x78\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x88\x01\0\0\0\0\0\0\ -\0\0\0\0\x0b\0\0\0\x98\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xa8\x01\0\0\0\0\0\0\0\ -\0\0\0\x0b\0\0\0\xb8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xc8\x01\0\0\0\0\0\0\0\0\ -\0\0\x0b\0\0\0\xd8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xe8\x01\0\0\0\0\0\0\0\0\0\ -\0\x0b\0\0\0\xf8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x08\x02\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x28\x02\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\x38\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x48\x02\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\x58\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x68\x02\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\x78\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x94\x02\0\0\0\0\0\0\0\0\0\0\ -\x0a\0\0\0\xa4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xb4\x02\0\0\0\0\0\0\0\0\0\0\ -\x0a\0\0\0\xc4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xd4\x02\0\0\0\0\0\0\0\0\0\0\ -\x0a\0\0\0\xe4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xf4\x02\0\0\0\0\0\0\0\0\0\0\ -\x0a\0\0\0\x0c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x1c\x03\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\x2c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x3c\x03\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\x4c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x5c\x03\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\x6c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x7c\x03\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\x8c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x9c\x03\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\xac\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xbc\x03\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\xcc\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xdc\x03\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\xec\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xfc\x03\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\x0c\x04\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x1c\x04\0\0\0\0\0\0\0\0\0\0\ -\x0b\0\0\0\x4e\x4f\x41\x42\x43\x44\x4d\0\x2e\x74\x65\x78\x74\0\x2e\x72\x65\x6c\ -\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\ -\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\ -\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\ -\x61\x70\0\x2e\x72\x65\x6c\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\ -\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x69\x74\ -\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x2e\x6c\x6c\x76\x6d\x5f\x61\x64\ -\x64\x72\x73\x69\x67\0\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x73\x74\x72\x74\x61\ -\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x2e\x72\x6f\x64\x61\x74\x61\0\x2e\x72\x65\ -\x6c\x2e\x42\x54\x46\0\x4c\x49\x43\x45\x4e\x53\x45\0\x4c\x42\x42\x31\x5f\x37\0\ -\x4c\x42\x42\x31\x5f\x36\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\x42\x31\x5f\x33\0\ -\x4c\x42\x42\x30\x5f\x33\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\ -\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x32\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\ -\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x31\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\x30\x3a\x30\0\x2f\x68\x6f\x6d\x65\x2f\x61\x6c\x72\x75\x61\x2f\x62\x75\x69\x6c\ +\x64\x2f\x6c\x69\x6e\x75\x78\x2f\x6b\x65\x72\x6e\x65\x6c\x2f\x62\x70\x66\x2f\ +\x70\x72\x65\x6c\x6f\x61\x64\x2f\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2f\x69\ +\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\x2e\x63\0\x09\x73\x74\x72\x75\ +\x63\x74\x20\x73\x65\x71\x5f\x66\x69\x6c\x65\x20\x2a\x73\x65\x71\x20\x3d\x20\ +\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\x65\x71\x3b\0\x62\x70\x66\x5f\ +\x69\x74\x65\x72\x5f\x6d\x65\x74\x61\0\x73\x65\x71\0\x73\x65\x73\x73\x69\x6f\ +\x6e\x5f\x69\x64\0\x73\x65\x71\x5f\x6e\x75\x6d\0\x73\x65\x71\x5f\x66\x69\x6c\ +\x65\0\x5f\x5f\x75\x36\x34\0\x6c\x6f\x6e\x67\x20\x6c\x6f\x6e\x67\x20\x75\x6e\ +\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x30\x3a\x31\0\x09\x73\x74\x72\x75\ +\x63\x74\x20\x62\x70\x66\x5f\x6d\x61\x70\x20\x2a\x6d\x61\x70\x20\x3d\x20\x63\ +\x74\x78\x2d\x3e\x6d\x61\x70\x3b\0\x09\x69\x66\x20\x28\x21\x6d\x61\x70\x29\0\ +\x30\x3a\x32\0\x09\x5f\x5f\x75\x36\x34\x20\x73\x65\x71\x5f\x6e\x75\x6d\x20\x3d\ +\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\x65\x71\x5f\x6e\x75\x6d\ +\x3b\0\x09\x69\x66\x20\x28\x73\x65\x71\x5f\x6e\x75\x6d\x20\x3d\x3d\x20\x30\x29\ +\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\ +\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\ +\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x5c\ +\x6e\x22\x29\x3b\0\x62\x70\x66\x5f\x6d\x61\x70\0\x69\x64\0\x6e\x61\x6d\x65\0\ +\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x5f\x5f\x75\x33\x32\0\x75\x6e\ +\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x63\x68\x61\x72\0\x5f\x5f\x41\x52\ +\x52\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\x5f\0\x09\x42\x50\x46\ +\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\ +\x34\x75\x20\x25\x2d\x31\x36\x73\x25\x36\x64\x5c\x6e\x22\x2c\x20\x6d\x61\x70\ +\x2d\x3e\x69\x64\x2c\x20\x6d\x61\x70\x2d\x3e\x6e\x61\x6d\x65\x2c\x20\x6d\x61\ +\x70\x2d\x3e\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x29\x3b\0\x7d\0\x62\ +\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x70\x72\ +\x6f\x67\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x69\x74\x65\ +\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x09\x73\x74\x72\x75\x63\x74\x20\x62\ +\x70\x66\x5f\x70\x72\x6f\x67\x20\x2a\x70\x72\x6f\x67\x20\x3d\x20\x63\x74\x78\ +\x2d\x3e\x70\x72\x6f\x67\x3b\0\x09\x69\x66\x20\x28\x21\x70\x72\x6f\x67\x29\0\ +\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x61\x75\x78\0\x09\x61\x75\x78\x20\x3d\x20\ +\x70\x72\x6f\x67\x2d\x3e\x61\x75\x78\x3b\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\ +\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\ +\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\ +\x74\x61\x63\x68\x65\x64\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\x5f\x70\x72\x6f\x67\ +\x5f\x61\x75\x78\0\x61\x74\x74\x61\x63\x68\x5f\x66\x75\x6e\x63\x5f\x6e\x61\x6d\ +\x65\0\x64\x73\x74\x5f\x70\x72\x6f\x67\0\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\ +\x62\x74\x66\0\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\ +\x73\x65\x71\x2c\x20\x22\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\ +\x25\x73\x5c\x6e\x22\x2c\x20\x61\x75\x78\x2d\x3e\x69\x64\x2c\0\x30\x3a\x34\0\ +\x30\x3a\x35\0\x09\x69\x66\x20\x28\x21\x62\x74\x66\x29\0\x62\x70\x66\x5f\x66\ +\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x69\x6e\x73\x6e\x5f\x6f\x66\x66\0\x74\x79\ +\x70\x65\x5f\x69\x64\0\x30\0\x73\x74\x72\x69\x6e\x67\x73\0\x74\x79\x70\x65\x73\ +\0\x68\x64\x72\0\x62\x74\x66\x5f\x68\x65\x61\x64\x65\x72\0\x73\x74\x72\x5f\x6c\ +\x65\x6e\0\x09\x74\x79\x70\x65\x73\x20\x3d\x20\x62\x74\x66\x2d\x3e\x74\x79\x70\ +\x65\x73\x3b\0\x09\x62\x70\x66\x5f\x70\x72\x6f\x62\x65\x5f\x72\x65\x61\x64\x5f\ +\x6b\x65\x72\x6e\x65\x6c\x28\x26\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\ +\x29\x2c\x20\x74\x79\x70\x65\x73\x20\x2b\x20\x62\x74\x66\x5f\x69\x64\x29\x3b\0\ +\x09\x73\x74\x72\x20\x3d\x20\x62\x74\x66\x2d\x3e\x73\x74\x72\x69\x6e\x67\x73\ +\x3b\0\x62\x74\x66\x5f\x74\x79\x70\x65\0\x6e\x61\x6d\x65\x5f\x6f\x66\x66\0\x09\ +\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3d\x20\x42\x50\x46\x5f\x43\x4f\x52\x45\ +\x5f\x52\x45\x41\x44\x28\x74\x2c\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x29\x3b\0\ +\x30\x3a\x32\x3a\x30\0\x09\x69\x66\x20\x28\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\ +\x3e\x3d\x20\x62\x74\x66\x2d\x3e\x68\x64\x72\x2e\x73\x74\x72\x5f\x6c\x65\x6e\ +\x29\0\x09\x72\x65\x74\x75\x72\x6e\x20\x73\x74\x72\x20\x2b\x20\x6e\x61\x6d\x65\ +\x5f\x6f\x66\x66\x3b\0\x30\x3a\x33\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\ +\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\ +\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x31\0\x64\x75\x6d\x70\x5f\x62\x70\x66\ +\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\ +\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x32\0\x4c\x49\x43\x45\ +\x4e\x53\x45\0\x2e\x72\x6f\x64\x61\x74\x61\0\x6c\x69\x63\x65\x6e\x73\x65\0\x9f\ +\xeb\x01\0\x20\0\0\0\0\0\0\0\x24\0\0\0\x24\0\0\0\x44\x02\0\0\x68\x02\0\0\xa4\ +\x01\0\0\x08\0\0\0\x31\0\0\0\x01\0\0\0\0\0\0\0\x07\0\0\0\x62\x02\0\0\x01\0\0\0\ +\0\0\0\0\x17\0\0\0\x10\0\0\0\x31\0\0\0\x09\0\0\0\0\0\0\0\x42\0\0\0\x87\0\0\0\ +\x1e\x40\x01\0\x08\0\0\0\x42\0\0\0\x87\0\0\0\x24\x40\x01\0\x10\0\0\0\x42\0\0\0\ +\xfe\0\0\0\x1d\x48\x01\0\x18\0\0\0\x42\0\0\0\x1f\x01\0\0\x06\x50\x01\0\x20\0\0\ +\0\x42\0\0\0\x2e\x01\0\0\x1d\x44\x01\0\x28\0\0\0\x42\0\0\0\x53\x01\0\0\x06\x5c\ +\x01\0\x38\0\0\0\x42\0\0\0\x66\x01\0\0\x03\x60\x01\0\x70\0\0\0\x42\0\0\0\xec\ +\x01\0\0\x02\x68\x01\0\xf0\0\0\0\x42\0\0\0\x3a\x02\0\0\x01\x70\x01\0\x62\x02\0\ +\0\x1a\0\0\0\0\0\0\0\x42\0\0\0\x87\0\0\0\x1e\x84\x01\0\x08\0\0\0\x42\0\0\0\x87\ +\0\0\0\x24\x84\x01\0\x10\0\0\0\x42\0\0\0\x70\x02\0\0\x1f\x8c\x01\0\x18\0\0\0\ +\x42\0\0\0\x94\x02\0\0\x06\x98\x01\0\x20\0\0\0\x42\0\0\0\xad\x02\0\0\x0e\xa4\ +\x01\0\x28\0\0\0\x42\0\0\0\x2e\x01\0\0\x1d\x88\x01\0\x30\0\0\0\x42\0\0\0\x53\ +\x01\0\0\x06\xa8\x01\0\x40\0\0\0\x42\0\0\0\xbf\x02\0\0\x03\xac\x01\0\x80\0\0\0\ +\x42\0\0\0\x2f\x03\0\0\x02\xb4\x01\0\xb8\0\0\0\x42\0\0\0\x6a\x03\0\0\x06\x08\ +\x01\0\xd0\0\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\xd8\0\0\0\x42\0\0\0\xbb\x03\0\0\x0f\ +\x14\x01\0\xe0\0\0\0\x42\0\0\0\xd0\x03\0\0\x2d\x18\x01\0\xf0\0\0\0\x42\0\0\0\ +\x07\x04\0\0\x0d\x10\x01\0\0\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x08\x01\0\0\x42\ +\0\0\0\xd0\x03\0\0\x02\x18\x01\0\x20\x01\0\0\x42\0\0\0\x2e\x04\0\0\x0d\x1c\x01\ +\0\x38\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x40\x01\0\0\x42\0\0\0\x2e\x04\0\0\x0d\ +\x1c\x01\0\x58\x01\0\0\x42\0\0\0\x2e\x04\0\0\x0d\x1c\x01\0\x60\x01\0\0\x42\0\0\ +\0\x5c\x04\0\0\x1b\x20\x01\0\x68\x01\0\0\x42\0\0\0\x5c\x04\0\0\x06\x20\x01\0\ +\x70\x01\0\0\x42\0\0\0\x7f\x04\0\0\x0d\x28\x01\0\x78\x01\0\0\x42\0\0\0\0\0\0\0\ +\0\0\0\0\x80\x01\0\0\x42\0\0\0\x2f\x03\0\0\x02\xb4\x01\0\xf8\x01\0\0\x42\0\0\0\ +\x3a\x02\0\0\x01\xc4\x01\0\x10\0\0\0\x31\0\0\0\x07\0\0\0\0\0\0\0\x02\0\0\0\x3e\ +\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x02\0\0\0\xfa\0\ +\0\0\0\0\0\0\x20\0\0\0\x08\0\0\0\x2a\x01\0\0\0\0\0\0\x70\0\0\0\x0d\0\0\0\x3e\0\ +\0\0\0\0\0\0\x80\0\0\0\x0d\0\0\0\xfa\0\0\0\0\0\0\0\xa0\0\0\0\x0d\0\0\0\x2a\x01\ +\0\0\0\0\0\0\x62\x02\0\0\x12\0\0\0\0\0\0\0\x14\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\ +\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x14\0\0\0\xfa\0\0\0\0\0\0\0\x20\0\0\0\ +\x18\0\0\0\x3e\0\0\0\0\0\0\0\x28\0\0\0\x08\0\0\0\x2a\x01\0\0\0\0\0\0\x80\0\0\0\ +\x1a\0\0\0\x3e\0\0\0\0\0\0\0\x90\0\0\0\x1a\0\0\0\xfa\0\0\0\0\0\0\0\xa8\0\0\0\ +\x1a\0\0\0\x62\x03\0\0\0\0\0\0\xb0\0\0\0\x1a\0\0\0\x66\x03\0\0\0\0\0\0\xc0\0\0\ +\0\x1f\0\0\0\x94\x03\0\0\0\0\0\0\xd8\0\0\0\x20\0\0\0\xfa\0\0\0\0\0\0\0\xf0\0\0\ +\0\x20\0\0\0\x3e\0\0\0\0\0\0\0\x18\x01\0\0\x24\0\0\0\x3e\0\0\0\0\0\0\0\x50\x01\ +\0\0\x1a\0\0\0\xfa\0\0\0\0\0\0\0\x60\x01\0\0\x20\0\0\0\x56\x04\0\0\0\0\0\0\x88\ +\x01\0\0\x1a\0\0\0\x2a\x01\0\0\0\0\0\0\x98\x01\0\0\x1a\0\0\0\x97\x04\0\0\0\0\0\ +\0\xa0\x01\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\x91\0\0\0\x04\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe6\0\0\ +\0\0\0\x02\0\x70\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd8\0\0\0\0\0\x02\0\xf0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\xdf\0\0\0\0\0\x03\0\x78\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\xd1\0\0\0\0\0\x03\0\x80\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xca\0\0\0\0\0\x03\0\ +\xf8\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x14\0\0\0\x01\0\x04\0\0\0\0\0\0\0\0\0\x23\ +\0\0\0\0\0\0\0\x04\x01\0\0\x01\0\x04\0\x23\0\0\0\0\0\0\0\x0e\0\0\0\0\0\0\0\x28\ +\0\0\0\x01\0\x04\0\x31\0\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\xed\0\0\0\x01\0\x04\0\ +\x51\0\0\0\0\0\0\0\x11\0\0\0\0\0\0\0\0\0\0\0\x03\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\ +\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc2\0\0\0\x11\0\x05\0\0\0\0\0\0\0\0\0\ +\x04\0\0\0\0\0\0\0\x3d\0\0\0\x12\0\x02\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\x5b\ +\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\0\x08\x02\0\0\0\0\0\0\x48\0\0\0\0\0\0\0\x01\0\ +\0\0\x0d\0\0\0\xc8\0\0\0\0\0\0\0\x01\0\0\0\x0d\0\0\0\x50\0\0\0\0\0\0\0\x01\0\0\ +\0\x0d\0\0\0\xd0\x01\0\0\0\0\0\0\x01\0\0\0\x0d\0\0\0\xf0\x03\0\0\0\0\0\0\x0a\0\ +\0\0\x0d\0\0\0\xfc\x03\0\0\0\0\0\0\x0a\0\0\0\x0d\0\0\0\x08\x04\0\0\0\0\0\0\x0a\ +\0\0\0\x0d\0\0\0\x14\x04\0\0\0\0\0\0\x0a\0\0\0\x0d\0\0\0\x2c\x04\0\0\0\0\0\0\0\ +\0\0\0\x0e\0\0\0\x2c\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x3c\0\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x50\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x60\0\0\0\0\0\0\0\0\0\0\0\x0b\0\ +\0\0\x70\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\ +\x90\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xa0\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xb0\0\ +\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xc0\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xd0\0\0\0\0\ +\0\0\0\0\0\0\0\x0b\0\0\0\xe8\0\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xf8\0\0\0\0\0\0\0\ +\0\0\0\0\x0c\0\0\0\x08\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x18\x01\0\0\0\0\0\0\0\ +\0\0\0\x0c\0\0\0\x28\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x38\x01\0\0\0\0\0\0\0\0\ +\0\0\x0c\0\0\0\x48\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x58\x01\0\0\0\0\0\0\0\0\0\ +\0\x0c\0\0\0\x68\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x78\x01\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x88\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x98\x01\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\xa8\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xb8\x01\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\xc8\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xd8\x01\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\xe8\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xf8\x01\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x08\x02\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x28\x02\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x38\x02\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x48\x02\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x58\x02\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x68\x02\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x78\x02\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x94\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xa4\x02\0\0\0\0\0\0\0\0\0\0\ +\x0b\0\0\0\xb4\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xc4\x02\0\0\0\0\0\0\0\0\0\0\ +\x0b\0\0\0\xd4\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xe4\x02\0\0\0\0\0\0\0\0\0\0\ +\x0b\0\0\0\xf4\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x0c\x03\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x1c\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x2c\x03\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x3c\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x4c\x03\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x5c\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x6c\x03\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x7c\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x8c\x03\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x9c\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xac\x03\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\xbc\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xcc\x03\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\xdc\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xec\x03\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\xfc\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x0c\x04\0\0\0\0\0\0\0\0\0\0\ +\x0c\0\0\0\x1c\x04\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x4d\x4e\x40\x41\x42\x43\x4c\0\ +\x2e\x74\x65\x78\x74\0\x2e\x72\x65\x6c\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x64\ +\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\ +\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\0\ +\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x2e\x72\x65\x6c\x69\x74\x65\ +\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\ +\x72\x6f\x67\0\x2e\x72\x65\x6c\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\ +\x67\0\x2e\x6c\x6c\x76\x6d\x5f\x61\x64\x64\x72\x73\x69\x67\0\x6c\x69\x63\x65\ +\x6e\x73\x65\0\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\x2e\x63\0\ +\x2e\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x2e\x72\x6f\x64\ +\x61\x74\x61\0\x2e\x72\x65\x6c\x2e\x42\x54\x46\0\x4c\x49\x43\x45\x4e\x53\x45\0\ +\x4c\x42\x42\x31\x5f\x37\0\x4c\x42\x42\x31\x5f\x36\0\x4c\x42\x42\x30\x5f\x34\0\ +\x4c\x42\x42\x31\x5f\x33\0\x4c\x42\x42\x30\x5f\x33\0\x64\x75\x6d\x70\x5f\x62\ +\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x32\0\x64\x75\x6d\ +\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x31\0\0\0\ \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\x4e\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\ -\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\x6d\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x01\0\0\0\0\0\0\x08\ -\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa1\0\0\0\ -\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x03\0\0\0\0\0\0\x62\0\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x89\0\0\0\x01\0\0\0\x03\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\x03\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xad\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\0\xae\x03\0\0\0\0\0\0\x34\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\xe2\x0c\0\0\0\0\0\0\x2c\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\x99\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\x11\0\0\0\ -\0\0\0\x80\x01\0\0\0\0\0\0\x0e\0\0\0\x0d\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\ -\0\0\x4a\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x90\x12\0\0\0\0\0\0\ -\x20\0\0\0\0\0\0\0\x08\0\0\0\x02\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x69\ -\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb0\x12\0\0\0\0\0\0\x20\0\0\0\ -\0\0\0\0\x08\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\xa9\0\0\0\x09\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd0\x12\0\0\0\0\0\0\x50\0\0\0\0\0\0\0\ -\x08\0\0\0\x06\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x07\0\0\0\x09\0\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x13\0\0\0\0\0\0\xe0\x03\0\0\0\0\0\0\x08\0\0\ -\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x7b\0\0\0\x03\x4c\xff\x6f\0\0\ -\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\0\x17\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x91\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\ -\0\0\0\0\0\0\x07\x17\0\0\0\0\0\0\x0a\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\ -\0\0\0\0\0\0\0\0\0\0\0\0"; +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\ +\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4e\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\x6d\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\x40\x01\0\0\0\0\0\0\x08\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\xb1\0\0\0\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x03\0\ +\0\0\0\0\0\x62\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\x89\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\x03\0\0\0\0\0\0\x04\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbd\0\0\0\x01\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xae\x03\0\0\0\0\0\0\x3d\x09\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x01\0\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\0\0\0\0\xeb\x0c\0\0\0\0\0\0\x2c\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa9\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\0\0\0\0\x18\x11\0\0\0\0\0\0\x98\x01\0\0\0\0\0\0\x0e\0\0\0\x0e\0\0\0\x08\0\0\ +\0\0\0\0\0\x18\0\0\0\0\0\0\0\x4a\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\ +\0\xb0\x12\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\x08\0\0\0\x02\0\0\0\x08\0\0\0\0\0\0\0\ +\x10\0\0\0\0\0\0\0\x69\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd0\x12\ +\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\x08\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\ +\0\0\0\0\xb9\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf0\x12\0\0\0\0\0\ +\0\x50\0\0\0\0\0\0\0\x08\0\0\0\x06\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\ +\x07\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x13\0\0\0\0\0\0\xe0\ +\x03\0\0\0\0\0\0\x08\0\0\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x7b\0\ +\0\0\x03\x4c\xff\x6f\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\x20\x17\0\0\0\0\0\0\x07\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa1\0\0\0\x03\ +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x27\x17\0\0\0\0\0\0\x1a\x01\0\0\0\0\0\0\ +\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; return 0; err: diff --git a/kernel/bpf/reuseport_array.c b/kernel/bpf/reuseport_array.c index 5a2ba1182493..a55cd542f2ce 100644 --- a/kernel/bpf/reuseport_array.c +++ b/kernel/bpf/reuseport_array.c @@ -191,7 +191,7 @@ int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, rcu_read_lock(); sk = reuseport_array_lookup_elem(map, key); if (sk) { - *(u64 *)value = sock_gen_cookie(sk); + *(u64 *)value = __sock_gen_cookie(sk); err = 0; } else { err = -ENOENT; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 34268491d2de..f1528c2a6927 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4,6 +4,7 @@ #include <linux/bpf.h> #include <linux/bpf_trace.h> #include <linux/bpf_lirc.h> +#include <linux/bpf_verifier.h> #include <linux/btf.h> #include <linux/syscalls.h> #include <linux/slab.h> @@ -2154,14 +2155,14 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) prog->expected_attach_type = attr->expected_attach_type; prog->aux->attach_btf_id = attr->attach_btf_id; if (attr->attach_prog_fd) { - struct bpf_prog *tgt_prog; + struct bpf_prog *dst_prog; - tgt_prog = bpf_prog_get(attr->attach_prog_fd); - if (IS_ERR(tgt_prog)) { - err = PTR_ERR(tgt_prog); + dst_prog = bpf_prog_get(attr->attach_prog_fd); + if (IS_ERR(dst_prog)) { + err = PTR_ERR(dst_prog); goto free_prog_nouncharge; } - prog->aux->linked_prog = tgt_prog; + prog->aux->dst_prog = dst_prog; } prog->aux->offload_requested = !!attr->prog_ifindex; @@ -2345,8 +2346,12 @@ void bpf_link_put(struct bpf_link *link) if (!atomic64_dec_and_test(&link->refcnt)) return; - INIT_WORK(&link->work, bpf_link_put_deferred); - schedule_work(&link->work); + if (in_atomic()) { + INIT_WORK(&link->work, bpf_link_put_deferred); + schedule_work(&link->work); + } else { + bpf_link_free(link); + } } static int bpf_link_release(struct inode *inode, struct file *filp) @@ -2494,11 +2499,23 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd) struct bpf_tracing_link { struct bpf_link link; enum bpf_attach_type attach_type; + struct bpf_trampoline *trampoline; + struct bpf_prog *tgt_prog; }; static void bpf_tracing_link_release(struct bpf_link *link) { - WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog)); + struct bpf_tracing_link *tr_link = + container_of(link, struct bpf_tracing_link, link); + + WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog, + tr_link->trampoline)); + + bpf_trampoline_put(tr_link->trampoline); + + /* tgt_prog is NULL if target is a kernel function */ + if (tr_link->tgt_prog) + bpf_prog_put(tr_link->tgt_prog); } static void bpf_tracing_link_dealloc(struct bpf_link *link) @@ -2538,10 +2555,15 @@ static const struct bpf_link_ops bpf_tracing_link_lops = { .fill_link_info = bpf_tracing_link_fill_link_info, }; -static int bpf_tracing_prog_attach(struct bpf_prog *prog) +static int bpf_tracing_prog_attach(struct bpf_prog *prog, + int tgt_prog_fd, + u32 btf_id) { struct bpf_link_primer link_primer; + struct bpf_prog *tgt_prog = NULL; + struct bpf_trampoline *tr = NULL; struct bpf_tracing_link *link; + u64 key = 0; int err; switch (prog->type) { @@ -2570,6 +2592,28 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog) goto out_put_prog; } + if (!!tgt_prog_fd != !!btf_id) { + err = -EINVAL; + goto out_put_prog; + } + + if (tgt_prog_fd) { + /* For now we only allow new targets for BPF_PROG_TYPE_EXT */ + if (prog->type != BPF_PROG_TYPE_EXT) { + err = -EINVAL; + goto out_put_prog; + } + + tgt_prog = bpf_prog_get(tgt_prog_fd); + if (IS_ERR(tgt_prog)) { + err = PTR_ERR(tgt_prog); + tgt_prog = NULL; + goto out_put_prog; + } + + key = bpf_trampoline_compute_key(tgt_prog, btf_id); + } + link = kzalloc(sizeof(*link), GFP_USER); if (!link) { err = -ENOMEM; @@ -2579,20 +2623,100 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog) &bpf_tracing_link_lops, prog); link->attach_type = prog->expected_attach_type; - err = bpf_link_prime(&link->link, &link_primer); - if (err) { - kfree(link); - goto out_put_prog; + mutex_lock(&prog->aux->dst_mutex); + + /* There are a few possible cases here: + * + * - if prog->aux->dst_trampoline is set, the program was just loaded + * and not yet attached to anything, so we can use the values stored + * in prog->aux + * + * - if prog->aux->dst_trampoline is NULL, the program has already been + * attached to a target and its initial target was cleared (below) + * + * - if tgt_prog != NULL, the caller specified tgt_prog_fd + + * target_btf_id using the link_create API. + * + * - if tgt_prog == NULL when this function was called using the old + * raw_tracepoint_open API, and we need a target from prog->aux + * + * The combination of no saved target in prog->aux, and no target + * specified on load is illegal, and we reject that here. + */ + if (!prog->aux->dst_trampoline && !tgt_prog) { + err = -ENOENT; + goto out_unlock; } - err = bpf_trampoline_link_prog(prog); + if (!prog->aux->dst_trampoline || + (key && key != prog->aux->dst_trampoline->key)) { + /* If there is no saved target, or the specified target is + * different from the destination specified at load time, we + * need a new trampoline and a check for compatibility + */ + struct bpf_attach_target_info tgt_info = {}; + + err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id, + &tgt_info); + if (err) + goto out_unlock; + + tr = bpf_trampoline_get(key, &tgt_info); + if (!tr) { + err = -ENOMEM; + goto out_unlock; + } + } else { + /* The caller didn't specify a target, or the target was the + * same as the destination supplied during program load. This + * means we can reuse the trampoline and reference from program + * load time, and there is no need to allocate a new one. This + * can only happen once for any program, as the saved values in + * prog->aux are cleared below. + */ + tr = prog->aux->dst_trampoline; + tgt_prog = prog->aux->dst_prog; + } + + err = bpf_link_prime(&link->link, &link_primer); + if (err) + goto out_unlock; + + err = bpf_trampoline_link_prog(prog, tr); if (err) { bpf_link_cleanup(&link_primer); - goto out_put_prog; + link = NULL; + goto out_unlock; } + link->tgt_prog = tgt_prog; + link->trampoline = tr; + + /* Always clear the trampoline and target prog from prog->aux to make + * sure the original attach destination is not kept alive after a + * program is (re-)attached to another target. + */ + if (prog->aux->dst_prog && + (tgt_prog_fd || tr != prog->aux->dst_trampoline)) + /* got extra prog ref from syscall, or attaching to different prog */ + bpf_prog_put(prog->aux->dst_prog); + if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline) + /* we allocated a new trampoline, so free the old one */ + bpf_trampoline_put(prog->aux->dst_trampoline); + + prog->aux->dst_prog = NULL; + prog->aux->dst_trampoline = NULL; + mutex_unlock(&prog->aux->dst_mutex); + return bpf_link_settle(&link_primer); +out_unlock: + if (tr && tr != prog->aux->dst_trampoline) + bpf_trampoline_put(tr); + mutex_unlock(&prog->aux->dst_mutex); + kfree(link); out_put_prog: + if (tgt_prog_fd && tgt_prog) + bpf_prog_put(tgt_prog); bpf_prog_put(prog); return err; } @@ -2706,7 +2830,7 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr) tp_name = prog->aux->attach_func_name; break; } - return bpf_tracing_prog_attach(prog); + return bpf_tracing_prog_attach(prog, 0, 0); case BPF_PROG_TYPE_RAW_TRACEPOINT: case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: if (strncpy_from_user(buf, @@ -2975,7 +3099,7 @@ static int bpf_prog_query(const union bpf_attr *attr, } } -#define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out +#define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu static int bpf_prog_test_run(const union bpf_attr *attr, union bpf_attr __user *uattr) @@ -3890,10 +4014,15 @@ err_put: static int tracing_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) { - if (attr->link_create.attach_type == BPF_TRACE_ITER && - prog->expected_attach_type == BPF_TRACE_ITER) - return bpf_iter_link_attach(attr, prog); + if (attr->link_create.attach_type != prog->expected_attach_type) + return -EINVAL; + if (prog->expected_attach_type == BPF_TRACE_ITER) + return bpf_iter_link_attach(attr, prog); + else if (prog->type == BPF_PROG_TYPE_EXT) + return bpf_tracing_prog_attach(prog, + attr->link_create.target_fd, + attr->link_create.target_btf_id); return -EINVAL; } @@ -3907,18 +4036,25 @@ static int link_create(union bpf_attr *attr) if (CHECK_ATTR(BPF_LINK_CREATE)) return -EINVAL; - ptype = attach_type_to_prog_type(attr->link_create.attach_type); - if (ptype == BPF_PROG_TYPE_UNSPEC) - return -EINVAL; - - prog = bpf_prog_get_type(attr->link_create.prog_fd, ptype); + prog = bpf_prog_get(attr->link_create.prog_fd); if (IS_ERR(prog)) return PTR_ERR(prog); ret = bpf_prog_attach_check_attach_type(prog, attr->link_create.attach_type); if (ret) - goto err_out; + goto out; + + if (prog->type == BPF_PROG_TYPE_EXT) { + ret = tracing_bpf_link_attach(attr, prog); + goto out; + } + + ptype = attach_type_to_prog_type(attr->link_create.attach_type); + if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) { + ret = -EINVAL; + goto out; + } switch (ptype) { case BPF_PROG_TYPE_CGROUP_SKB: @@ -3946,7 +4082,7 @@ static int link_create(union bpf_attr *attr) ret = -EINVAL; } -err_out: +out: if (ret < 0) bpf_prog_put(prog); return ret; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 7dd523a7e32d..35c5887d82ff 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -65,7 +65,7 @@ static void bpf_trampoline_ksym_add(struct bpf_trampoline *tr) bpf_image_ksym_add(tr->image, ksym); } -struct bpf_trampoline *bpf_trampoline_lookup(u64 key) +static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) { struct bpf_trampoline *tr; struct hlist_head *head; @@ -261,14 +261,12 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) } } -int bpf_trampoline_link_prog(struct bpf_prog *prog) +int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) { enum bpf_tramp_prog_type kind; - struct bpf_trampoline *tr; int err = 0; int cnt; - tr = prog->aux->trampoline; kind = bpf_attach_type_to_tramp(prog); mutex_lock(&tr->mutex); if (tr->extension_prog) { @@ -301,7 +299,7 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog) } hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]); tr->progs_cnt[kind]++; - err = bpf_trampoline_update(prog->aux->trampoline); + err = bpf_trampoline_update(tr); if (err) { hlist_del(&prog->aux->tramp_hlist); tr->progs_cnt[kind]--; @@ -312,13 +310,11 @@ out: } /* bpf_trampoline_unlink_prog() should never fail. */ -int bpf_trampoline_unlink_prog(struct bpf_prog *prog) +int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) { enum bpf_tramp_prog_type kind; - struct bpf_trampoline *tr; int err; - tr = prog->aux->trampoline; kind = bpf_attach_type_to_tramp(prog); mutex_lock(&tr->mutex); if (kind == BPF_TRAMP_REPLACE) { @@ -330,12 +326,32 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog) } hlist_del(&prog->aux->tramp_hlist); tr->progs_cnt[kind]--; - err = bpf_trampoline_update(prog->aux->trampoline); + err = bpf_trampoline_update(tr); out: mutex_unlock(&tr->mutex); return err; } +struct bpf_trampoline *bpf_trampoline_get(u64 key, + struct bpf_attach_target_info *tgt_info) +{ + struct bpf_trampoline *tr; + + tr = bpf_trampoline_lookup(key); + if (!tr) + return NULL; + + mutex_lock(&tr->mutex); + if (tr->func.addr) + goto out; + + memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); + tr->func.addr = (void *)tgt_info->tgt_addr; +out: + mutex_unlock(&tr->mutex); + return tr; +} + void bpf_trampoline_put(struct bpf_trampoline *tr) { if (!tr) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 42dee5dcbc74..015a1c074b6b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -486,7 +486,12 @@ static bool is_acquire_function(enum bpf_func_id func_id, static bool is_ptr_cast_function(enum bpf_func_id func_id) { return func_id == BPF_FUNC_tcp_sock || - func_id == BPF_FUNC_sk_fullsock; + func_id == BPF_FUNC_sk_fullsock || + func_id == BPF_FUNC_skc_to_tcp_sock || + func_id == BPF_FUNC_skc_to_tcp6_sock || + func_id == BPF_FUNC_skc_to_udp6_sock || + func_id == BPF_FUNC_skc_to_tcp_timewait_sock || + func_id == BPF_FUNC_skc_to_tcp_request_sock; } /* string representation of 'enum bpf_reg_type' */ @@ -2643,8 +2648,7 @@ static int check_map_access(struct bpf_verifier_env *env, u32 regno, static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog) { - return prog->aux->linked_prog ? prog->aux->linked_prog->type - : prog->type; + return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type; } static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, @@ -3938,7 +3942,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, case BPF_MAP_TYPE_SOCKMAP: case BPF_MAP_TYPE_SOCKHASH: if (*arg_type == ARG_PTR_TO_MAP_VALUE) { - *arg_type = ARG_PTR_TO_SOCKET; + *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; } else { verbose(env, "invalid arg_type for sockmap/sockhash\n"); return -EINVAL; @@ -3953,6 +3957,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, struct bpf_reg_types { const enum bpf_reg_type types[10]; + u32 *btf_id; }; static const struct bpf_reg_types map_key_value_types = { @@ -3973,6 +3978,17 @@ static const struct bpf_reg_types sock_types = { }, }; +static const struct bpf_reg_types btf_id_sock_common_types = { + .types = { + PTR_TO_SOCK_COMMON, + PTR_TO_SOCKET, + PTR_TO_TCP_SOCK, + PTR_TO_XDP_SOCK, + PTR_TO_BTF_ID, + }, + .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], +}; + static const struct bpf_reg_types mem_types = { .types = { PTR_TO_STACK, @@ -4014,6 +4030,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_CTX] = &context_types, [ARG_PTR_TO_CTX_OR_NULL] = &context_types, [ARG_PTR_TO_SOCK_COMMON] = &sock_types, + [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, [ARG_PTR_TO_SOCKET] = &fullsock_types, [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types, [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, @@ -4028,19 +4045,27 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { }; static int check_reg_type(struct bpf_verifier_env *env, u32 regno, - const struct bpf_reg_types *compatible) + enum bpf_arg_type arg_type, + const u32 *arg_btf_id) { struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; enum bpf_reg_type expected, type = reg->type; + const struct bpf_reg_types *compatible; int i, j; + compatible = compatible_reg_types[arg_type]; + if (!compatible) { + verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); + return -EFAULT; + } + for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { expected = compatible->types[i]; if (expected == NOT_INIT) break; if (type == expected) - return 0; + goto found; } verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]); @@ -4048,6 +4073,33 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, verbose(env, "%s, ", reg_type_str[compatible->types[j]]); verbose(env, "%s\n", reg_type_str[compatible->types[j]]); return -EACCES; + +found: + if (type == PTR_TO_BTF_ID) { + if (!arg_btf_id) { + if (!compatible->btf_id) { + verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); + return -EFAULT; + } + arg_btf_id = compatible->btf_id; + } + + if (!btf_struct_ids_match(&env->log, reg->off, reg->btf_id, + *arg_btf_id)) { + verbose(env, "R%d is of type %s but %s is expected\n", + regno, kernel_type_name(reg->btf_id), + kernel_type_name(*arg_btf_id)); + return -EACCES; + } + + if (!tnum_is_const(reg->var_off) || reg->var_off.value) { + verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n", + regno); + return -EACCES; + } + } + + return 0; } static int check_func_arg(struct bpf_verifier_env *env, u32 arg, @@ -4057,7 +4109,6 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 regno = BPF_REG_1 + arg; struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; enum bpf_arg_type arg_type = fn->arg_type[arg]; - const struct bpf_reg_types *compatible; enum bpf_reg_type type = reg->type; int err = 0; @@ -4097,35 +4148,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, */ goto skip_type_check; - compatible = compatible_reg_types[arg_type]; - if (!compatible) { - verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); - return -EFAULT; - } - - err = check_reg_type(env, regno, compatible); + err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]); if (err) return err; - if (type == PTR_TO_BTF_ID) { - const u32 *btf_id = fn->arg_btf_id[arg]; - - if (!btf_id) { - verbose(env, "verifier internal error: missing BTF ID\n"); - return -EFAULT; - } - - if (!btf_struct_ids_match(&env->log, reg->off, reg->btf_id, *btf_id)) { - verbose(env, "R%d is of type %s but %s is expected\n", - regno, kernel_type_name(reg->btf_id), kernel_type_name(*btf_id)); - return -EACCES; - } - if (!tnum_is_const(reg->var_off) || reg->var_off.value) { - verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n", - regno); - return -EACCES; - } - } else if (type == PTR_TO_CTX) { + if (type == PTR_TO_CTX) { err = check_ctx_reg(env, reg, regno); if (err < 0) return err; @@ -4573,10 +4600,14 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) { int i; - for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) + for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i]) return false; + if (fn->arg_type[i] != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i]) + return false; + } + return true; } @@ -5817,8 +5848,7 @@ static void scalar_min_max_and(struct bpf_reg_state *dst_reg, u64 umax_val = src_reg->umax_value; if (src_known && dst_known) { - __mark_reg_known(dst_reg, dst_reg->var_off.value & - src_reg->var_off.value); + __mark_reg_known(dst_reg, dst_reg->var_off.value); return; } @@ -5888,8 +5918,7 @@ static void scalar_min_max_or(struct bpf_reg_state *dst_reg, u64 umin_val = src_reg->umin_value; if (src_known && dst_known) { - __mark_reg_known(dst_reg, dst_reg->var_off.value | - src_reg->var_off.value); + __mark_reg_known(dst_reg, dst_reg->var_off.value); return; } @@ -11173,11 +11202,10 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) } #define SECURITY_PREFIX "security_" -static int check_attach_modify_return(struct bpf_prog *prog, unsigned long addr) +static int check_attach_modify_return(unsigned long addr, const char *func_name) { if (within_error_injection_list(addr) || - !strncmp(SECURITY_PREFIX, prog->aux->attach_func_name, - sizeof(SECURITY_PREFIX) - 1)) + !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) return 0; return -EINVAL; @@ -11214,55 +11242,39 @@ static int check_non_sleepable_error_inject(u32 btf_id) return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); } -static int check_attach_btf_id(struct bpf_verifier_env *env) +int bpf_check_attach_target(struct bpf_verifier_log *log, + const struct bpf_prog *prog, + const struct bpf_prog *tgt_prog, + u32 btf_id, + struct bpf_attach_target_info *tgt_info) { - struct bpf_prog *prog = env->prog; bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; - struct bpf_prog *tgt_prog = prog->aux->linked_prog; - u32 btf_id = prog->aux->attach_btf_id; const char prefix[] = "btf_trace_"; - struct btf_func_model fmodel; int ret = 0, subprog = -1, i; - struct bpf_trampoline *tr; const struct btf_type *t; bool conservative = true; const char *tname; struct btf *btf; - long addr; - u64 key; - - if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && - prog->type != BPF_PROG_TYPE_LSM) { - verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n"); - return -EINVAL; - } - - if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) - return check_struct_ops_btf_id(env); - - if (prog->type != BPF_PROG_TYPE_TRACING && - prog->type != BPF_PROG_TYPE_LSM && - !prog_extension) - return 0; + long addr = 0; if (!btf_id) { - verbose(env, "Tracing programs must provide btf_id\n"); + bpf_log(log, "Tracing programs must provide btf_id\n"); return -EINVAL; } - btf = bpf_prog_get_target_btf(prog); + btf = tgt_prog ? tgt_prog->aux->btf : btf_vmlinux; if (!btf) { - verbose(env, + bpf_log(log, "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); return -EINVAL; } t = btf_type_by_id(btf, btf_id); if (!t) { - verbose(env, "attach_btf_id %u is invalid\n", btf_id); + bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); return -EINVAL; } tname = btf_name_by_offset(btf, t->name_off); if (!tname) { - verbose(env, "attach_btf_id %u doesn't have a name\n", btf_id); + bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); return -EINVAL; } if (tgt_prog) { @@ -11274,26 +11286,24 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) break; } if (subprog == -1) { - verbose(env, "Subprog %s doesn't exist\n", tname); + bpf_log(log, "Subprog %s doesn't exist\n", tname); return -EINVAL; } conservative = aux->func_info_aux[subprog].unreliable; if (prog_extension) { if (conservative) { - verbose(env, + bpf_log(log, "Cannot replace static functions\n"); return -EINVAL; } if (!prog->jit_requested) { - verbose(env, + bpf_log(log, "Extension programs should be JITed\n"); return -EINVAL; } - env->ops = bpf_verifier_ops[tgt_prog->type]; - prog->expected_attach_type = tgt_prog->expected_attach_type; } if (!tgt_prog->jited) { - verbose(env, "Can attach to only JITed progs\n"); + bpf_log(log, "Can attach to only JITed progs\n"); return -EINVAL; } if (tgt_prog->type == prog->type) { @@ -11301,7 +11311,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) * Cannot attach program extension to another extension. * It's ok to attach fentry/fexit to extension program. */ - verbose(env, "Cannot recursively attach\n"); + bpf_log(log, "Cannot recursively attach\n"); return -EINVAL; } if (tgt_prog->type == BPF_PROG_TYPE_TRACING && @@ -11323,32 +11333,30 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) * reasonable stack size. Hence extending fentry is not * allowed. */ - verbose(env, "Cannot extend fentry/fexit\n"); + bpf_log(log, "Cannot extend fentry/fexit\n"); return -EINVAL; } - key = ((u64)aux->id) << 32 | btf_id; } else { if (prog_extension) { - verbose(env, "Cannot replace kernel functions\n"); + bpf_log(log, "Cannot replace kernel functions\n"); return -EINVAL; } - key = btf_id; } switch (prog->expected_attach_type) { case BPF_TRACE_RAW_TP: if (tgt_prog) { - verbose(env, + bpf_log(log, "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); return -EINVAL; } if (!btf_type_is_typedef(t)) { - verbose(env, "attach_btf_id %u is not a typedef\n", + bpf_log(log, "attach_btf_id %u is not a typedef\n", btf_id); return -EINVAL; } if (strncmp(prefix, tname, sizeof(prefix) - 1)) { - verbose(env, "attach_btf_id %u points to wrong type name %s\n", + bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", btf_id, tname); return -EINVAL; } @@ -11362,29 +11370,20 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) /* should never happen in valid vmlinux build */ return -EINVAL; - /* remember two read only pointers that are valid for - * the life time of the kernel - */ - prog->aux->attach_func_name = tname; - prog->aux->attach_func_proto = t; - prog->aux->attach_btf_trace = true; - return 0; + break; case BPF_TRACE_ITER: if (!btf_type_is_func(t)) { - verbose(env, "attach_btf_id %u is not a function\n", + bpf_log(log, "attach_btf_id %u is not a function\n", btf_id); return -EINVAL; } t = btf_type_by_id(btf, t->type); if (!btf_type_is_func_proto(t)) return -EINVAL; - prog->aux->attach_func_name = tname; - prog->aux->attach_func_proto = t; - if (!bpf_iter_prog_supported(prog)) - return -EINVAL; - ret = btf_distill_func_proto(&env->log, btf, t, - tname, &fmodel); - return ret; + ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); + if (ret) + return ret; + break; default: if (!prog_extension) return -EINVAL; @@ -11393,42 +11392,30 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) case BPF_LSM_MAC: case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: - prog->aux->attach_func_name = tname; - if (prog->type == BPF_PROG_TYPE_LSM) { - ret = bpf_lsm_verify_prog(&env->log, prog); - if (ret < 0) - return ret; - } - if (!btf_type_is_func(t)) { - verbose(env, "attach_btf_id %u is not a function\n", + bpf_log(log, "attach_btf_id %u is not a function\n", btf_id); return -EINVAL; } if (prog_extension && - btf_check_type_match(env, prog, btf, t)) + btf_check_type_match(log, prog, btf, t)) return -EINVAL; t = btf_type_by_id(btf, t->type); if (!btf_type_is_func_proto(t)) return -EINVAL; - tr = bpf_trampoline_lookup(key); - if (!tr) - return -ENOMEM; - /* t is either vmlinux type or another program's type */ - prog->aux->attach_func_proto = t; - mutex_lock(&tr->mutex); - if (tr->func.addr) { - prog->aux->trampoline = tr; - goto out; - } - if (tgt_prog && conservative) { - prog->aux->attach_func_proto = NULL; + + if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && + (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || + prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) + return -EINVAL; + + if (tgt_prog && conservative) t = NULL; - } - ret = btf_distill_func_proto(&env->log, btf, t, - tname, &tr->func.model); + + ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); if (ret < 0) - goto out; + return ret; + if (tgt_prog) { if (subprog == 0) addr = (long) tgt_prog->bpf_func; @@ -11437,11 +11424,10 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) } else { addr = kallsyms_lookup_name(tname); if (!addr) { - verbose(env, + bpf_log(log, "The address of function %s cannot be found\n", tname); - ret = -ENOENT; - goto out; + return -ENOENT; } } @@ -11466,25 +11452,109 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) default: break; } - if (ret) - verbose(env, "%s is not sleepable\n", - prog->aux->attach_func_name); + if (ret) { + bpf_log(log, "%s is not sleepable\n", tname); + return ret; + } } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { - ret = check_attach_modify_return(prog, addr); - if (ret) - verbose(env, "%s() is not modifiable\n", - prog->aux->attach_func_name); + if (tgt_prog) { + bpf_log(log, "can't modify return codes of BPF programs\n"); + return -EINVAL; + } + ret = check_attach_modify_return(addr, tname); + if (ret) { + bpf_log(log, "%s() is not modifiable\n", tname); + return ret; + } } - if (ret) - goto out; - tr->func.addr = (void *)addr; - prog->aux->trampoline = tr; -out: - mutex_unlock(&tr->mutex); - if (ret) - bpf_trampoline_put(tr); + + break; + } + tgt_info->tgt_addr = addr; + tgt_info->tgt_name = tname; + tgt_info->tgt_type = t; + return 0; +} + +static int check_attach_btf_id(struct bpf_verifier_env *env) +{ + struct bpf_prog *prog = env->prog; + struct bpf_prog *tgt_prog = prog->aux->dst_prog; + struct bpf_attach_target_info tgt_info = {}; + u32 btf_id = prog->aux->attach_btf_id; + struct bpf_trampoline *tr; + int ret; + u64 key; + + if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && + prog->type != BPF_PROG_TYPE_LSM) { + verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n"); + return -EINVAL; + } + + if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) + return check_struct_ops_btf_id(env); + + if (prog->type != BPF_PROG_TYPE_TRACING && + prog->type != BPF_PROG_TYPE_LSM && + prog->type != BPF_PROG_TYPE_EXT) + return 0; + + ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); + if (ret) return ret; + + if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { + /* to make freplace equivalent to their targets, they need to + * inherit env->ops and expected_attach_type for the rest of the + * verification + */ + env->ops = bpf_verifier_ops[tgt_prog->type]; + prog->expected_attach_type = tgt_prog->expected_attach_type; } + + /* store info about the attachment target that will be used later */ + prog->aux->attach_func_proto = tgt_info.tgt_type; + prog->aux->attach_func_name = tgt_info.tgt_name; + + if (tgt_prog) { + prog->aux->saved_dst_prog_type = tgt_prog->type; + prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; + } + + if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { + prog->aux->attach_btf_trace = true; + return 0; + } else if (prog->expected_attach_type == BPF_TRACE_ITER) { + if (!bpf_iter_prog_supported(prog)) + return -EINVAL; + return 0; + } + + if (prog->type == BPF_PROG_TYPE_LSM) { + ret = bpf_lsm_verify_prog(&env->log, prog); + if (ret < 0) + return ret; + } + + key = bpf_trampoline_compute_key(tgt_prog, btf_id); + tr = bpf_trampoline_get(key, &tgt_info); + if (!tr) + return -ENOMEM; + + prog->aux->dst_trampoline = tr; + return 0; +} + +struct btf *bpf_get_btf_vmlinux(void) +{ + if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { + mutex_lock(&bpf_verifier_lock); + if (!btf_vmlinux) + btf_vmlinux = btf_parse_vmlinux(); + mutex_unlock(&bpf_verifier_lock); + } + return btf_vmlinux; } int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, @@ -11520,12 +11590,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, env->ops = bpf_verifier_ops[env->prog->type]; is_priv = bpf_capable(); - if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { - mutex_lock(&bpf_verifier_lock); - if (!btf_vmlinux) - btf_vmlinux = btf_parse_vmlinux(); - mutex_unlock(&bpf_verifier_lock); - } + bpf_get_btf_vmlinux(); /* grab the mutex to protect few globals used by verifier */ if (!is_priv) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 05d3e1375e4c..d5d9f2d03e8a 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -28,6 +28,8 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @gp_func: This flavor's grace-period-wait function. * @gp_state: Grace period's most recent state transition (debugging). + * @gp_sleep: Per-grace-period sleep to prevent CPU-bound looping. + * @init_fract: Initial backoff sleep interval. * @gp_jiffies: Time of last @gp_state transition. * @gp_start: Most recent grace-period start in jiffies. * @n_gps: Number of grace periods completed since boot. @@ -48,6 +50,8 @@ struct rcu_tasks { struct wait_queue_head cbs_wq; raw_spinlock_t cbs_lock; int gp_state; + int gp_sleep; + int init_fract; unsigned long gp_jiffies; unsigned long gp_start; unsigned long n_gps; @@ -81,7 +85,7 @@ static struct rcu_tasks rt_name = \ DEFINE_STATIC_SRCU(tasks_rcu_exit_srcu); /* Avoid IPIing CPUs early in the grace period. */ -#define RCU_TASK_IPI_DELAY (HZ / 2) +#define RCU_TASK_IPI_DELAY (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) ? HZ / 2 : 0) static int rcu_task_ipi_delay __read_mostly = RCU_TASK_IPI_DELAY; module_param(rcu_task_ipi_delay, int, 0644); @@ -231,7 +235,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) cond_resched(); } /* Paranoid sleep to keep this from entering a tight loop */ - schedule_timeout_idle(HZ/10); + schedule_timeout_idle(rtp->gp_sleep); set_tasks_gp_state(rtp, RTGS_WAIT_CBS); } @@ -329,8 +333,10 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp) */ lastreport = jiffies; - /* Start off with HZ/10 wait and slowly back off to 1 HZ wait. */ - fract = 10; + // Start off with initial wait and slowly back off to 1 HZ wait. + fract = rtp->init_fract; + if (fract > HZ) + fract = HZ; for (;;) { bool firstreport; @@ -553,6 +559,8 @@ EXPORT_SYMBOL_GPL(rcu_barrier_tasks); static int __init rcu_spawn_tasks_kthread(void) { + rcu_tasks.gp_sleep = HZ / 10; + rcu_tasks.init_fract = 10; rcu_tasks.pregp_func = rcu_tasks_pregp_step; rcu_tasks.pertask_func = rcu_tasks_pertask; rcu_tasks.postscan_func = rcu_tasks_postscan; @@ -685,6 +693,7 @@ EXPORT_SYMBOL_GPL(rcu_barrier_tasks_rude); static int __init rcu_spawn_tasks_rude_kthread(void) { + rcu_tasks_rude.gp_sleep = HZ / 10; rcu_spawn_tasks_kthread_generic(&rcu_tasks_rude); return 0; } @@ -745,9 +754,9 @@ static DEFINE_PER_CPU(bool, trc_ipi_to_cpu); // The number of detections of task quiescent state relying on // heavyweight readers executing explicit memory barriers. -unsigned long n_heavy_reader_attempts; -unsigned long n_heavy_reader_updates; -unsigned long n_heavy_reader_ofl_updates; +static unsigned long n_heavy_reader_attempts; +static unsigned long n_heavy_reader_updates; +static unsigned long n_heavy_reader_ofl_updates; void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func); DEFINE_RCU_TASKS(rcu_tasks_trace, rcu_tasks_wait_gp, call_rcu_tasks_trace, @@ -821,6 +830,12 @@ static void trc_read_check_handler(void *t_in) WRITE_ONCE(t->trc_reader_checked, true); goto reset_ipi; } + // If we are racing with an rcu_read_unlock_trace(), try again later. + if (unlikely(t->trc_reader_nesting < 0)) { + if (WARN_ON_ONCE(atomic_dec_and_test(&trc_n_readers_need_end))) + wake_up(&trc_wait); + goto reset_ipi; + } WRITE_ONCE(t->trc_reader_checked, true); // Get here if the task is in a read-side critical section. Set @@ -911,7 +926,8 @@ static void trc_wait_for_one_reader(struct task_struct *t, // If currently running, send an IPI, either way, add to list. trc_add_holdout(t, bhp); - if (task_curr(t) && time_after(jiffies, rcu_tasks_trace.gp_start + rcu_task_ipi_delay)) { + if (task_curr(t) && + time_after(jiffies + 1, rcu_tasks_trace.gp_start + rcu_task_ipi_delay)) { // The task is currently running, so try IPIing it. cpu = task_cpu(t); @@ -1072,15 +1088,17 @@ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) if (ret) break; // Count reached zero. // Stall warning time, so make a list of the offenders. + rcu_read_lock(); for_each_process_thread(g, t) if (READ_ONCE(t->trc_reader_special.b.need_qs)) trc_add_holdout(t, &holdouts); + rcu_read_unlock(); firstreport = true; - list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) - if (READ_ONCE(t->trc_reader_special.b.need_qs)) { + list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) { + if (READ_ONCE(t->trc_reader_special.b.need_qs)) show_stalled_task_trace(t, &firstreport); - trc_del_holdout(t); - } + trc_del_holdout(t); // Release task_struct reference. + } if (firstreport) pr_err("INFO: rcu_tasks_trace detected stalls? (Counter/taskslist mismatch?)\n"); show_stalled_ipi_trace(); @@ -1163,6 +1181,17 @@ EXPORT_SYMBOL_GPL(rcu_barrier_tasks_trace); static int __init rcu_spawn_tasks_trace_kthread(void) { + if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) { + rcu_tasks_trace.gp_sleep = HZ / 10; + rcu_tasks_trace.init_fract = 10; + } else { + rcu_tasks_trace.gp_sleep = HZ / 200; + if (rcu_tasks_trace.gp_sleep <= 0) + rcu_tasks_trace.gp_sleep = 1; + rcu_tasks_trace.init_fract = HZ / 5; + if (rcu_tasks_trace.init_fract <= 0) + rcu_tasks_trace.init_fract = 1; + } rcu_tasks_trace.pregp_func = rcu_tasks_trace_pregp_step; rcu_tasks_trace.pertask_func = rcu_tasks_trace_pertask; rcu_tasks_trace.postscan_func = rcu_tasks_trace_postscan; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 36508f46a8db..e118a83439c3 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -7,6 +7,7 @@ #include <linux/slab.h> #include <linux/bpf.h> #include <linux/bpf_perf_event.h> +#include <linux/btf.h> #include <linux/filter.h> #include <linux/uaccess.h> #include <linux/ctype.h> @@ -16,6 +17,9 @@ #include <linux/error-injection.h> #include <linux/btf_ids.h> +#include <uapi/linux/bpf.h> +#include <uapi/linux/btf.h> + #include <asm/tlb.h> #include "trace_probe.h" @@ -67,6 +71,10 @@ static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); +static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size, + u64 flags, const struct btf **btf, + s32 *btf_id); + /** * trace_call_bpf - invoke BPF program * @call: tracepoint event @@ -772,6 +780,31 @@ static const struct bpf_func_proto bpf_seq_write_proto = { .arg3_type = ARG_CONST_SIZE_OR_ZERO, }; +BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr, + u32, btf_ptr_size, u64, flags) +{ + const struct btf *btf; + s32 btf_id; + int ret; + + ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id); + if (ret) + return ret; + + return btf_type_seq_show_flags(btf, btf_id, ptr->ptr, m, flags); +} + +static const struct bpf_func_proto bpf_seq_printf_btf_proto = { + .func = bpf_seq_printf_btf, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_BTF_ID, + .arg1_btf_id = &btf_seq_file_ids[0], + .arg2_type = ARG_PTR_TO_MEM, + .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_ANYTHING, +}; + static __always_inline int get_map_perf_counter(struct bpf_map *map, u64 flags, u64 *value, u64 *enabled, u64 *running) @@ -1147,6 +1180,65 @@ static const struct bpf_func_proto bpf_d_path_proto = { .allowed = bpf_d_path_allowed, }; +#define BTF_F_ALL (BTF_F_COMPACT | BTF_F_NONAME | \ + BTF_F_PTR_RAW | BTF_F_ZERO) + +static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size, + u64 flags, const struct btf **btf, + s32 *btf_id) +{ + const struct btf_type *t; + + if (unlikely(flags & ~(BTF_F_ALL))) + return -EINVAL; + + if (btf_ptr_size != sizeof(struct btf_ptr)) + return -EINVAL; + + *btf = bpf_get_btf_vmlinux(); + + if (IS_ERR_OR_NULL(*btf)) + return PTR_ERR(*btf); + + if (ptr->type_id > 0) + *btf_id = ptr->type_id; + else + return -EINVAL; + + if (*btf_id > 0) + t = btf_type_by_id(*btf, *btf_id); + if (*btf_id <= 0 || !t) + return -ENOENT; + + return 0; +} + +BPF_CALL_5(bpf_snprintf_btf, char *, str, u32, str_size, struct btf_ptr *, ptr, + u32, btf_ptr_size, u64, flags) +{ + const struct btf *btf; + s32 btf_id; + int ret; + + ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id); + if (ret) + return ret; + + return btf_type_snprintf_show(btf, btf_id, ptr->ptr, str, str_size, + flags); +} + +const struct bpf_func_proto bpf_snprintf_btf_proto = { + .func = bpf_snprintf_btf, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_CONST_SIZE, + .arg3_type = ARG_PTR_TO_MEM, + .arg4_type = ARG_CONST_SIZE, + .arg5_type = ARG_ANYTHING, +}; + const struct bpf_func_proto * bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { @@ -1233,6 +1325,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_get_task_stack_proto; case BPF_FUNC_copy_from_user: return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL; + case BPF_FUNC_snprintf_btf: + return &bpf_snprintf_btf_proto; default: return NULL; } @@ -1630,6 +1724,10 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return prog->expected_attach_type == BPF_TRACE_ITER ? &bpf_seq_write_proto : NULL; + case BPF_FUNC_seq_printf_btf: + return prog->expected_attach_type == BPF_TRACE_ITER ? + &bpf_seq_printf_btf_proto : + NULL; case BPF_FUNC_d_path: return &bpf_d_path_proto; default: @@ -1678,6 +1776,7 @@ const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { }; const struct bpf_prog_ops raw_tracepoint_prog_ops = { + .test_run = bpf_prog_test_run_raw_tp, }; const struct bpf_verifier_ops tracing_verifier_ops = { diff --git a/net/Kconfig b/net/Kconfig index 3831206977a1..d6567162c1cf 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -434,7 +434,6 @@ config NET_SOCK_MSG config NET_DEVLINK bool default n - imply NET_DROP_MONITOR config PAGE_POOL bool diff --git a/net/bluetooth/Kconfig b/net/bluetooth/Kconfig index e2497d764e97..64e669acd42f 100644 --- a/net/bluetooth/Kconfig +++ b/net/bluetooth/Kconfig @@ -64,7 +64,6 @@ source "net/bluetooth/hidp/Kconfig" config BT_HS bool "Bluetooth High Speed (HS) features" depends on BT_BREDR - default y help Bluetooth High Speed includes support for off-loading Bluetooth connections via 802.11 (wifi) physical layer diff --git a/net/bluetooth/a2mp.c b/net/bluetooth/a2mp.c index 26526be579c7..da7fd7c8c2dc 100644 --- a/net/bluetooth/a2mp.c +++ b/net/bluetooth/a2mp.c @@ -226,6 +226,9 @@ static int a2mp_discover_rsp(struct amp_mgr *mgr, struct sk_buff *skb, struct a2mp_info_req req; found = true; + + memset(&req, 0, sizeof(req)); + req.id = cl->id; a2mp_send(mgr, A2MP_GETINFO_REQ, __next_ident(mgr), sizeof(req), &req); @@ -305,6 +308,8 @@ static int a2mp_getinfo_req(struct amp_mgr *mgr, struct sk_buff *skb, if (!hdev || hdev->dev_type != HCI_AMP) { struct a2mp_info_rsp rsp; + memset(&rsp, 0, sizeof(rsp)); + rsp.id = req->id; rsp.status = A2MP_STATUS_INVALID_CTRL_ID; @@ -348,6 +353,8 @@ static int a2mp_getinfo_rsp(struct amp_mgr *mgr, struct sk_buff *skb, if (!ctrl) return -ENOMEM; + memset(&req, 0, sizeof(req)); + req.id = rsp->id; a2mp_send(mgr, A2MP_GETAMPASSOC_REQ, __next_ident(mgr), sizeof(req), &req); @@ -376,6 +383,8 @@ static int a2mp_getampassoc_req(struct amp_mgr *mgr, struct sk_buff *skb, struct a2mp_amp_assoc_rsp rsp; rsp.id = req->id; + memset(&rsp, 0, sizeof(rsp)); + if (tmp) { rsp.status = A2MP_STATUS_COLLISION_OCCURED; amp_mgr_put(tmp); @@ -464,7 +473,6 @@ static int a2mp_createphyslink_req(struct amp_mgr *mgr, struct sk_buff *skb, struct a2mp_cmd *hdr) { struct a2mp_physlink_req *req = (void *) skb->data; - struct a2mp_physlink_rsp rsp; struct hci_dev *hdev; struct hci_conn *hcon; @@ -475,6 +483,8 @@ static int a2mp_createphyslink_req(struct amp_mgr *mgr, struct sk_buff *skb, BT_DBG("local_id %d, remote_id %d", req->local_id, req->remote_id); + memset(&rsp, 0, sizeof(rsp)); + rsp.local_id = req->remote_id; rsp.remote_id = req->local_id; @@ -553,6 +563,8 @@ static int a2mp_discphyslink_req(struct amp_mgr *mgr, struct sk_buff *skb, BT_DBG("local_id %d remote_id %d", req->local_id, req->remote_id); + memset(&rsp, 0, sizeof(rsp)); + rsp.local_id = req->remote_id; rsp.remote_id = req->local_id; rsp.status = A2MP_STATUS_SUCCESS; @@ -675,6 +687,8 @@ static int a2mp_chan_recv_cb(struct l2cap_chan *chan, struct sk_buff *skb) if (err) { struct a2mp_cmd_rej rej; + memset(&rej, 0, sizeof(rej)); + rej.reason = cpu_to_le16(0); hdr = (void *) skb->data; @@ -898,6 +912,8 @@ void a2mp_send_getinfo_rsp(struct hci_dev *hdev) BT_DBG("%s mgr %p", hdev->name, mgr); + memset(&rsp, 0, sizeof(rsp)); + rsp.id = hdev->id; rsp.status = A2MP_STATUS_INVALID_CTRL_ID; @@ -995,6 +1011,8 @@ void a2mp_send_create_phy_link_rsp(struct hci_dev *hdev, u8 status) if (!mgr) return; + memset(&rsp, 0, sizeof(rsp)); + hs_hcon = hci_conn_hash_lookup_state(hdev, AMP_LINK, BT_CONNECT); if (!hs_hcon) { rsp.status = A2MP_STATUS_UNABLE_START_LINK_CREATION; @@ -1027,6 +1045,8 @@ void a2mp_discover_amp(struct l2cap_chan *chan) mgr->bredr_chan = chan; + memset(&req, 0, sizeof(req)); + req.mtu = cpu_to_le16(L2CAP_A2MP_DEFAULT_MTU); req.ext_feat = 0; a2mp_send(mgr, A2MP_DISCOVER_REQ, 1, sizeof(req), &req); diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index b0209e35284a..502552d6e9af 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -2963,7 +2963,7 @@ int hci_add_adv_instance(struct hci_dev *hdev, u8 instance, u32 flags, sizeof(adv_instance->scan_rsp_data)); } else { if (hdev->adv_instance_cnt >= hdev->le_num_of_adv_sets || - instance < 1 || instance > HCI_MAX_ADV_INSTANCES) + instance < 1 || instance > hdev->le_num_of_adv_sets) return -EOVERFLOW; adv_instance = kzalloc(sizeof(*adv_instance), GFP_KERNEL); @@ -3061,6 +3061,7 @@ static int free_adv_monitor(int id, void *ptr, void *data) idr_remove(&hdev->adv_monitors_idr, monitor->handle); hci_free_adv_monitor(monitor); + hdev->adv_monitors_cnt--; return 0; } @@ -3077,6 +3078,7 @@ int hci_remove_adv_monitor(struct hci_dev *hdev, u16 handle) idr_remove(&hdev->adv_monitors_idr, monitor->handle); hci_free_adv_monitor(monitor); + hdev->adv_monitors_cnt--; } else { /* Remove all monitors if handle is 0. */ idr_for_each(&hdev->adv_monitors_idr, &free_adv_monitor, hdev); @@ -3442,6 +3444,16 @@ void hci_copy_identity_address(struct hci_dev *hdev, bdaddr_t *bdaddr, } } +static void hci_suspend_clear_tasks(struct hci_dev *hdev) +{ + int i; + + for (i = 0; i < __SUSPEND_NUM_TASKS; i++) + clear_bit(i, hdev->suspend_tasks); + + wake_up(&hdev->suspend_wait_q); +} + static int hci_suspend_wait_event(struct hci_dev *hdev) { #define WAKE_COND \ @@ -3487,12 +3499,24 @@ static int hci_change_suspend_state(struct hci_dev *hdev, return hci_suspend_wait_event(hdev); } +static void hci_clear_wake_reason(struct hci_dev *hdev) +{ + hci_dev_lock(hdev); + + hdev->wake_reason = 0; + bacpy(&hdev->wake_addr, BDADDR_ANY); + hdev->wake_addr_type = 0; + + hci_dev_unlock(hdev); +} + static int hci_suspend_notifier(struct notifier_block *nb, unsigned long action, void *data) { struct hci_dev *hdev = container_of(nb, struct hci_dev, suspend_notifier); int ret = 0; + u8 state = BT_RUNNING; /* If powering down, wait for completion. */ if (mgmt_powering_down(hdev)) { @@ -3513,15 +3537,27 @@ static int hci_suspend_notifier(struct notifier_block *nb, unsigned long action, * - Second, program event filter/whitelist and enable scan */ ret = hci_change_suspend_state(hdev, BT_SUSPEND_DISCONNECT); + if (!ret) + state = BT_SUSPEND_DISCONNECT; /* Only configure whitelist if disconnect succeeded and wake * isn't being prevented. */ - if (!ret && !(hdev->prevent_wake && hdev->prevent_wake(hdev))) + if (!ret && !(hdev->prevent_wake && hdev->prevent_wake(hdev))) { ret = hci_change_suspend_state(hdev, BT_SUSPEND_CONFIGURE_WAKE); + if (!ret) + state = BT_SUSPEND_CONFIGURE_WAKE; + } + + hci_clear_wake_reason(hdev); + mgmt_suspending(hdev, state); + } else if (action == PM_POST_SUSPEND) { ret = hci_change_suspend_state(hdev, BT_RUNNING); + + mgmt_resuming(hdev, hdev->wake_reason, &hdev->wake_addr, + hdev->wake_addr_type); } done: @@ -3784,6 +3820,7 @@ void hci_unregister_dev(struct hci_dev *hdev) cancel_work_sync(&hdev->power_on); + hci_suspend_clear_tasks(hdev); unregister_pm_notifier(&hdev->suspend_notifier); cancel_work_sync(&hdev->suspend_prepare); diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 4b7fc430793c..f04963914366 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -2569,7 +2569,6 @@ static void hci_inquiry_result_evt(struct hci_dev *hdev, struct sk_buff *skb) static void hci_conn_complete_evt(struct hci_dev *hdev, struct sk_buff *skb) { struct hci_ev_conn_complete *ev = (void *) skb->data; - struct inquiry_entry *ie; struct hci_conn *conn; BT_DBG("%s", hdev->name); @@ -2578,13 +2577,19 @@ static void hci_conn_complete_evt(struct hci_dev *hdev, struct sk_buff *skb) conn = hci_conn_hash_lookup_ba(hdev, ev->link_type, &ev->bdaddr); if (!conn) { - /* Connection may not exist if auto-connected. Check the inquiry - * cache to see if we've already discovered this bdaddr before. - * If found and link is an ACL type, create a connection class + /* Connection may not exist if auto-connected. Check the bredr + * allowlist to see if this device is allowed to auto connect. + * If link is an ACL type, create a connection class * automatically. + * + * Auto-connect will only occur if the event filter is + * programmed with a given address. Right now, event filter is + * only used during suspend. */ - ie = hci_inquiry_cache_lookup(hdev, &ev->bdaddr); - if (ie && ev->link_type == ACL_LINK) { + if (ev->link_type == ACL_LINK && + hci_bdaddr_list_lookup_with_flags(&hdev->whitelist, + &ev->bdaddr, + BDADDR_BREDR)) { conn = hci_conn_add(hdev, ev->link_type, &ev->bdaddr, HCI_ROLE_SLAVE); if (!conn) { @@ -6012,6 +6017,75 @@ static bool hci_get_cmd_complete(struct hci_dev *hdev, u16 opcode, return true; } +static void hci_store_wake_reason(struct hci_dev *hdev, u8 event, + struct sk_buff *skb) +{ + struct hci_ev_le_advertising_info *adv; + struct hci_ev_le_direct_adv_info *direct_adv; + struct hci_ev_le_ext_adv_report *ext_adv; + const struct hci_ev_conn_complete *conn_complete = (void *)skb->data; + const struct hci_ev_conn_request *conn_request = (void *)skb->data; + + hci_dev_lock(hdev); + + /* If we are currently suspended and this is the first BT event seen, + * save the wake reason associated with the event. + */ + if (!hdev->suspended || hdev->wake_reason) + goto unlock; + + /* Default to remote wake. Values for wake_reason are documented in the + * Bluez mgmt api docs. + */ + hdev->wake_reason = MGMT_WAKE_REASON_REMOTE_WAKE; + + /* Once configured for remote wakeup, we should only wake up for + * reconnections. It's useful to see which device is waking us up so + * keep track of the bdaddr of the connection event that woke us up. + */ + if (event == HCI_EV_CONN_REQUEST) { + bacpy(&hdev->wake_addr, &conn_complete->bdaddr); + hdev->wake_addr_type = BDADDR_BREDR; + } else if (event == HCI_EV_CONN_COMPLETE) { + bacpy(&hdev->wake_addr, &conn_request->bdaddr); + hdev->wake_addr_type = BDADDR_BREDR; + } else if (event == HCI_EV_LE_META) { + struct hci_ev_le_meta *le_ev = (void *)skb->data; + u8 subevent = le_ev->subevent; + u8 *ptr = &skb->data[sizeof(*le_ev)]; + u8 num_reports = *ptr; + + if ((subevent == HCI_EV_LE_ADVERTISING_REPORT || + subevent == HCI_EV_LE_DIRECT_ADV_REPORT || + subevent == HCI_EV_LE_EXT_ADV_REPORT) && + num_reports) { + adv = (void *)(ptr + 1); + direct_adv = (void *)(ptr + 1); + ext_adv = (void *)(ptr + 1); + + switch (subevent) { + case HCI_EV_LE_ADVERTISING_REPORT: + bacpy(&hdev->wake_addr, &adv->bdaddr); + hdev->wake_addr_type = adv->bdaddr_type; + break; + case HCI_EV_LE_DIRECT_ADV_REPORT: + bacpy(&hdev->wake_addr, &direct_adv->bdaddr); + hdev->wake_addr_type = direct_adv->bdaddr_type; + break; + case HCI_EV_LE_EXT_ADV_REPORT: + bacpy(&hdev->wake_addr, &ext_adv->bdaddr); + hdev->wake_addr_type = ext_adv->bdaddr_type; + break; + } + } + } else { + hdev->wake_reason = MGMT_WAKE_REASON_UNEXPECTED; + } + +unlock: + hci_dev_unlock(hdev); +} + void hci_event_packet(struct hci_dev *hdev, struct sk_buff *skb) { struct hci_event_hdr *hdr = (void *) skb->data; @@ -6045,6 +6119,9 @@ void hci_event_packet(struct hci_dev *hdev, struct sk_buff *skb) skb_pull(skb, HCI_EVENT_HDR_SIZE); + /* Store wake reason if we're suspended */ + hci_store_wake_reason(hdev, event, skb); + switch (event) { case HCI_EV_INQUIRY_COMPLETE: hci_inquiry_complete_evt(hdev, skb); diff --git a/net/bluetooth/hci_request.c b/net/bluetooth/hci_request.c index e0269192f2e5..6f12bab4d2fa 100644 --- a/net/bluetooth/hci_request.c +++ b/net/bluetooth/hci_request.c @@ -1027,6 +1027,9 @@ void hci_req_add_le_passive_scan(struct hci_request *req) } else if (hci_is_le_conn_scanning(hdev)) { window = hdev->le_scan_window_connect; interval = hdev->le_scan_int_connect; + } else if (hci_is_adv_monitoring(hdev)) { + window = hdev->le_scan_window_adv_monitor; + interval = hdev->le_scan_int_adv_monitor; } else { window = hdev->le_scan_window; interval = hdev->le_scan_interval; @@ -1111,6 +1114,53 @@ static void hci_req_config_le_suspend_scan(struct hci_request *req) set_bit(SUSPEND_SCAN_ENABLE, req->hdev->suspend_tasks); } +static void cancel_adv_timeout(struct hci_dev *hdev) +{ + if (hdev->adv_instance_timeout) { + hdev->adv_instance_timeout = 0; + cancel_delayed_work(&hdev->adv_instance_expire); + } +} + +/* This function requires the caller holds hdev->lock */ +static void hci_suspend_adv_instances(struct hci_request *req) +{ + bt_dev_dbg(req->hdev, "Suspending advertising instances"); + + /* Call to disable any advertisements active on the controller. + * This will succeed even if no advertisements are configured. + */ + __hci_req_disable_advertising(req); + + /* If we are using software rotation, pause the loop */ + if (!ext_adv_capable(req->hdev)) + cancel_adv_timeout(req->hdev); +} + +/* This function requires the caller holds hdev->lock */ +static void hci_resume_adv_instances(struct hci_request *req) +{ + struct adv_info *adv; + + bt_dev_dbg(req->hdev, "Resuming advertising instances"); + + if (ext_adv_capable(req->hdev)) { + /* Call for each tracked instance to be re-enabled */ + list_for_each_entry(adv, &req->hdev->adv_instances, list) { + __hci_req_enable_ext_advertising(req, + adv->instance); + } + + } else { + /* Schedule for most recent instance to be restarted and begin + * the software rotation loop + */ + __hci_req_schedule_adv_instance(req, + req->hdev->cur_adv_instance, + true); + } +} + static void suspend_req_complete(struct hci_dev *hdev, u8 status, u16 opcode) { bt_dev_dbg(hdev, "Request complete opcode=0x%x, status=0x%x", opcode, @@ -1153,7 +1203,7 @@ void hci_req_prepare_suspend(struct hci_dev *hdev, enum suspended_state next) hdev->discovery_paused = true; hdev->discovery_old_state = old_state; - /* Stop advertising */ + /* Stop directed advertising */ old_state = hci_dev_test_flag(hdev, HCI_ADVERTISING); if (old_state) { set_bit(SUSPEND_PAUSE_ADVERTISING, hdev->suspend_tasks); @@ -1162,6 +1212,10 @@ void hci_req_prepare_suspend(struct hci_dev *hdev, enum suspended_state next) &hdev->discov_off, 0); } + /* Pause other advertisements */ + if (hdev->adv_instance_cnt) + hci_suspend_adv_instances(&req); + hdev->advertising_paused = true; hdev->advertising_old_state = old_state; /* Disable page scan */ @@ -1212,7 +1266,7 @@ void hci_req_prepare_suspend(struct hci_dev *hdev, enum suspended_state next) /* Reset passive/background scanning to normal */ hci_req_config_le_suspend_scan(&req); - /* Unpause advertising */ + /* Unpause directed advertising */ hdev->advertising_paused = false; if (hdev->advertising_old_state) { set_bit(SUSPEND_UNPAUSE_ADVERTISING, @@ -1223,6 +1277,10 @@ void hci_req_prepare_suspend(struct hci_dev *hdev, enum suspended_state next) hdev->advertising_old_state = 0; } + /* Resume other advertisements */ + if (hdev->adv_instance_cnt) + hci_resume_adv_instances(&req); + /* Unpause discovery */ hdev->discovery_paused = false; if (hdev->discovery_old_state != DISCOVERY_STOPPED && @@ -1533,11 +1591,14 @@ void __hci_req_update_scan_rsp_data(struct hci_request *req, u8 instance) memset(&cp, 0, sizeof(cp)); - if (instance) + /* Extended scan response data doesn't allow a response to be + * set if the instance isn't scannable. + */ + if (get_adv_instance_scan_rsp_len(hdev, instance)) len = create_instance_scan_rsp_data(hdev, instance, cp.data); else - len = create_default_scan_rsp_data(hdev, cp.data); + len = 0; if (hdev->scan_rsp_data_len == len && !memcmp(cp.data, hdev->scan_rsp_data, len)) @@ -1824,7 +1885,13 @@ int hci_get_random_address(struct hci_dev *hdev, bool require_privacy, if (use_rpa) { int to; - *own_addr_type = ADDR_LE_DEV_RANDOM; + /* If Controller supports LL Privacy use own address type is + * 0x03 + */ + if (use_ll_privacy(hdev)) + *own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED; + else + *own_addr_type = ADDR_LE_DEV_RANDOM; if (adv_instance) { if (!adv_instance->rpa_expired && @@ -2183,14 +2250,6 @@ int __hci_req_schedule_adv_instance(struct hci_request *req, u8 instance, return 0; } -static void cancel_adv_timeout(struct hci_dev *hdev) -{ - if (hdev->adv_instance_timeout) { - hdev->adv_instance_timeout = 0; - cancel_delayed_work(&hdev->adv_instance_expire); - } -} - /* For a single instance: * - force == true: The instance will be removed even when its remaining * lifetime is not zero. diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index ade83e224567..1ab27b90ddcb 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -7301,9 +7301,10 @@ static int l2cap_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) goto drop; } - if ((chan->mode == L2CAP_MODE_ERTM || - chan->mode == L2CAP_MODE_STREAMING) && sk_filter(chan->data, skb)) - goto drop; + if (chan->ops->filter) { + if (chan->ops->filter(chan, skb)) + goto drop; + } if (!control->sframe) { int err; diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index e1a3e66b1754..f1b1edd0b697 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -1521,8 +1521,6 @@ static void l2cap_sock_teardown_cb(struct l2cap_chan *chan, int err) parent = bt_sk(sk)->parent; - sock_set_flag(sk, SOCK_ZAPPED); - switch (chan->state) { case BT_OPEN: case BT_BOUND: @@ -1549,8 +1547,11 @@ static void l2cap_sock_teardown_cb(struct l2cap_chan *chan, int err) break; } - release_sock(sk); + + /* Only zap after cleanup to avoid use after free race */ + sock_set_flag(sk, SOCK_ZAPPED); + } static void l2cap_sock_state_change_cb(struct l2cap_chan *chan, int state, @@ -1663,6 +1664,19 @@ static void l2cap_sock_suspend_cb(struct l2cap_chan *chan) sk->sk_state_change(sk); } +static int l2cap_sock_filter(struct l2cap_chan *chan, struct sk_buff *skb) +{ + struct sock *sk = chan->data; + + switch (chan->mode) { + case L2CAP_MODE_ERTM: + case L2CAP_MODE_STREAMING: + return sk_filter(sk, skb); + } + + return 0; +} + static const struct l2cap_ops l2cap_chan_ops = { .name = "L2CAP Socket Interface", .new_connection = l2cap_sock_new_connection_cb, @@ -1678,6 +1692,7 @@ static const struct l2cap_ops l2cap_chan_ops = { .get_sndtimeo = l2cap_sock_get_sndtimeo_cb, .get_peer_pid = l2cap_sock_get_peer_pid_cb, .alloc_skb = l2cap_sock_alloc_skb_cb, + .filter = l2cap_sock_filter, }; static void l2cap_sock_destruct(struct sock *sk) diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 5bbe71002fb9..12d7b368b428 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -163,6 +163,8 @@ static const u16 mgmt_events[] = { MGMT_EV_PHY_CONFIGURATION_CHANGED, MGMT_EV_EXP_FEATURE_CHANGED, MGMT_EV_DEVICE_FLAGS_CHANGED, + MGMT_EV_CONTROLLER_SUSPEND, + MGMT_EV_CONTROLLER_RESUME, }; static const u16 mgmt_untrusted_commands[] = { @@ -782,7 +784,8 @@ static u32 get_supported_settings(struct hci_dev *hdev) if (lmp_ssp_capable(hdev)) { settings |= MGMT_SETTING_SSP; - settings |= MGMT_SETTING_HS; + if (IS_ENABLED(CONFIG_BT_HS)) + settings |= MGMT_SETTING_HS; } if (lmp_sc_capable(hdev)) @@ -1815,6 +1818,10 @@ static int set_hs(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) bt_dev_dbg(hdev, "sock %p", sk); + if (!IS_ENABLED(CONFIG_BT_HS)) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_HS, + MGMT_STATUS_NOT_SUPPORTED); + status = mgmt_bredr_support(hdev); if (status) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_HS, status); @@ -4157,7 +4164,7 @@ static int read_adv_mon_features(struct sock *sk, struct hci_dev *hdev, { struct adv_monitor *monitor = NULL; struct mgmt_rp_read_adv_monitor_features *rp = NULL; - int handle; + int handle, err; size_t rp_size = 0; __u32 supported = 0; __u16 num_handles = 0; @@ -4192,9 +4199,13 @@ static int read_adv_mon_features(struct sock *sk, struct hci_dev *hdev, if (num_handles) memcpy(&rp->handles, &handles, (num_handles * sizeof(u16))); - return mgmt_cmd_complete(sk, hdev->id, - MGMT_OP_READ_ADV_MONITOR_FEATURES, - MGMT_STATUS_SUCCESS, rp, rp_size); + err = mgmt_cmd_complete(sk, hdev->id, + MGMT_OP_READ_ADV_MONITOR_FEATURES, + MGMT_STATUS_SUCCESS, rp, rp_size); + + kfree(rp); + + return err; } static int add_adv_patterns_monitor(struct sock *sk, struct hci_dev *hdev, @@ -7202,6 +7213,8 @@ static u32 get_supported_adv_flags(struct hci_dev *hdev) if (ext_adv_capable(hdev)) { flags |= MGMT_ADV_FLAG_SEC_1M; + flags |= MGMT_ADV_FLAG_HW_OFFLOAD; + flags |= MGMT_ADV_FLAG_CAN_SET_TX_POWER; if (hdev->le_features[1] & HCI_LE_PHY_2M) flags |= MGMT_ADV_FLAG_SEC_2M; @@ -7250,7 +7263,7 @@ static int read_adv_features(struct sock *sk, struct hci_dev *hdev, rp->supported_flags = cpu_to_le32(supported_flags); rp->max_adv_data_len = HCI_MAX_AD_LENGTH; rp->max_scan_rsp_len = HCI_MAX_AD_LENGTH; - rp->max_instances = HCI_MAX_ADV_INSTANCES; + rp->max_instances = hdev->le_num_of_adv_sets; rp->num_instances = hdev->adv_instance_cnt; instance = rp->instance; @@ -7446,7 +7459,7 @@ static int add_advertising(struct sock *sk, struct hci_dev *hdev, return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_ADVERTISING, MGMT_STATUS_NOT_SUPPORTED); - if (cp->instance < 1 || cp->instance > HCI_MAX_ADV_INSTANCES) + if (cp->instance < 1 || cp->instance > hdev->le_num_of_adv_sets) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_ADVERTISING, MGMT_STATUS_INVALID_PARAMS); @@ -7699,7 +7712,7 @@ static int get_adv_size_info(struct sock *sk, struct hci_dev *hdev, return mgmt_cmd_status(sk, hdev->id, MGMT_OP_GET_ADV_SIZE_INFO, MGMT_STATUS_REJECTED); - if (cp->instance < 1 || cp->instance > HCI_MAX_ADV_INSTANCES) + if (cp->instance < 1 || cp->instance > hdev->le_num_of_adv_sets) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_GET_ADV_SIZE_INFO, MGMT_STATUS_INVALID_PARAMS); @@ -8262,6 +8275,10 @@ void mgmt_device_disconnected(struct hci_dev *hdev, bdaddr_t *bdaddr, ev.addr.type = link_to_bdaddr(link_type, addr_type); ev.reason = reason; + /* Report disconnects due to suspend */ + if (hdev->suspended) + ev.reason = MGMT_DEV_DISCONN_LOCAL_HOST_SUSPEND; + mgmt_event(MGMT_EV_DEVICE_DISCONNECTED, hdev, &ev, sizeof(ev), sk); if (sk) @@ -8868,6 +8885,30 @@ void mgmt_discovering(struct hci_dev *hdev, u8 discovering) mgmt_event(MGMT_EV_DISCOVERING, hdev, &ev, sizeof(ev), NULL); } +void mgmt_suspending(struct hci_dev *hdev, u8 state) +{ + struct mgmt_ev_controller_suspend ev; + + ev.suspend_state = state; + mgmt_event(MGMT_EV_CONTROLLER_SUSPEND, hdev, &ev, sizeof(ev), NULL); +} + +void mgmt_resuming(struct hci_dev *hdev, u8 reason, bdaddr_t *bdaddr, + u8 addr_type) +{ + struct mgmt_ev_controller_resume ev; + + ev.wake_reason = reason; + if (bdaddr) { + bacpy(&ev.addr.bdaddr, bdaddr); + ev.addr.type = addr_type; + } else { + memset(&ev.addr, 0, sizeof(ev.addr)); + } + + mgmt_event(MGMT_EV_CONTROLLER_RESUME, hdev, &ev, sizeof(ev), NULL); +} + static struct hci_mgmt_chan chan = { .channel = HCI_CHANNEL_CONTROL, .handler_count = ARRAY_SIZE(mgmt_handlers), diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index dcf7f96ff417..79ffcdef0b7a 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -1001,6 +1001,12 @@ static int sco_sock_getsockopt(struct socket *sock, int level, int optname, err = -EFAULT; break; + case BT_SNDMTU: + case BT_RCVMTU: + if (put_user(sco_pi(sk)->conn->mtu, (u32 __user *)optval)) + err = -EFAULT; + break; + default: err = -ENOPROTOOPT; break; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index a66f211726e7..c1c30a9f76f3 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -11,6 +11,7 @@ #include <net/sock.h> #include <net/tcp.h> #include <linux/error-injection.h> +#include <linux/smp.h> #define CREATE_TRACE_POINTS #include <trace/events/bpf_test_run.h> @@ -204,6 +205,9 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog, int b = 2, err = -EFAULT; u32 retval = 0; + if (kattr->test.flags || kattr->test.cpu) + return -EINVAL; + switch (prog->expected_attach_type) { case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: @@ -236,6 +240,84 @@ out: return err; } +struct bpf_raw_tp_test_run_info { + struct bpf_prog *prog; + void *ctx; + u32 retval; +}; + +static void +__bpf_prog_test_run_raw_tp(void *data) +{ + struct bpf_raw_tp_test_run_info *info = data; + + rcu_read_lock(); + info->retval = BPF_PROG_RUN(info->prog, info->ctx); + rcu_read_unlock(); +} + +int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, + const union bpf_attr *kattr, + union bpf_attr __user *uattr) +{ + void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); + __u32 ctx_size_in = kattr->test.ctx_size_in; + struct bpf_raw_tp_test_run_info info; + int cpu = kattr->test.cpu, err = 0; + int current_cpu; + + /* doesn't support data_in/out, ctx_out, duration, or repeat */ + if (kattr->test.data_in || kattr->test.data_out || + kattr->test.ctx_out || kattr->test.duration || + kattr->test.repeat) + return -EINVAL; + + if (ctx_size_in < prog->aux->max_ctx_offset) + return -EINVAL; + + if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) + return -EINVAL; + + if (ctx_size_in) { + info.ctx = kzalloc(ctx_size_in, GFP_USER); + if (!info.ctx) + return -ENOMEM; + if (copy_from_user(info.ctx, ctx_in, ctx_size_in)) { + err = -EFAULT; + goto out; + } + } else { + info.ctx = NULL; + } + + info.prog = prog; + + current_cpu = get_cpu(); + if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || + cpu == current_cpu) { + __bpf_prog_test_run_raw_tp(&info); + } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) { + /* smp_call_function_single() also checks cpu_online() + * after csd_lock(). However, since cpu is from user + * space, let's do an extra quick check to filter out + * invalid value before smp_call_function_single(). + */ + err = -ENXIO; + } else { + err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp, + &info, 1); + } + put_cpu(); + + if (!err && + copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) + err = -EFAULT; + +out: + kfree(info.ctx); + return err; +} + static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) { void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); @@ -410,6 +492,9 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, void *data; int ret; + if (kattr->test.flags || kattr->test.cpu) + return -EINVAL; + data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); if (IS_ERR(data)) @@ -607,6 +692,9 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) return -EINVAL; + if (kattr->test.flags || kattr->test.cpu) + return -EINVAL; + if (size < ETH_HLEN) return -EINVAL; diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index 838efc682cff..c907f0dc7f87 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -269,7 +269,7 @@ BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, { struct bpf_local_storage_data *sdata; - if (flags > BPF_SK_STORAGE_GET_F_CREATE) + if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE) return (unsigned long)NULL; sdata = sk_storage_lookup(sk, map, true); @@ -299,6 +299,9 @@ BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk) { + if (!sk || !sk_fullsock(sk)) + return -EINVAL; + if (refcount_inc_not_zero(&sk->sk_refcnt)) { int err; @@ -355,7 +358,7 @@ const struct bpf_func_proto bpf_sk_storage_get_proto = { .gpl_only = false, .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, .arg1_type = ARG_CONST_MAP_PTR, - .arg2_type = ARG_PTR_TO_SOCKET, + .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, .arg4_type = ARG_ANYTHING, }; @@ -375,27 +378,7 @@ const struct bpf_func_proto bpf_sk_storage_delete_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, - .arg2_type = ARG_PTR_TO_SOCKET, -}; - -const struct bpf_func_proto sk_storage_get_btf_proto = { - .func = bpf_sk_storage_get, - .gpl_only = false, - .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, - .arg1_type = ARG_CONST_MAP_PTR, - .arg2_type = ARG_PTR_TO_BTF_ID, - .arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], - .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, - .arg4_type = ARG_ANYTHING, -}; - -const struct bpf_func_proto sk_storage_delete_btf_proto = { - .func = bpf_sk_storage_delete, - .gpl_only = false, - .ret_type = RET_INTEGER, - .arg1_type = ARG_CONST_MAP_PTR, - .arg2_type = ARG_PTR_TO_BTF_ID, - .arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], + .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, }; struct bpf_sk_storage_diag { diff --git a/net/core/dev.c b/net/core/dev.c index 873b50ac9668..9d55bf5d1a65 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4841,6 +4841,21 @@ int netif_rx_ni(struct sk_buff *skb) } EXPORT_SYMBOL(netif_rx_ni); +int netif_rx_any_context(struct sk_buff *skb) +{ + /* + * If invoked from contexts which do not invoke bottom half + * processing either at return from interrupt or when softrqs are + * reenabled, use netif_rx_ni() which invokes bottomhalf processing + * directly. + */ + if (in_interrupt()) + return netif_rx(skb); + else + return netif_rx_ni(skb); +} +EXPORT_SYMBOL(netif_rx_any_context); + static __latent_entropy void net_tx_action(struct softirq_action *h) { struct softnet_data *sd = this_cpu_ptr(&softnet_data); diff --git a/net/core/devlink.c b/net/core/devlink.c index 7a38f9e25922..6f2863e717a9 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -27,7 +27,6 @@ #include <net/net_namespace.h> #include <net/sock.h> #include <net/devlink.h> -#include <net/drop_monitor.h> #define CREATE_TRACE_POINTS #include <trace/events/devlink.h> @@ -84,6 +83,7 @@ EXPORT_SYMBOL(devlink_dpipe_header_ipv6); EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg); EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr); +EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report); static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = { [DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY }, @@ -9261,20 +9261,19 @@ devlink_trap_stats_update(struct devlink_stats __percpu *trap_stats, } static void -devlink_trap_report_metadata_fill(struct net_dm_hw_metadata *hw_metadata, - const struct devlink_trap_item *trap_item, - struct devlink_port *in_devlink_port, - const struct flow_action_cookie *fa_cookie) +devlink_trap_report_metadata_set(struct devlink_trap_metadata *metadata, + const struct devlink_trap_item *trap_item, + struct devlink_port *in_devlink_port, + const struct flow_action_cookie *fa_cookie) { - struct devlink_trap_group_item *group_item = trap_item->group_item; - - hw_metadata->trap_group_name = group_item->group->name; - hw_metadata->trap_name = trap_item->trap->name; - hw_metadata->fa_cookie = fa_cookie; + metadata->trap_name = trap_item->trap->name; + metadata->trap_group_name = trap_item->group_item->group->name; + metadata->fa_cookie = fa_cookie; + metadata->trap_type = trap_item->trap->type; spin_lock(&in_devlink_port->type_lock); if (in_devlink_port->type == DEVLINK_PORT_TYPE_ETH) - hw_metadata->input_dev = in_devlink_port->type_dev; + metadata->input_dev = in_devlink_port->type_dev; spin_unlock(&in_devlink_port->type_lock); } @@ -9292,21 +9291,17 @@ void devlink_trap_report(struct devlink *devlink, struct sk_buff *skb, { struct devlink_trap_item *trap_item = trap_ctx; - struct net_dm_hw_metadata hw_metadata = {}; devlink_trap_stats_update(trap_item->stats, skb->len); devlink_trap_stats_update(trap_item->group_item->stats, skb->len); - /* Control packets were not dropped by the device or encountered an - * exception during forwarding and therefore should not be reported to - * the kernel's drop monitor. - */ - if (trap_item->trap->type == DEVLINK_TRAP_TYPE_CONTROL) - return; + if (trace_devlink_trap_report_enabled()) { + struct devlink_trap_metadata metadata = {}; - devlink_trap_report_metadata_fill(&hw_metadata, trap_item, - in_devlink_port, fa_cookie); - net_dm_hw_report(skb, &hw_metadata); + devlink_trap_report_metadata_set(&metadata, trap_item, + in_devlink_port, fa_cookie); + trace_devlink_trap_report(devlink, skb, &metadata); + } } EXPORT_SYMBOL_GPL(devlink_trap_report); diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c index 9704522b0872..a28b743489c5 100644 --- a/net/core/drop_monitor.c +++ b/net/core/drop_monitor.c @@ -26,13 +26,14 @@ #include <linux/bitops.h> #include <linux/slab.h> #include <linux/module.h> -#include <net/drop_monitor.h> #include <net/genetlink.h> #include <net/netevent.h> #include <net/flow_offload.h> +#include <net/devlink.h> #include <trace/events/skb.h> #include <trace/events/napi.h> +#include <trace/events/devlink.h> #include <asm/unaligned.h> @@ -114,13 +115,14 @@ struct net_dm_alert_ops { int work, int budget); void (*work_item_func)(struct work_struct *work); void (*hw_work_item_func)(struct work_struct *work); - void (*hw_probe)(struct sk_buff *skb, - const struct net_dm_hw_metadata *hw_metadata); + void (*hw_trap_probe)(void *ignore, const struct devlink *devlink, + struct sk_buff *skb, + const struct devlink_trap_metadata *metadata); }; struct net_dm_skb_cb { union { - struct net_dm_hw_metadata *hw_metadata; + struct devlink_trap_metadata *hw_metadata; void *pc; }; }; @@ -432,8 +434,9 @@ out: } static void -net_dm_hw_summary_probe(struct sk_buff *skb, - const struct net_dm_hw_metadata *hw_metadata) +net_dm_hw_trap_summary_probe(void *ignore, const struct devlink *devlink, + struct sk_buff *skb, + const struct devlink_trap_metadata *metadata) { struct net_dm_hw_entries *hw_entries; struct net_dm_hw_entry *hw_entry; @@ -441,6 +444,9 @@ net_dm_hw_summary_probe(struct sk_buff *skb, unsigned long flags; int i; + if (metadata->trap_type == DEVLINK_TRAP_TYPE_CONTROL) + return; + hw_data = this_cpu_ptr(&dm_hw_cpu_data); spin_lock_irqsave(&hw_data->lock, flags); hw_entries = hw_data->hw_entries; @@ -450,7 +456,7 @@ net_dm_hw_summary_probe(struct sk_buff *skb, for (i = 0; i < hw_entries->num_entries; i++) { hw_entry = &hw_entries->entries[i]; - if (!strncmp(hw_entry->trap_name, hw_metadata->trap_name, + if (!strncmp(hw_entry->trap_name, metadata->trap_name, NET_DM_MAX_HW_TRAP_NAME_LEN - 1)) { hw_entry->count++; goto out; @@ -460,7 +466,7 @@ net_dm_hw_summary_probe(struct sk_buff *skb, goto out; hw_entry = &hw_entries->entries[hw_entries->num_entries]; - strlcpy(hw_entry->trap_name, hw_metadata->trap_name, + strlcpy(hw_entry->trap_name, metadata->trap_name, NET_DM_MAX_HW_TRAP_NAME_LEN - 1); hw_entry->count = 1; hw_entries->num_entries++; @@ -479,7 +485,7 @@ static const struct net_dm_alert_ops net_dm_alert_summary_ops = { .napi_poll_probe = trace_napi_poll_hit, .work_item_func = send_dm_alert, .hw_work_item_func = net_dm_hw_summary_work, - .hw_probe = net_dm_hw_summary_probe, + .hw_trap_probe = net_dm_hw_trap_summary_probe, }; static void net_dm_packet_trace_kfree_skb_hit(void *ignore, @@ -705,7 +711,7 @@ static void net_dm_packet_work(struct work_struct *work) } static size_t -net_dm_flow_action_cookie_size(const struct net_dm_hw_metadata *hw_metadata) +net_dm_flow_action_cookie_size(const struct devlink_trap_metadata *hw_metadata) { return hw_metadata->fa_cookie ? nla_total_size(hw_metadata->fa_cookie->cookie_len) : 0; @@ -713,7 +719,7 @@ net_dm_flow_action_cookie_size(const struct net_dm_hw_metadata *hw_metadata) static size_t net_dm_hw_packet_report_size(size_t payload_len, - const struct net_dm_hw_metadata *hw_metadata) + const struct devlink_trap_metadata *hw_metadata) { size_t size; @@ -743,7 +749,7 @@ net_dm_hw_packet_report_size(size_t payload_len, static int net_dm_hw_packet_report_fill(struct sk_buff *msg, struct sk_buff *skb, size_t payload_len) { - struct net_dm_hw_metadata *hw_metadata; + struct devlink_trap_metadata *hw_metadata; struct nlattr *attr; void *hdr; @@ -810,56 +816,56 @@ nla_put_failure: return -EMSGSIZE; } -static struct net_dm_hw_metadata * -net_dm_hw_metadata_clone(const struct net_dm_hw_metadata *hw_metadata) +static struct devlink_trap_metadata * +net_dm_hw_metadata_copy(const struct devlink_trap_metadata *metadata) { const struct flow_action_cookie *fa_cookie; - struct net_dm_hw_metadata *n_hw_metadata; + struct devlink_trap_metadata *hw_metadata; const char *trap_group_name; const char *trap_name; - n_hw_metadata = kzalloc(sizeof(*hw_metadata), GFP_ATOMIC); - if (!n_hw_metadata) + hw_metadata = kzalloc(sizeof(*hw_metadata), GFP_ATOMIC); + if (!hw_metadata) return NULL; - trap_group_name = kstrdup(hw_metadata->trap_group_name, GFP_ATOMIC); + trap_group_name = kstrdup(metadata->trap_group_name, GFP_ATOMIC); if (!trap_group_name) goto free_hw_metadata; - n_hw_metadata->trap_group_name = trap_group_name; + hw_metadata->trap_group_name = trap_group_name; - trap_name = kstrdup(hw_metadata->trap_name, GFP_ATOMIC); + trap_name = kstrdup(metadata->trap_name, GFP_ATOMIC); if (!trap_name) goto free_trap_group; - n_hw_metadata->trap_name = trap_name; + hw_metadata->trap_name = trap_name; - if (hw_metadata->fa_cookie) { + if (metadata->fa_cookie) { size_t cookie_size = sizeof(*fa_cookie) + - hw_metadata->fa_cookie->cookie_len; + metadata->fa_cookie->cookie_len; - fa_cookie = kmemdup(hw_metadata->fa_cookie, cookie_size, + fa_cookie = kmemdup(metadata->fa_cookie, cookie_size, GFP_ATOMIC); if (!fa_cookie) goto free_trap_name; - n_hw_metadata->fa_cookie = fa_cookie; + hw_metadata->fa_cookie = fa_cookie; } - n_hw_metadata->input_dev = hw_metadata->input_dev; - if (n_hw_metadata->input_dev) - dev_hold(n_hw_metadata->input_dev); + hw_metadata->input_dev = metadata->input_dev; + if (hw_metadata->input_dev) + dev_hold(hw_metadata->input_dev); - return n_hw_metadata; + return hw_metadata; free_trap_name: kfree(trap_name); free_trap_group: kfree(trap_group_name); free_hw_metadata: - kfree(n_hw_metadata); + kfree(hw_metadata); return NULL; } static void -net_dm_hw_metadata_free(const struct net_dm_hw_metadata *hw_metadata) +net_dm_hw_metadata_free(const struct devlink_trap_metadata *hw_metadata) { if (hw_metadata->input_dev) dev_put(hw_metadata->input_dev); @@ -871,7 +877,7 @@ net_dm_hw_metadata_free(const struct net_dm_hw_metadata *hw_metadata) static void net_dm_hw_packet_report(struct sk_buff *skb) { - struct net_dm_hw_metadata *hw_metadata; + struct devlink_trap_metadata *hw_metadata; struct sk_buff *msg; size_t payload_len; int rc; @@ -924,15 +930,19 @@ static void net_dm_hw_packet_work(struct work_struct *work) } static void -net_dm_hw_packet_probe(struct sk_buff *skb, - const struct net_dm_hw_metadata *hw_metadata) +net_dm_hw_trap_packet_probe(void *ignore, const struct devlink *devlink, + struct sk_buff *skb, + const struct devlink_trap_metadata *metadata) { - struct net_dm_hw_metadata *n_hw_metadata; + struct devlink_trap_metadata *n_hw_metadata; ktime_t tstamp = ktime_get_real(); struct per_cpu_dm_data *hw_data; struct sk_buff *nskb; unsigned long flags; + if (metadata->trap_type == DEVLINK_TRAP_TYPE_CONTROL) + return; + if (!skb_mac_header_was_set(skb)) return; @@ -940,7 +950,7 @@ net_dm_hw_packet_probe(struct sk_buff *skb, if (!nskb) return; - n_hw_metadata = net_dm_hw_metadata_clone(hw_metadata); + n_hw_metadata = net_dm_hw_metadata_copy(metadata); if (!n_hw_metadata) goto free; @@ -975,7 +985,7 @@ static const struct net_dm_alert_ops net_dm_alert_packet_ops = { .napi_poll_probe = net_dm_packet_trace_napi_poll_hit, .work_item_func = net_dm_packet_work, .hw_work_item_func = net_dm_hw_packet_work, - .hw_probe = net_dm_hw_packet_probe, + .hw_trap_probe = net_dm_hw_trap_packet_probe, }; static const struct net_dm_alert_ops *net_dm_alert_ops_arr[] = { @@ -983,25 +993,32 @@ static const struct net_dm_alert_ops *net_dm_alert_ops_arr[] = { [NET_DM_ALERT_MODE_PACKET] = &net_dm_alert_packet_ops, }; -void net_dm_hw_report(struct sk_buff *skb, - const struct net_dm_hw_metadata *hw_metadata) +#if IS_ENABLED(CONFIG_NET_DEVLINK) +static int net_dm_hw_probe_register(const struct net_dm_alert_ops *ops) { - rcu_read_lock(); - - if (!monitor_hw) - goto out; + return register_trace_devlink_trap_report(ops->hw_trap_probe, NULL); +} - net_dm_alert_ops_arr[net_dm_alert_mode]->hw_probe(skb, hw_metadata); +static void net_dm_hw_probe_unregister(const struct net_dm_alert_ops *ops) +{ + unregister_trace_devlink_trap_report(ops->hw_trap_probe, NULL); + tracepoint_synchronize_unregister(); +} +#else +static int net_dm_hw_probe_register(const struct net_dm_alert_ops *ops) +{ + return -EOPNOTSUPP; +} -out: - rcu_read_unlock(); +static void net_dm_hw_probe_unregister(const struct net_dm_alert_ops *ops) +{ } -EXPORT_SYMBOL_GPL(net_dm_hw_report); +#endif static int net_dm_hw_monitor_start(struct netlink_ext_ack *extack) { const struct net_dm_alert_ops *ops; - int cpu; + int cpu, rc; if (monitor_hw) { NL_SET_ERR_MSG_MOD(extack, "Hardware monitoring already enabled"); @@ -1025,13 +1042,24 @@ static int net_dm_hw_monitor_start(struct netlink_ext_ack *extack) kfree(hw_entries); } + rc = net_dm_hw_probe_register(ops); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, "Failed to connect probe to devlink_trap_probe() tracepoint"); + goto err_module_put; + } + monitor_hw = true; return 0; + +err_module_put: + module_put(THIS_MODULE); + return rc; } static void net_dm_hw_monitor_stop(struct netlink_ext_ack *extack) { + const struct net_dm_alert_ops *ops; int cpu; if (!monitor_hw) { @@ -1039,12 +1067,11 @@ static void net_dm_hw_monitor_stop(struct netlink_ext_ack *extack) return; } + ops = net_dm_alert_ops_arr[net_dm_alert_mode]; + monitor_hw = false; - /* After this call returns we are guaranteed that no CPU is processing - * any hardware drops. - */ - synchronize_rcu(); + net_dm_hw_probe_unregister(ops); for_each_possible_cpu(cpu) { struct per_cpu_dm_data *hw_data = &per_cpu(dm_hw_cpu_data, cpu); @@ -1053,7 +1080,7 @@ static void net_dm_hw_monitor_stop(struct netlink_ext_ack *extack) del_timer_sync(&hw_data->send_timer); cancel_work_sync(&hw_data->dm_alert_work); while ((skb = __skb_dequeue(&hw_data->drop_queue))) { - struct net_dm_hw_metadata *hw_metadata; + struct devlink_trap_metadata *hw_metadata; hw_metadata = NET_DM_SKB_CB(skb)->hw_metadata; net_dm_hw_metadata_free(hw_metadata); diff --git a/net/core/filter.c b/net/core/filter.c index 706f8db0ccf8..3fb6adad1957 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -77,6 +77,9 @@ #include <net/transp_v6.h> #include <linux/btf_ids.h> +static const struct bpf_func_proto * +bpf_sk_base_func_proto(enum bpf_func_id func_id); + int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len) { if (in_compat_syscall()) { @@ -2160,13 +2163,233 @@ static int __bpf_redirect(struct sk_buff *skb, struct net_device *dev, return __bpf_redirect_no_mac(skb, dev, flags); } +#if IS_ENABLED(CONFIG_IPV6) +static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb) +{ + struct dst_entry *dst = skb_dst(skb); + struct net_device *dev = dst->dev; + u32 hh_len = LL_RESERVED_SPACE(dev); + const struct in6_addr *nexthop; + struct neighbour *neigh; + + if (dev_xmit_recursion()) { + net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n"); + goto out_drop; + } + + skb->dev = dev; + skb->tstamp = 0; + + if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { + struct sk_buff *skb2; + + skb2 = skb_realloc_headroom(skb, hh_len); + if (unlikely(!skb2)) { + kfree_skb(skb); + return -ENOMEM; + } + if (skb->sk) + skb_set_owner_w(skb2, skb->sk); + consume_skb(skb); + skb = skb2; + } + + rcu_read_lock_bh(); + nexthop = rt6_nexthop(container_of(dst, struct rt6_info, dst), + &ipv6_hdr(skb)->daddr); + neigh = ip_neigh_gw6(dev, nexthop); + if (likely(!IS_ERR(neigh))) { + int ret; + + sock_confirm_neigh(skb, neigh); + dev_xmit_recursion_inc(); + ret = neigh_output(neigh, skb, false); + dev_xmit_recursion_dec(); + rcu_read_unlock_bh(); + return ret; + } + rcu_read_unlock_bh(); + IP6_INC_STATS(dev_net(dst->dev), + ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES); +out_drop: + kfree_skb(skb); + return -ENETDOWN; +} + +static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev) +{ + const struct ipv6hdr *ip6h = ipv6_hdr(skb); + struct net *net = dev_net(dev); + int err, ret = NET_XMIT_DROP; + struct dst_entry *dst; + struct flowi6 fl6 = { + .flowi6_flags = FLOWI_FLAG_ANYSRC, + .flowi6_mark = skb->mark, + .flowlabel = ip6_flowinfo(ip6h), + .flowi6_oif = dev->ifindex, + .flowi6_proto = ip6h->nexthdr, + .daddr = ip6h->daddr, + .saddr = ip6h->saddr, + }; + + dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL); + if (IS_ERR(dst)) + goto out_drop; + + skb_dst_set(skb, dst); + + err = bpf_out_neigh_v6(net, skb); + if (unlikely(net_xmit_eval(err))) + dev->stats.tx_errors++; + else + ret = NET_XMIT_SUCCESS; + goto out_xmit; +out_drop: + dev->stats.tx_errors++; + kfree_skb(skb); +out_xmit: + return ret; +} +#else +static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev) +{ + kfree_skb(skb); + return NET_XMIT_DROP; +} +#endif /* CONFIG_IPV6 */ + +#if IS_ENABLED(CONFIG_INET) +static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb) +{ + struct dst_entry *dst = skb_dst(skb); + struct rtable *rt = container_of(dst, struct rtable, dst); + struct net_device *dev = dst->dev; + u32 hh_len = LL_RESERVED_SPACE(dev); + struct neighbour *neigh; + bool is_v6gw = false; + + if (dev_xmit_recursion()) { + net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n"); + goto out_drop; + } + + skb->dev = dev; + skb->tstamp = 0; + + if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { + struct sk_buff *skb2; + + skb2 = skb_realloc_headroom(skb, hh_len); + if (unlikely(!skb2)) { + kfree_skb(skb); + return -ENOMEM; + } + if (skb->sk) + skb_set_owner_w(skb2, skb->sk); + consume_skb(skb); + skb = skb2; + } + + rcu_read_lock_bh(); + neigh = ip_neigh_for_gw(rt, skb, &is_v6gw); + if (likely(!IS_ERR(neigh))) { + int ret; + + sock_confirm_neigh(skb, neigh); + dev_xmit_recursion_inc(); + ret = neigh_output(neigh, skb, is_v6gw); + dev_xmit_recursion_dec(); + rcu_read_unlock_bh(); + return ret; + } + rcu_read_unlock_bh(); +out_drop: + kfree_skb(skb); + return -ENETDOWN; +} + +static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev) +{ + const struct iphdr *ip4h = ip_hdr(skb); + struct net *net = dev_net(dev); + int err, ret = NET_XMIT_DROP; + struct rtable *rt; + struct flowi4 fl4 = { + .flowi4_flags = FLOWI_FLAG_ANYSRC, + .flowi4_mark = skb->mark, + .flowi4_tos = RT_TOS(ip4h->tos), + .flowi4_oif = dev->ifindex, + .flowi4_proto = ip4h->protocol, + .daddr = ip4h->daddr, + .saddr = ip4h->saddr, + }; + + rt = ip_route_output_flow(net, &fl4, NULL); + if (IS_ERR(rt)) + goto out_drop; + if (rt->rt_type != RTN_UNICAST && rt->rt_type != RTN_LOCAL) { + ip_rt_put(rt); + goto out_drop; + } + + skb_dst_set(skb, &rt->dst); + + err = bpf_out_neigh_v4(net, skb); + if (unlikely(net_xmit_eval(err))) + dev->stats.tx_errors++; + else + ret = NET_XMIT_SUCCESS; + goto out_xmit; +out_drop: + dev->stats.tx_errors++; + kfree_skb(skb); +out_xmit: + return ret; +} +#else +static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev) +{ + kfree_skb(skb); + return NET_XMIT_DROP; +} +#endif /* CONFIG_INET */ + +static int __bpf_redirect_neigh(struct sk_buff *skb, struct net_device *dev) +{ + struct ethhdr *ethh = eth_hdr(skb); + + if (unlikely(skb->mac_header >= skb->network_header)) + goto out; + bpf_push_mac_rcsum(skb); + if (is_multicast_ether_addr(ethh->h_dest)) + goto out; + + skb_pull(skb, sizeof(*ethh)); + skb_unset_mac_header(skb); + skb_reset_network_header(skb); + + if (skb->protocol == htons(ETH_P_IP)) + return __bpf_redirect_neigh_v4(skb, dev); + else if (skb->protocol == htons(ETH_P_IPV6)) + return __bpf_redirect_neigh_v6(skb, dev); +out: + kfree_skb(skb); + return -ENOTSUPP; +} + +/* Internal, non-exposed redirect flags. */ +enum { + BPF_F_NEIGH = (1ULL << 1), +#define BPF_F_REDIRECT_INTERNAL (BPF_F_NEIGH) +}; + BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags) { struct net_device *dev; struct sk_buff *clone; int ret; - if (unlikely(flags & ~(BPF_F_INGRESS))) + if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL))) return -EINVAL; dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex); @@ -2203,23 +2426,11 @@ static const struct bpf_func_proto bpf_clone_redirect_proto = { DEFINE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info); EXPORT_PER_CPU_SYMBOL_GPL(bpf_redirect_info); -BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags) -{ - struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); - - if (unlikely(flags & ~(BPF_F_INGRESS))) - return TC_ACT_SHOT; - - ri->flags = flags; - ri->tgt_index = ifindex; - - return TC_ACT_REDIRECT; -} - int skb_do_redirect(struct sk_buff *skb) { struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); struct net_device *dev; + u32 flags = ri->flags; dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->tgt_index); ri->tgt_index = 0; @@ -2228,7 +2439,22 @@ int skb_do_redirect(struct sk_buff *skb) return -EINVAL; } - return __bpf_redirect(skb, dev, ri->flags); + return flags & BPF_F_NEIGH ? + __bpf_redirect_neigh(skb, dev) : + __bpf_redirect(skb, dev, flags); +} + +BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags) +{ + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); + + if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL))) + return TC_ACT_SHOT; + + ri->flags = flags; + ri->tgt_index = ifindex; + + return TC_ACT_REDIRECT; } static const struct bpf_func_proto bpf_redirect_proto = { @@ -2239,6 +2465,27 @@ static const struct bpf_func_proto bpf_redirect_proto = { .arg2_type = ARG_ANYTHING, }; +BPF_CALL_2(bpf_redirect_neigh, u32, ifindex, u64, flags) +{ + struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); + + if (unlikely(flags)) + return TC_ACT_SHOT; + + ri->flags = BPF_F_NEIGH; + ri->tgt_index = ifindex; + + return TC_ACT_REDIRECT; +} + +static const struct bpf_func_proto bpf_redirect_neigh_proto = { + .func = bpf_redirect_neigh, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_ANYTHING, + .arg2_type = ARG_ANYTHING, +}; + BPF_CALL_2(bpf_msg_apply_bytes, struct sk_msg *, msg, u32, bytes) { msg->apply_bytes = bytes; @@ -2704,6 +2951,23 @@ static const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = { .gpl_only = false, .ret_type = RET_INTEGER, }; + +BPF_CALL_1(bpf_skb_cgroup_classid, const struct sk_buff *, skb) +{ + struct sock *sk = skb_to_full_sk(skb); + + if (!sk || !sk_fullsock(sk)) + return 0; + + return sock_cgroup_classid(&sk->sk_cgrp_data); +} + +static const struct bpf_func_proto bpf_skb_cgroup_classid_proto = { + .func = bpf_skb_cgroup_classid, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, +}; #endif BPF_CALL_1(bpf_get_cgroup_classid, const struct sk_buff *, skb) @@ -4085,18 +4349,17 @@ static inline u64 __bpf_sk_cgroup_id(struct sock *sk) { struct cgroup *cgrp; + sk = sk_to_full_sk(sk); + if (!sk || !sk_fullsock(sk)) + return 0; + cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); return cgroup_id(cgrp); } BPF_CALL_1(bpf_skb_cgroup_id, const struct sk_buff *, skb) { - struct sock *sk = skb_to_full_sk(skb); - - if (!sk || !sk_fullsock(sk)) - return 0; - - return __bpf_sk_cgroup_id(sk); + return __bpf_sk_cgroup_id(skb->sk); } static const struct bpf_func_proto bpf_skb_cgroup_id_proto = { @@ -4112,6 +4375,10 @@ static inline u64 __bpf_sk_ancestor_cgroup_id(struct sock *sk, struct cgroup *ancestor; struct cgroup *cgrp; + sk = sk_to_full_sk(sk); + if (!sk || !sk_fullsock(sk)) + return 0; + cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); ancestor = cgroup_ancestor(cgrp, ancestor_level); if (!ancestor) @@ -4123,12 +4390,7 @@ static inline u64 __bpf_sk_ancestor_cgroup_id(struct sock *sk, BPF_CALL_2(bpf_skb_ancestor_cgroup_id, const struct sk_buff *, skb, int, ancestor_level) { - struct sock *sk = skb_to_full_sk(skb); - - if (!sk || !sk_fullsock(sk)) - return 0; - - return __bpf_sk_ancestor_cgroup_id(sk, ancestor_level); + return __bpf_sk_ancestor_cgroup_id(skb->sk, ancestor_level); } static const struct bpf_func_proto bpf_skb_ancestor_cgroup_id_proto = { @@ -4148,7 +4410,7 @@ static const struct bpf_func_proto bpf_sk_cgroup_id_proto = { .func = bpf_sk_cgroup_id, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_SOCKET, + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, }; BPF_CALL_2(bpf_sk_ancestor_cgroup_id, struct sock *, sk, int, ancestor_level) @@ -4160,7 +4422,7 @@ static const struct bpf_func_proto bpf_sk_ancestor_cgroup_id_proto = { .func = bpf_sk_ancestor_cgroup_id, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_SOCKET, + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .arg2_type = ARG_ANYTHING, }; #endif @@ -4214,7 +4476,7 @@ const struct bpf_func_proto bpf_xdp_output_proto = { BPF_CALL_1(bpf_get_socket_cookie, struct sk_buff *, skb) { - return skb->sk ? sock_gen_cookie(skb->sk) : 0; + return skb->sk ? __sock_gen_cookie(skb->sk) : 0; } static const struct bpf_func_proto bpf_get_socket_cookie_proto = { @@ -4226,7 +4488,7 @@ static const struct bpf_func_proto bpf_get_socket_cookie_proto = { BPF_CALL_1(bpf_get_socket_cookie_sock_addr, struct bpf_sock_addr_kern *, ctx) { - return sock_gen_cookie(ctx->sk); + return __sock_gen_cookie(ctx->sk); } static const struct bpf_func_proto bpf_get_socket_cookie_sock_addr_proto = { @@ -4238,7 +4500,7 @@ static const struct bpf_func_proto bpf_get_socket_cookie_sock_addr_proto = { BPF_CALL_1(bpf_get_socket_cookie_sock, struct sock *, ctx) { - return sock_gen_cookie(ctx); + return __sock_gen_cookie(ctx); } static const struct bpf_func_proto bpf_get_socket_cookie_sock_proto = { @@ -4250,7 +4512,7 @@ static const struct bpf_func_proto bpf_get_socket_cookie_sock_proto = { BPF_CALL_1(bpf_get_socket_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx) { - return sock_gen_cookie(ctx->sk); + return __sock_gen_cookie(ctx->sk); } static const struct bpf_func_proto bpf_get_socket_cookie_sock_ops_proto = { @@ -4263,7 +4525,7 @@ static const struct bpf_func_proto bpf_get_socket_cookie_sock_ops_proto = { static u64 __bpf_get_netns_cookie(struct sock *sk) { #ifdef CONFIG_NET_NS - return net_gen_cookie(sk ? sk->sk_net.net : &init_net); + return __net_gen_cookie(sk ? sk->sk_net.net : &init_net); #else return 0; #endif @@ -5694,7 +5956,7 @@ static const struct bpf_func_proto bpf_sk_lookup_udp_proto = { BPF_CALL_1(bpf_sk_release, struct sock *, sk) { - if (sk_is_refcounted(sk)) + if (sk && sk_is_refcounted(sk)) sock_gen_put(sk); return 0; } @@ -5703,7 +5965,7 @@ static const struct bpf_func_proto bpf_sk_release_proto = { .func = bpf_sk_release, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_SOCK_COMMON, + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, }; BPF_CALL_5(bpf_xdp_sk_lookup_udp, struct xdp_buff *, ctx, @@ -6085,7 +6347,7 @@ BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len u32 cookie; int ret; - if (unlikely(th_len < sizeof(*th))) + if (unlikely(!sk || th_len < sizeof(*th))) return -EINVAL; /* sk_listener() allows TCP_NEW_SYN_RECV, which makes no sense here. */ @@ -6138,7 +6400,7 @@ static const struct bpf_func_proto bpf_tcp_check_syncookie_proto = { .gpl_only = true, .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_SOCK_COMMON, + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .arg2_type = ARG_PTR_TO_MEM, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_PTR_TO_MEM, @@ -6152,7 +6414,7 @@ BPF_CALL_5(bpf_tcp_gen_syncookie, struct sock *, sk, void *, iph, u32, iph_len, u32 cookie; u16 mss; - if (unlikely(th_len < sizeof(*th) || th_len != th->doff * 4)) + if (unlikely(!sk || th_len < sizeof(*th) || th_len != th->doff * 4)) return -EINVAL; if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN) @@ -6207,7 +6469,7 @@ static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = { .gpl_only = true, /* __cookie_v*_init_sequence() is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_SOCK_COMMON, + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .arg2_type = ARG_PTR_TO_MEM, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_PTR_TO_MEM, @@ -6216,7 +6478,7 @@ static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = { BPF_CALL_3(bpf_sk_assign, struct sk_buff *, skb, struct sock *, sk, u64, flags) { - if (flags != 0) + if (!sk || flags != 0) return -EINVAL; if (!skb_at_tc_ingress(skb)) return -EOPNOTSUPP; @@ -6240,7 +6502,7 @@ static const struct bpf_func_proto bpf_sk_assign_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_SOCK_COMMON, + .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .arg3_type = ARG_ANYTHING, }; @@ -6620,7 +6882,7 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return NULL; } default: - return bpf_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id); } } @@ -6639,7 +6901,7 @@ sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_skb_event_output_proto; default: - return bpf_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id); } } @@ -6741,6 +7003,8 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return bpf_get_skb_set_tunnel_proto(func_id); case BPF_FUNC_redirect: return &bpf_redirect_proto; + case BPF_FUNC_redirect_neigh: + return &bpf_redirect_neigh_proto; case BPF_FUNC_get_route_realm: return &bpf_get_route_realm_proto; case BPF_FUNC_get_hash_recalc: @@ -6771,6 +7035,10 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_get_xfrm_state: return &bpf_skb_get_xfrm_state_proto; #endif +#ifdef CONFIG_CGROUP_NET_CLASSID + case BPF_FUNC_skb_cgroup_classid: + return &bpf_skb_cgroup_classid_proto; +#endif #ifdef CONFIG_SOCK_CGROUP_DATA case BPF_FUNC_skb_cgroup_id: return &bpf_skb_cgroup_id_proto; @@ -6800,7 +7068,7 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_sk_assign_proto; #endif default: - return bpf_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id); } } @@ -6841,7 +7109,7 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_tcp_gen_syncookie_proto; #endif default: - return bpf_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id); } } @@ -6883,7 +7151,7 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_tcp_sock_proto; #endif /* CONFIG_INET */ default: - return bpf_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id); } } @@ -6929,7 +7197,7 @@ sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_get_cgroup_classid_curr_proto; #endif default: - return bpf_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id); } } @@ -6971,7 +7239,7 @@ sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_skc_lookup_tcp_proto; #endif default: - return bpf_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id); } } @@ -6982,7 +7250,7 @@ flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_load_bytes: return &bpf_flow_dissector_load_bytes_proto; default: - return bpf_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id); } } @@ -7009,7 +7277,7 @@ lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_under_cgroup: return &bpf_skb_under_cgroup_proto; default: - return bpf_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id); } } @@ -9746,7 +10014,7 @@ sk_lookup_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_sk_release: return &bpf_sk_release_proto; default: - return bpf_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id); } } @@ -9913,8 +10181,7 @@ const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto = { .func = bpf_skc_to_tcp6_sock, .gpl_only = false, .ret_type = RET_PTR_TO_BTF_ID_OR_NULL, - .arg1_type = ARG_PTR_TO_BTF_ID, - .arg1_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP6], }; @@ -9930,8 +10197,7 @@ const struct bpf_func_proto bpf_skc_to_tcp_sock_proto = { .func = bpf_skc_to_tcp_sock, .gpl_only = false, .ret_type = RET_PTR_TO_BTF_ID_OR_NULL, - .arg1_type = ARG_PTR_TO_BTF_ID, - .arg1_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP], }; @@ -9954,8 +10220,7 @@ const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto = { .func = bpf_skc_to_tcp_timewait_sock, .gpl_only = false, .ret_type = RET_PTR_TO_BTF_ID_OR_NULL, - .arg1_type = ARG_PTR_TO_BTF_ID, - .arg1_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP_TW], }; @@ -9978,8 +10243,7 @@ const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto = { .func = bpf_skc_to_tcp_request_sock, .gpl_only = false, .ret_type = RET_PTR_TO_BTF_ID_OR_NULL, - .arg1_type = ARG_PTR_TO_BTF_ID, - .arg1_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_TCP_REQ], }; @@ -10000,7 +10264,37 @@ const struct bpf_func_proto bpf_skc_to_udp6_sock_proto = { .func = bpf_skc_to_udp6_sock, .gpl_only = false, .ret_type = RET_PTR_TO_BTF_ID_OR_NULL, - .arg1_type = ARG_PTR_TO_BTF_ID, - .arg1_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .ret_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_UDP6], }; + +static const struct bpf_func_proto * +bpf_sk_base_func_proto(enum bpf_func_id func_id) +{ + const struct bpf_func_proto *func; + + switch (func_id) { + case BPF_FUNC_skc_to_tcp6_sock: + func = &bpf_skc_to_tcp6_sock_proto; + break; + case BPF_FUNC_skc_to_tcp_sock: + func = &bpf_skc_to_tcp_sock_proto; + break; + case BPF_FUNC_skc_to_tcp_timewait_sock: + func = &bpf_skc_to_tcp_timewait_sock_proto; + break; + case BPF_FUNC_skc_to_tcp_request_sock: + func = &bpf_skc_to_tcp_request_sock_proto; + break; + case BPF_FUNC_skc_to_udp6_sock: + func = &bpf_skc_to_udp6_sock_proto; + break; + default: + return bpf_base_func_proto(func_id); + } + + if (!perfmon_capable()) + return NULL; + + return func; +} diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index efec66fa78b7..94fff0700bdd 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -1158,8 +1158,8 @@ static ssize_t traffic_class_show(struct netdev_queue *queue, * belongs to the root device it will be reported with just the * traffic class, so just "0" for TC 0 for example. */ - return dev->num_tc < 0 ? sprintf(buf, "%u%d\n", tc, dev->num_tc) : - sprintf(buf, "%u\n", tc); + return dev->num_tc < 0 ? sprintf(buf, "%d%d\n", tc, dev->num_tc) : + sprintf(buf, "%d\n", tc); } #ifdef CONFIG_XPS diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index 944ab214e5ae..dbc66b896287 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -19,6 +19,7 @@ #include <linux/net_namespace.h> #include <linux/sched/task.h> #include <linux/uidgid.h> +#include <linux/cookie.h> #include <net/sock.h> #include <net/netlink.h> @@ -69,16 +70,16 @@ EXPORT_SYMBOL_GPL(pernet_ops_rwsem); static unsigned int max_gen_ptrs = INITIAL_NET_GEN_PTRS; -static atomic64_t cookie_gen; +DEFINE_COOKIE(net_cookie); -u64 net_gen_cookie(struct net *net) +u64 __net_gen_cookie(struct net *net) { while (1) { u64 res = atomic64_read(&net->net_cookie); if (res) return res; - res = atomic64_inc_return(&cookie_gen); + res = gen_cookie_next(&net_cookie); atomic64_cmpxchg(&net->net_cookie, 0, res); } } @@ -1101,7 +1102,10 @@ static int __init net_ns_init(void) panic("Could not allocate generic netns"); rcu_assign_pointer(init_net.gen, ng); - net_gen_cookie(&init_net); + + preempt_disable(); + __net_gen_cookie(&init_net); + preempt_enable(); down_write(&pernet_ops_rwsem); if (setup_net(&init_net, &init_user_ns)) diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 44fdbb9c6e53..105978604ffd 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -922,7 +922,7 @@ static ssize_t pktgen_if_write(struct file *file, pkt_dev->min_pkt_size = value; pkt_dev->cur_pkt_size = value; } - sprintf(pg_result, "OK: min_pkt_size=%u", + sprintf(pg_result, "OK: min_pkt_size=%d", pkt_dev->min_pkt_size); return count; } @@ -939,7 +939,7 @@ static ssize_t pktgen_if_write(struct file *file, pkt_dev->max_pkt_size = value; pkt_dev->cur_pkt_size = value; } - sprintf(pg_result, "OK: max_pkt_size=%u", + sprintf(pg_result, "OK: max_pkt_size=%d", pkt_dev->max_pkt_size); return count; } @@ -959,7 +959,7 @@ static ssize_t pktgen_if_write(struct file *file, pkt_dev->max_pkt_size = value; pkt_dev->cur_pkt_size = value; } - sprintf(pg_result, "OK: pkt_size=%u", pkt_dev->min_pkt_size); + sprintf(pg_result, "OK: pkt_size=%d", pkt_dev->min_pkt_size); return count; } @@ -981,7 +981,7 @@ static ssize_t pktgen_if_write(struct file *file, i += len; pkt_dev->nfrags = value; - sprintf(pg_result, "OK: frags=%u", pkt_dev->nfrags); + sprintf(pg_result, "OK: frags=%d", pkt_dev->nfrags); return count; } if (!strcmp(name, "delay")) { @@ -1146,7 +1146,7 @@ static ssize_t pktgen_if_write(struct file *file, (!(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING))))) return -ENOTSUPP; pkt_dev->burst = value < 1 ? 1 : value; - sprintf(pg_result, "OK: burst=%d", pkt_dev->burst); + sprintf(pg_result, "OK: burst=%u", pkt_dev->burst); return count; } if (!strcmp(name, "node")) { diff --git a/net/core/sock_diag.c b/net/core/sock_diag.c index c13ffbd33d8d..c9c45b935f99 100644 --- a/net/core/sock_diag.c +++ b/net/core/sock_diag.c @@ -11,7 +11,7 @@ #include <linux/tcp.h> #include <linux/workqueue.h> #include <linux/nospec.h> - +#include <linux/cookie.h> #include <linux/inet_diag.h> #include <linux/sock_diag.h> @@ -19,16 +19,17 @@ static const struct sock_diag_handler *sock_diag_handlers[AF_MAX]; static int (*inet_rcv_compat)(struct sk_buff *skb, struct nlmsghdr *nlh); static DEFINE_MUTEX(sock_diag_table_mutex); static struct workqueue_struct *broadcast_wq; -static atomic64_t cookie_gen; -u64 sock_gen_cookie(struct sock *sk) +DEFINE_COOKIE(sock_cookie); + +u64 __sock_gen_cookie(struct sock *sk) { while (1) { u64 res = atomic64_read(&sk->sk_cookie); if (res) return res; - res = atomic64_inc_return(&cookie_gen); + res = gen_cookie_next(&sock_cookie); atomic64_cmpxchg(&sk->sk_cookie, 0, res); } } diff --git a/net/core/sock_map.c b/net/core/sock_map.c index e1f05e3fa1d0..e83a80e8f13b 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -401,7 +401,7 @@ static void *sock_map_lookup_sys(struct bpf_map *map, void *key) if (!sk) return ERR_PTR(-ENOENT); - sock_gen_cookie(sk); + __sock_gen_cookie(sk); return &sk->sk_cookie; } @@ -610,6 +610,9 @@ static int sock_map_update_elem(struct bpf_map *map, void *key, struct sock *sk = (struct sock *)value; int ret; + if (unlikely(!sk || !sk_fullsock(sk))) + return -EINVAL; + if (!sock_map_sk_is_suitable(sk)) return -EOPNOTSUPP; @@ -1206,7 +1209,7 @@ static void *sock_hash_lookup_sys(struct bpf_map *map, void *key) if (!sk) return ERR_PTR(-ENOENT); - sock_gen_cookie(sk); + __sock_gen_cookie(sk); return &sk->sk_cookie; } diff --git a/net/dccp/timer.c b/net/dccp/timer.c index 927c796d7682..a934d2932373 100644 --- a/net/dccp/timer.c +++ b/net/dccp/timer.c @@ -176,7 +176,6 @@ static void dccp_delack_timer(struct timer_list *t) bh_lock_sock(sk); if (sock_owned_by_user(sk)) { /* Try again later. */ - icsk->icsk_ack.blocked = 1; __NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOCKED); sk_reset_timer(sk, &icsk->icsk_delack_timer, jiffies + TCP_DELACK_MIN); diff --git a/net/dsa/dsa.c b/net/dsa/dsa.c index 5c18c0214aac..dec4ab59b7c4 100644 --- a/net/dsa/dsa.c +++ b/net/dsa/dsa.c @@ -225,6 +225,15 @@ static int dsa_switch_rcv(struct sk_buff *skb, struct net_device *dev, skb->pkt_type = PACKET_HOST; skb->protocol = eth_type_trans(skb, skb->dev); + if (unlikely(cpu_dp->ds->untag_bridge_pvid)) { + nskb = dsa_untag_bridge_pvid(skb); + if (!nskb) { + kfree_skb(skb); + return 0; + } + skb = nskb; + } + s = this_cpu_ptr(p->stats64); u64_stats_update_begin(&s->syncp); s->rx_packets++; diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h index 0348dbab4131..12998bf04e55 100644 --- a/net/dsa/dsa_priv.h +++ b/net/dsa/dsa_priv.h @@ -201,11 +201,9 @@ dsa_slave_to_master(const struct net_device *dev) static inline struct sk_buff *dsa_untag_bridge_pvid(struct sk_buff *skb) { struct dsa_port *dp = dsa_slave_to_port(skb->dev); - struct vlan_ethhdr *hdr = vlan_eth_hdr(skb); struct net_device *br = dp->bridge_dev; struct net_device *dev = skb->dev; struct net_device *upper_dev; - struct list_head *iter; u16 vid, pvid, proto; int err; @@ -217,7 +215,7 @@ static inline struct sk_buff *dsa_untag_bridge_pvid(struct sk_buff *skb) return skb; /* Move VLAN tag from data to hwaccel */ - if (!skb_vlan_tag_present(skb) && hdr->h_vlan_proto == htons(proto)) { + if (!skb_vlan_tag_present(skb) && skb->protocol == htons(proto)) { skb = skb_vlan_untag(skb); if (!skb) return NULL; @@ -247,13 +245,9 @@ static inline struct sk_buff *dsa_untag_bridge_pvid(struct sk_buff *skb) * supports because vlan_filtering is 0. In that case, we should * definitely keep the tag, to make sure it keeps working. */ - netdev_for_each_upper_dev_rcu(dev, upper_dev, iter) { - if (!is_vlan_dev(upper_dev)) - continue; - - if (vid == vlan_dev_vlan_id(upper_dev)) - return skb; - } + upper_dev = __vlan_find_dev_deep_rcu(br, htons(proto), vid); + if (upper_dev) + return skb; __vlan_hwaccel_clear_tag(skb); diff --git a/net/dsa/tag_brcm.c b/net/dsa/tag_brcm.c index 69d6b8c597a9..ad72dff8d524 100644 --- a/net/dsa/tag_brcm.c +++ b/net/dsa/tag_brcm.c @@ -152,11 +152,6 @@ static struct sk_buff *brcm_tag_rcv_ll(struct sk_buff *skb, /* Remove Broadcom tag and update checksum */ skb_pull_rcsum(skb, BRCM_TAG_LEN); - /* Set the MAC header to where it should point for - * dsa_untag_bridge_pvid() to parse the correct VLAN header. - */ - skb_set_mac_header(skb, -ETH_HLEN); - skb->offload_fwd_mark = 1; return skb; @@ -187,7 +182,7 @@ static struct sk_buff *brcm_tag_rcv(struct sk_buff *skb, struct net_device *dev, nskb->data - ETH_HLEN - BRCM_TAG_LEN, 2 * ETH_ALEN); - return dsa_untag_bridge_pvid(nskb); + return nskb; } static const struct dsa_device_ops brcm_netdev_ops = { @@ -214,14 +209,8 @@ static struct sk_buff *brcm_tag_rcv_prepend(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt) { - struct sk_buff *nskb; - /* tag is prepended to the packet */ - nskb = brcm_tag_rcv_ll(skb, dev, pt, ETH_HLEN); - if (!nskb) - return nskb; - - return dsa_untag_bridge_pvid(nskb); + return brcm_tag_rcv_ll(skb, dev, pt, ETH_HLEN); } static const struct dsa_device_ops brcm_prepend_netdev_ops = { diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 74a2ef598c31..618954f82764 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -28,22 +28,6 @@ static u32 unsupported_ops[] = { static const struct btf_type *tcp_sock_type; static u32 tcp_sock_id, sock_id; -static struct bpf_func_proto btf_sk_storage_get_proto __read_mostly; -static struct bpf_func_proto btf_sk_storage_delete_proto __read_mostly; - -static void convert_sk_func_proto(struct bpf_func_proto *to, const struct bpf_func_proto *from) -{ - int i; - - *to = *from; - for (i = 0; i < ARRAY_SIZE(to->arg_type); i++) { - if (to->arg_type[i] == ARG_PTR_TO_SOCKET) { - to->arg_type[i] = ARG_PTR_TO_BTF_ID; - to->arg_btf_id[i] = &tcp_sock_id; - } - } -} - static int bpf_tcp_ca_init(struct btf *btf) { s32 type_id; @@ -59,9 +43,6 @@ static int bpf_tcp_ca_init(struct btf *btf) tcp_sock_id = type_id; tcp_sock_type = btf_type_by_id(btf, tcp_sock_id); - convert_sk_func_proto(&btf_sk_storage_get_proto, &bpf_sk_storage_get_proto); - convert_sk_func_proto(&btf_sk_storage_delete_proto, &bpf_sk_storage_delete_proto); - return 0; } @@ -188,9 +169,9 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id, case BPF_FUNC_tcp_send_ack: return &bpf_tcp_send_ack_proto; case BPF_FUNC_sk_storage_get: - return &btf_sk_storage_get_proto; + return &bpf_sk_storage_get_proto; case BPF_FUNC_sk_storage_delete: - return &btf_sk_storage_delete_proto; + return &bpf_sk_storage_delete_proto; default: return bpf_base_func_proto(func_id); } diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index b457dd2d6c75..4148f5f78f31 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -564,7 +564,7 @@ void inet_csk_clear_xmit_timers(struct sock *sk) { struct inet_connection_sock *icsk = inet_csk(sk); - icsk->icsk_pending = icsk->icsk_ack.pending = icsk->icsk_ack.blocked = 0; + icsk->icsk_pending = icsk->icsk_ack.pending = 0; sk_stop_timer(sk, &icsk->icsk_retransmit_timer); sk_stop_timer(sk, &icsk->icsk_delack_timer); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 2a8bfa89a515..ed2805564424 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1538,10 +1538,8 @@ void tcp_cleanup_rbuf(struct sock *sk, int copied) if (inet_csk_ack_scheduled(sk)) { const struct inet_connection_sock *icsk = inet_csk(sk); - /* Delayed ACKs frequently hit locked sockets during bulk - * receive. */ - if (icsk->icsk_ack.blocked || - /* Once-per-two-segments ACK was not sent by tcp_input.c */ + + if (/* Once-per-two-segments ACK was not sent by tcp_input.c */ tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss || /* * If this read emptied read buffer, we send ACK, if diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 386978dcd318..bf48cd73e967 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -3911,11 +3911,8 @@ void tcp_send_delayed_ack(struct sock *sk) /* Use new timeout only if there wasn't a older one earlier. */ if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) { - /* If delack timer was blocked or is about to expire, - * send ACK now. - */ - if (icsk->icsk_ack.blocked || - time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) { + /* If delack timer is about to expire, send ACK now. */ + if (time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) { tcp_send_ack(sk); return; } @@ -3944,10 +3941,15 @@ void __tcp_send_ack(struct sock *sk, u32 rcv_nxt) buff = alloc_skb(MAX_TCP_HEADER, sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN)); if (unlikely(!buff)) { + struct inet_connection_sock *icsk = inet_csk(sk); + unsigned long delay; + + delay = TCP_DELACK_MAX << icsk->icsk_ack.retry; + if (delay < TCP_RTO_MAX) + icsk->icsk_ack.retry++; inet_csk_schedule_ack(sk); - inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN; - inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, - TCP_DELACK_MAX, TCP_RTO_MAX); + icsk->icsk_ack.ato = TCP_ATO_MIN; + inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, delay, TCP_RTO_MAX); return; } diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index 0c08c420fbc2..6c62b9ea1320 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -331,7 +331,6 @@ static void tcp_delack_timer(struct timer_list *t) if (!sock_owned_by_user(sk)) { tcp_delack_timer_handler(sk); } else { - icsk->icsk_ack.blocked = 1; __NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOCKED); /* deleguate our work to tcp_release_cb() */ if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, &sk->sk_tsq_flags)) diff --git a/net/ipv4/udp_tunnel_nic.c b/net/ipv4/udp_tunnel_nic.c index 69962165c0e8..0d122edc368d 100644 --- a/net/ipv4/udp_tunnel_nic.c +++ b/net/ipv4/udp_tunnel_nic.c @@ -19,8 +19,9 @@ enum udp_tunnel_nic_table_entry_flags { struct udp_tunnel_nic_table_entry { __be16 port; u8 type; - u8 use_cnt; u8 flags; + u16 use_cnt; +#define UDP_TUNNEL_NIC_USE_CNT_MAX U16_MAX u8 hw_priv; }; @@ -370,6 +371,8 @@ udp_tunnel_nic_entry_adj(struct udp_tunnel_nic *utn, bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL; unsigned int from, to; + WARN_ON(entry->use_cnt + (u32)use_cnt_adj > U16_MAX); + /* If not going from used to unused or vice versa - all done. * For dodgy entries make sure we try to sync again (queue the entry). */ @@ -675,6 +678,7 @@ static void udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn) { const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; + struct udp_tunnel_nic_shared_node *node; unsigned int i, j; /* Freeze all the ports we are already tracking so that the replay @@ -686,7 +690,12 @@ udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn) utn->missed = 0; utn->need_replay = 0; - udp_tunnel_get_rx_info(dev); + if (!info->shared) { + udp_tunnel_get_rx_info(dev); + } else { + list_for_each_entry(node, &info->shared->devices, list) + udp_tunnel_get_rx_info(node->dev); + } for (i = 0; i < utn->n_tables; i++) for (j = 0; j < info->tables[i].n_entries; j++) @@ -742,20 +751,39 @@ err_free_utn: return NULL; } +static void udp_tunnel_nic_free(struct udp_tunnel_nic *utn) +{ + unsigned int i; + + for (i = 0; i < utn->n_tables; i++) + kfree(utn->entries[i]); + kfree(utn->entries); + kfree(utn); +} + static int udp_tunnel_nic_register(struct net_device *dev) { const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; + struct udp_tunnel_nic_shared_node *node = NULL; struct udp_tunnel_nic *utn; unsigned int n_tables, i; BUILD_BUG_ON(sizeof(utn->missed) * BITS_PER_BYTE < UDP_TUNNEL_NIC_MAX_TABLES); + /* Expect use count of at most 2 (IPv4, IPv6) per device */ + BUILD_BUG_ON(UDP_TUNNEL_NIC_USE_CNT_MAX < + UDP_TUNNEL_NIC_MAX_SHARING_DEVICES * 2); + /* Check that the driver info is sane */ if (WARN_ON(!info->set_port != !info->unset_port) || WARN_ON(!info->set_port == !info->sync_table) || WARN_ON(!info->tables[0].n_entries)) return -EINVAL; + if (WARN_ON(info->shared && + info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)) + return -EINVAL; + n_tables = 1; for (i = 1; i < UDP_TUNNEL_NIC_MAX_TABLES; i++) { if (!info->tables[i].n_entries) @@ -766,9 +794,33 @@ static int udp_tunnel_nic_register(struct net_device *dev) return -EINVAL; } - utn = udp_tunnel_nic_alloc(info, n_tables); - if (!utn) - return -ENOMEM; + /* Create UDP tunnel state structures */ + if (info->shared) { + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + node->dev = dev; + } + + if (info->shared && info->shared->udp_tunnel_nic_info) { + utn = info->shared->udp_tunnel_nic_info; + } else { + utn = udp_tunnel_nic_alloc(info, n_tables); + if (!utn) { + kfree(node); + return -ENOMEM; + } + } + + if (info->shared) { + if (!info->shared->udp_tunnel_nic_info) { + INIT_LIST_HEAD(&info->shared->devices); + info->shared->udp_tunnel_nic_info = utn; + } + + list_add_tail(&node->list, &info->shared->devices); + } utn->dev = dev; dev_hold(dev); @@ -783,7 +835,33 @@ static int udp_tunnel_nic_register(struct net_device *dev) static void udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn) { - unsigned int i; + const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; + + /* For a shared table remove this dev from the list of sharing devices + * and if there are other devices just detach. + */ + if (info->shared) { + struct udp_tunnel_nic_shared_node *node, *first; + + list_for_each_entry(node, &info->shared->devices, list) + if (node->dev == dev) + break; + if (node->dev != dev) + return; + + list_del(&node->list); + kfree(node); + + first = list_first_entry_or_null(&info->shared->devices, + typeof(*first), list); + if (first) { + udp_tunnel_drop_rx_info(dev); + utn->dev = first->dev; + goto release_dev; + } + + info->shared->udp_tunnel_nic_info = NULL; + } /* Flush before we check work, so we don't waste time adding entries * from the work which we will boot immediately. @@ -796,10 +874,8 @@ udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn) if (utn->work_pending) return; - for (i = 0; i < utn->n_tables; i++) - kfree(utn->entries[i]); - kfree(utn->entries); - kfree(utn); + udp_tunnel_nic_free(utn); +release_dev: dev->udp_tunnel_nic = NULL; dev_put(dev); } diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 3a57fb9ce049..931b186d2e48 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -707,6 +707,17 @@ static int prepare_ip6gre_xmit_ipv6(struct sk_buff *skb, return 0; } +static struct ip_tunnel_info *skb_tunnel_info_txcheck(struct sk_buff *skb) +{ + struct ip_tunnel_info *tun_info; + + tun_info = skb_tunnel_info(skb); + if (unlikely(!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX))) + return ERR_PTR(-EINVAL); + + return tun_info; +} + static netdev_tx_t __gre6_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield, struct flowi6 *fl6, int encap_limit, @@ -734,10 +745,9 @@ static netdev_tx_t __gre6_xmit(struct sk_buff *skb, const struct ip_tunnel_key *key; __be16 flags; - tun_info = skb_tunnel_info(skb); - if (unlikely(!tun_info || - !(tun_info->mode & IP_TUNNEL_INFO_TX) || - ip_tunnel_info_af(tun_info) != AF_INET6)) + tun_info = skb_tunnel_info_txcheck(skb); + if (IS_ERR(tun_info) || + unlikely(ip_tunnel_info_af(tun_info) != AF_INET6)) return -EINVAL; key = &tun_info->key; @@ -908,7 +918,8 @@ static netdev_tx_t ip6gre_tunnel_xmit(struct sk_buff *skb, return NETDEV_TX_OK; tx_err: - stats->tx_errors++; + if (!t->parms.collect_md || !IS_ERR(skb_tunnel_info_txcheck(skb))) + stats->tx_errors++; stats->tx_dropped++; kfree_skb(skb); return NETDEV_TX_OK; @@ -917,6 +928,7 @@ tx_err: static netdev_tx_t ip6erspan_tunnel_xmit(struct sk_buff *skb, struct net_device *dev) { + struct ip_tunnel_info *tun_info = NULL; struct ip6_tnl *t = netdev_priv(dev); struct dst_entry *dst = skb_dst(skb); struct net_device_stats *stats; @@ -964,15 +976,13 @@ static netdev_tx_t ip6erspan_tunnel_xmit(struct sk_buff *skb, * for native mode, call prepare_ip6gre_xmit_{ipv4,ipv6}. */ if (t->parms.collect_md) { - struct ip_tunnel_info *tun_info; const struct ip_tunnel_key *key; struct erspan_metadata *md; __be32 tun_id; - tun_info = skb_tunnel_info(skb); - if (unlikely(!tun_info || - !(tun_info->mode & IP_TUNNEL_INFO_TX) || - ip_tunnel_info_af(tun_info) != AF_INET6)) + tun_info = skb_tunnel_info_txcheck(skb); + if (IS_ERR(tun_info) || + unlikely(ip_tunnel_info_af(tun_info) != AF_INET6)) goto tx_err; key = &tun_info->key; @@ -1065,7 +1075,8 @@ static netdev_tx_t ip6erspan_tunnel_xmit(struct sk_buff *skb, tx_err: stats = &t->dev->stats; - stats->tx_errors++; + if (!IS_ERR(tun_info)) + stats->tx_errors++; stats->tx_dropped++; kfree_skb(skb); return NETDEV_TX_OK; diff --git a/net/l2tp/l2tp_netlink.c b/net/l2tp/l2tp_netlink.c index 83c015f7f20d..5ca5056e9636 100644 --- a/net/l2tp/l2tp_netlink.c +++ b/net/l2tp/l2tp_netlink.c @@ -420,6 +420,9 @@ static int l2tp_nl_tunnel_send(struct sk_buff *skb, u32 portid, u32 seq, int fla nla_put_u64_64bit(skb, L2TP_ATTR_RX_SEQ_DISCARDS, atomic_long_read(&tunnel->stats.rx_seq_discards), L2TP_ATTR_STATS_PAD) || + nla_put_u64_64bit(skb, L2TP_ATTR_RX_COOKIE_DISCARDS, + atomic_long_read(&tunnel->stats.rx_cookie_discards), + L2TP_ATTR_STATS_PAD) || nla_put_u64_64bit(skb, L2TP_ATTR_RX_OOS_PACKETS, atomic_long_read(&tunnel->stats.rx_oos_packets), L2TP_ATTR_STATS_PAD) || @@ -760,6 +763,9 @@ static int l2tp_nl_session_send(struct sk_buff *skb, u32 portid, u32 seq, int fl nla_put_u64_64bit(skb, L2TP_ATTR_RX_SEQ_DISCARDS, atomic_long_read(&session->stats.rx_seq_discards), L2TP_ATTR_STATS_PAD) || + nla_put_u64_64bit(skb, L2TP_ATTR_RX_COOKIE_DISCARDS, + atomic_long_read(&session->stats.rx_cookie_discards), + L2TP_ATTR_STATS_PAD) || nla_put_u64_64bit(skb, L2TP_ATTR_RX_OOS_PACKETS, atomic_long_read(&session->stats.rx_oos_packets), L2TP_ATTR_STATS_PAD) || diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index 8d75a4045d6e..da70f174d629 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -1124,6 +1124,8 @@ static int ieee80211_start_ap(struct wiphy *wiphy, struct net_device *dev, sizeof(struct ieee80211_he_obss_pd)); memcpy(&sdata->vif.bss_conf.he_bss_color, ¶ms->he_bss_color, sizeof(struct ieee80211_he_bss_color)); + sdata->vif.bss_conf.s1g = params->chandef.chan->band == + NL80211_BAND_S1GHZ; sdata->vif.bss_conf.ssid_len = params->ssid_len; if (params->ssid_len) diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c index bdc0f29dc6cd..8f48aff74c7b 100644 --- a/net/mac80211/chan.c +++ b/net/mac80211/chan.c @@ -536,7 +536,14 @@ static void ieee80211_del_chanctx(struct ieee80211_local *local, if (!local->use_chanctx) { struct cfg80211_chan_def *chandef = &local->_oper_chandef; - chandef->width = NL80211_CHAN_WIDTH_20_NOHT; + /* S1G doesn't have 20MHz, so get the correct width for the + * current channel. + */ + if (chandef->chan->band == NL80211_BAND_S1GHZ) + chandef->width = + ieee80211_s1g_channel_width(chandef->chan); + else + chandef->width = NL80211_CHAN_WIDTH_20_NOHT; chandef->center_freq1 = chandef->chan->center_freq; chandef->freq1_offset = chandef->chan->freq_offset; chandef->center_freq2 = 0; diff --git a/net/mac80211/ibss.c b/net/mac80211/ibss.c index c0963969a465..1f552f374e97 100644 --- a/net/mac80211/ibss.c +++ b/net/mac80211/ibss.c @@ -1037,7 +1037,8 @@ static void ieee80211_update_sta_info(struct ieee80211_sub_if_data *sdata, } if (sta && !sta->sta.wme && - elems->wmm_info && local->hw.queues >= IEEE80211_NUM_ACS) { + (elems->wmm_info || elems->s1g_capab) && + local->hw.queues >= IEEE80211_NUM_ACS) { sta->sta.wme = true; ieee80211_check_fast_xmit(sta); } diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index ecd9229012bf..c3e3578574a6 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -544,6 +544,8 @@ struct ieee80211_if_managed { struct ieee80211_ht_cap ht_capa_mask; /* Valid parts of ht_capa */ struct ieee80211_vht_cap vht_capa; /* configured VHT overrides */ struct ieee80211_vht_cap vht_capa_mask; /* Valid parts of vht_capa */ + struct ieee80211_s1g_cap s1g_capa; /* configured S1G overrides */ + struct ieee80211_s1g_cap s1g_capa_mask; /* valid s1g_capa bits */ /* TDLS support */ u8 tdls_peer[ETH_ALEN] __aligned(2); @@ -1376,7 +1378,6 @@ struct ieee80211_local { */ bool pspolling; - bool offchannel_ps_enabled; /* * PS can only be enabled when we have exactly one managed * interface (and monitors) in PS, this then points there. @@ -1534,6 +1535,10 @@ struct ieee802_11_elems { u8 dtim_count; u8 dtim_period; const struct ieee80211_addba_ext_ie *addba_ext_ie; + const struct ieee80211_s1g_cap *s1g_capab; + const struct ieee80211_s1g_oper_ie *s1g_oper; + const struct ieee80211_s1g_bcn_compat_ie *s1g_bcn_compat; + const struct ieee80211_aid_response_ie *aid_resp; /* length of them, respectively */ u8 ext_capab_len; @@ -1652,6 +1657,8 @@ int ieee80211_set_arp_filter(struct ieee80211_sub_if_data *sdata); void ieee80211_sta_work(struct ieee80211_sub_if_data *sdata); void ieee80211_sta_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb); +void ieee80211_sta_rx_queued_ext(struct ieee80211_sub_if_data *sdata, + struct sk_buff *skb); void ieee80211_sta_reset_beacon_monitor(struct ieee80211_sub_if_data *sdata); void ieee80211_sta_reset_conn_monitor(struct ieee80211_sub_if_data *sdata); void ieee80211_mgd_stop(struct ieee80211_sub_if_data *sdata); @@ -2204,6 +2211,11 @@ int ieee80211_add_ext_srates_ie(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb, bool need_basic, enum nl80211_band band); u8 *ieee80211_add_wmm_info_ie(u8 *buf, u8 qosinfo); +void ieee80211_add_s1g_capab_ie(struct ieee80211_sub_if_data *sdata, + struct ieee80211_sta_s1g_cap *caps, + struct sk_buff *skb); +void ieee80211_add_aid_request_ie(struct ieee80211_sub_if_data *sdata, + struct sk_buff *skb); /* channel management */ bool ieee80211_chandef_ht_oper(const struct ieee80211_ht_operation *ht_oper, @@ -2215,6 +2227,8 @@ bool ieee80211_chandef_vht_oper(struct ieee80211_hw *hw, u32 vht_cap_info, bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_sub_if_data *sdata, const struct ieee80211_he_operation *he_oper, struct cfg80211_chan_def *chandef); +bool ieee80211_chandef_s1g_oper(const struct ieee80211_s1g_oper_ie *oper, + struct cfg80211_chan_def *chandef); u32 ieee80211_chandef_downgrade(struct cfg80211_chan_def *c); int __must_check @@ -2293,6 +2307,9 @@ void ieee80211_tdls_chsw_work(struct work_struct *wk); void ieee80211_tdls_handle_disconnect(struct ieee80211_sub_if_data *sdata, const u8 *peer, u16 reason); const char *ieee80211_get_reason_code_string(u16 reason_code); +u16 ieee80211_encode_usf(int val); +u8 *ieee80211_get_bssid(struct ieee80211_hdr *hdr, size_t len, + enum nl80211_iftype type); extern const struct ethtool_ops ieee80211_ethtool_ops; diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index 7ac9af66f545..240862a74a0f 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -1433,6 +1433,11 @@ static void ieee80211_iface_work(struct work_struct *work) WARN_ON(1); break; } + } else if (ieee80211_is_ext(mgmt->frame_control)) { + if (sdata->vif.type == NL80211_IFTYPE_STATION) + ieee80211_sta_rx_queued_ext(sdata, skb); + else + WARN_ON(1); } else if (ieee80211_is_data_qos(mgmt->frame_control)) { struct ieee80211_hdr *hdr = (void *)mgmt; /* diff --git a/net/mac80211/mesh_plink.c b/net/mac80211/mesh_plink.c index 15f2fc658f70..aca26df7587d 100644 --- a/net/mac80211/mesh_plink.c +++ b/net/mac80211/mesh_plink.c @@ -144,6 +144,7 @@ out: /** * mesh_set_ht_prot_mode - set correct HT protection mode + * @sdata: the (mesh) interface to handle * * Section 9.23.3.5 of IEEE 80211-2012 describes the protection rules for HT * mesh STA in a MBSS. Three HT protection modes are supported for now, non-HT diff --git a/net/mac80211/mesh_ps.c b/net/mac80211/mesh_ps.c index 76d19c09d26e..204830a55240 100644 --- a/net/mac80211/mesh_ps.c +++ b/net/mac80211/mesh_ps.c @@ -12,6 +12,7 @@ /** * mps_qos_null_get - create pre-addressed QoS Null frame for mesh powersave + * @sta: the station to get the frame for */ static struct sk_buff *mps_qos_null_get(struct sta_info *sta) { @@ -44,6 +45,7 @@ static struct sk_buff *mps_qos_null_get(struct sta_info *sta) /** * mps_qos_null_tx - send a QoS Null to indicate link-specific power mode + * @sta: the station to send to */ static void mps_qos_null_tx(struct sta_info *sta) { @@ -400,6 +402,8 @@ static void mpsp_trigger_send(struct sta_info *sta, bool rspi, bool eosp) /** * mpsp_qos_null_append - append QoS Null frame to MPSP skb queue if needed + * @sta: the station to handle + * @frames: the frame list to append to * * To properly end a mesh MPSP the last transmitted frame has to set the EOSP * flag in the QoS Control field. In case the current tailing frame is not a diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 2489c6c64c2d..e9a8e8e94ee6 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -149,6 +149,7 @@ ieee80211_determine_chantype(struct ieee80211_sub_if_data *sdata, const struct ieee80211_ht_operation *ht_oper, const struct ieee80211_vht_operation *vht_oper, const struct ieee80211_he_operation *he_oper, + const struct ieee80211_s1g_oper_ie *s1g_oper, struct cfg80211_chan_def *chandef, bool tracking) { struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; @@ -171,6 +172,18 @@ ieee80211_determine_chantype(struct ieee80211_sub_if_data *sdata, ret = 0; vht_chandef = *chandef; goto out; + } else if (sband->band == NL80211_BAND_S1GHZ) { + if (!ieee80211_chandef_s1g_oper(s1g_oper, chandef)) { + sdata_info(sdata, + "Missing S1G Operation Element? Trying operating == primary\n"); + chandef->width = ieee80211_s1g_channel_width(channel); + } + + ret = IEEE80211_STA_DISABLE_HT | IEEE80211_STA_DISABLE_40MHZ | + IEEE80211_STA_DISABLE_VHT | + IEEE80211_STA_DISABLE_80P80MHZ | + IEEE80211_STA_DISABLE_160MHZ; + goto out; } memcpy(&sta_ht_cap, &sband->ht_cap, sizeof(sta_ht_cap)); @@ -347,6 +360,7 @@ static int ieee80211_config_bw(struct ieee80211_sub_if_data *sdata, const struct ieee80211_ht_operation *ht_oper, const struct ieee80211_vht_operation *vht_oper, const struct ieee80211_he_operation *he_oper, + const struct ieee80211_s1g_oper_ie *s1g_oper, const u8 *bssid, u32 *changed) { struct ieee80211_local *local = sdata->local; @@ -393,7 +407,7 @@ static int ieee80211_config_bw(struct ieee80211_sub_if_data *sdata, /* calculate new channel (type) based on HT/VHT/HE operation IEs */ flags = ieee80211_determine_chantype(sdata, sband, chan, vht_cap_info, ht_oper, vht_oper, he_oper, - &chandef, true); + s1g_oper, &chandef, true); /* * Downgrade the new channel if we associated with restricted @@ -696,6 +710,7 @@ static void ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata) struct ieee80211_chanctx_conf *chanctx_conf; struct ieee80211_channel *chan; u32 rates = 0; + __le16 listen_int; struct element *ext_capa = NULL; /* we know it's writable, cast away the const */ @@ -784,13 +799,15 @@ static void ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata) memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN); memcpy(mgmt->bssid, assoc_data->bss->bssid, ETH_ALEN); + listen_int = cpu_to_le16(sband->band == NL80211_BAND_S1GHZ ? + ieee80211_encode_usf(local->hw.conf.listen_interval) : + local->hw.conf.listen_interval); if (!is_zero_ether_addr(assoc_data->prev_bssid)) { skb_put(skb, 10); mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_REASSOC_REQ); mgmt->u.reassoc_req.capab_info = cpu_to_le16(capab); - mgmt->u.reassoc_req.listen_interval = - cpu_to_le16(local->hw.conf.listen_interval); + mgmt->u.reassoc_req.listen_interval = listen_int; memcpy(mgmt->u.reassoc_req.current_ap, assoc_data->prev_bssid, ETH_ALEN); } else { @@ -798,8 +815,7 @@ static void ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata) mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_ASSOC_REQ); mgmt->u.assoc_req.capab_info = cpu_to_le16(capab); - mgmt->u.assoc_req.listen_interval = - cpu_to_le16(local->hw.conf.listen_interval); + mgmt->u.assoc_req.listen_interval = listen_int; } /* SSID */ @@ -809,6 +825,9 @@ static void ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata) *pos++ = assoc_data->ssid_len; memcpy(pos, assoc_data->ssid, assoc_data->ssid_len); + if (sband->band == NL80211_BAND_S1GHZ) + goto skip_rates; + /* add all rates which were marked to be used above */ supp_rates_len = rates_len; if (supp_rates_len > 8) @@ -844,6 +863,7 @@ static void ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata) } } +skip_rates: if (capab & WLAN_CAPABILITY_SPECTRUM_MGMT || capab & WLAN_CAPABILITY_RADIO_MEASURE) { pos = skb_put(skb, 4); @@ -1018,6 +1038,11 @@ static void ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata) pos = ieee80211_add_wmm_info_ie(skb_put(skb, 9), qos_info); } + if (sband->band == NL80211_BAND_S1GHZ) { + ieee80211_add_aid_request_ie(sdata, skb); + ieee80211_add_s1g_capab_ie(sdata, &sband->s1g_cap, skb); + } + /* add any remaining custom (i.e. vendor specific here) IEs */ if (assoc_data->ie_len) { noffset = assoc_data->ie_len; @@ -1597,6 +1622,9 @@ static u32 ieee80211_handle_pwr_constr(struct ieee80211_sub_if_data *sdata, int new_ap_level; __le16 capab = mgmt->u.probe_resp.capab_info; + if (ieee80211_is_s1g_beacon(mgmt->frame_control)) + return 0; /* TODO */ + if (country_ie && (capab & cpu_to_le16(WLAN_CAPABILITY_SPECTRUM_MGMT) || capab & cpu_to_le16(WLAN_CAPABILITY_RADIO_MEASURE))) { @@ -2508,7 +2536,9 @@ void ieee80211_sta_tx_notify(struct ieee80211_sub_if_data *sdata, !sdata->u.mgd.probe_send_count) return; - if (!ack) + if (ack) + sdata->u.mgd.probe_send_count = 0; + else sdata->u.mgd.nullfunc_failed = true; ieee80211_queue_work(&sdata->local->hw, &sdata->work); } @@ -3242,14 +3272,26 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata, const struct cfg80211_bss_ies *bss_ies = NULL; struct ieee80211_mgd_assoc_data *assoc_data = ifmgd->assoc_data; bool is_6ghz = cbss->channel->band == NL80211_BAND_6GHZ; + bool is_s1g = cbss->channel->band == NL80211_BAND_S1GHZ; u32 changed = 0; + u8 *pos; int err; bool ret; /* AssocResp and ReassocResp have identical structure */ + pos = mgmt->u.assoc_resp.variable; aid = le16_to_cpu(mgmt->u.assoc_resp.aid); + if (is_s1g) { + pos = (u8 *) mgmt->u.s1g_assoc_resp.variable; + aid = 0; /* TODO */ + } capab_info = le16_to_cpu(mgmt->u.assoc_resp.capab_info); + ieee802_11_parse_elems(pos, len - (pos - (u8 *)mgmt), false, elems, + mgmt->bssid, assoc_data->bss->bssid); + + if (elems->aid_resp) + aid = le16_to_cpu(elems->aid_resp->aid); /* * The 5 MSB of the AID field are reserved @@ -3266,7 +3308,7 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata, ifmgd->broken_ap = true; } - if (!elems->supp_rates) { + if (!is_s1g && !elems->supp_rates) { sdata_info(sdata, "no SuppRates element in AssocResp\n"); return false; } @@ -3508,7 +3550,8 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata, sta->sta.mfp = false; } - sta->sta.wme = elems->wmm_param && local->hw.queues >= IEEE80211_NUM_ACS; + sta->sta.wme = (elems->wmm_param || elems->s1g_capab) && + local->hw.queues >= IEEE80211_NUM_ACS; err = sta_info_move_state(sta, IEEE80211_STA_ASSOC); if (!err && !(ifmgd->flags & IEEE80211_STA_CONTROL_PORT)) @@ -3603,7 +3646,7 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, int ac, uapsd_queues = -1; u8 *pos; bool reassoc; - struct cfg80211_bss *bss; + struct cfg80211_bss *cbss; struct ieee80211_event event = { .type = MLME_EVENT, .u.mlme.data = ASSOC_EVENT, @@ -3613,9 +3656,12 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, if (!assoc_data) return; + if (!ether_addr_equal(assoc_data->bss->bssid, mgmt->bssid)) return; + cbss = assoc_data->bss; + /* * AssocResp and ReassocResp have identical structure, so process both * of them in this function. @@ -3627,7 +3673,12 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, reassoc = ieee80211_is_reassoc_resp(mgmt->frame_control); capab_info = le16_to_cpu(mgmt->u.assoc_resp.capab_info); status_code = le16_to_cpu(mgmt->u.assoc_resp.status_code); + pos = mgmt->u.assoc_resp.variable; aid = le16_to_cpu(mgmt->u.assoc_resp.aid); + if (cbss->channel->band == NL80211_BAND_S1GHZ) { + pos = (u8 *) mgmt->u.s1g_assoc_resp.variable; + aid = 0; /* TODO */ + } sdata_info(sdata, "RX %sssocResp from %pM (capab=0x%x status=%d aid=%d)\n", @@ -3638,7 +3689,6 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, fils_decrypt_assoc_resp(sdata, (u8 *)mgmt, &len, assoc_data) < 0) return; - pos = mgmt->u.assoc_resp.variable; ieee802_11_parse_elems(pos, len - (pos - (u8 *)mgmt), false, &elems, mgmt->bssid, assoc_data->bss->bssid); @@ -3658,8 +3708,6 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, return; } - bss = assoc_data->bss; - if (status_code != WLAN_STATUS_SUCCESS) { sdata_info(sdata, "%pM denied association (code=%d)\n", mgmt->sa, status_code); @@ -3668,10 +3716,10 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, event.u.mlme.reason = status_code; drv_event_callback(sdata->local, sdata, &event); } else { - if (!ieee80211_assoc_success(sdata, bss, mgmt, len, &elems)) { + if (!ieee80211_assoc_success(sdata, cbss, mgmt, len, &elems)) { /* oops -- internal error -- send timeout for now */ ieee80211_destroy_assoc_data(sdata, false, false); - cfg80211_assoc_timeout(sdata->dev, bss); + cfg80211_assoc_timeout(sdata->dev, cbss); return; } event.u.mlme.status = MLME_SUCCESS; @@ -3692,7 +3740,7 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, uapsd_queues |= ieee80211_ac_to_qos_mask[ac]; } - cfg80211_rx_assoc_resp(sdata->dev, bss, (u8 *)mgmt, len, uapsd_queues, + cfg80211_rx_assoc_resp(sdata->dev, cbss, (u8 *)mgmt, len, uapsd_queues, ifmgd->assoc_req_ies, ifmgd->assoc_req_ies_len); } @@ -3891,11 +3939,12 @@ static bool ieee80211_rx_our_beacon(const u8 *tx_bssid, } static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, - struct ieee80211_mgmt *mgmt, size_t len, + struct ieee80211_hdr *hdr, size_t len, struct ieee80211_rx_status *rx_status) { struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; struct ieee80211_bss_conf *bss_conf = &sdata->vif.bss_conf; + struct ieee80211_mgmt *mgmt = (void *) hdr; size_t baselen; struct ieee802_11_elems elems; struct ieee80211_local *local = sdata->local; @@ -3905,14 +3954,24 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, u32 changed = 0; bool erp_valid; u8 erp_value = 0; - u32 ncrc; - u8 *bssid; + u32 ncrc = 0; + u8 *bssid, *variable = mgmt->u.beacon.variable; u8 deauth_buf[IEEE80211_DEAUTH_FRAME_LEN]; sdata_assert_lock(sdata); /* Process beacon from the current BSS */ - baselen = (u8 *) mgmt->u.beacon.variable - (u8 *) mgmt; + bssid = ieee80211_get_bssid(hdr, len, sdata->vif.type); + if (ieee80211_is_s1g_beacon(mgmt->frame_control)) { + struct ieee80211_ext *ext = (void *) mgmt; + + if (ieee80211_is_s1g_short_beacon(ext->frame_control)) + variable = ext->u.s1g_short_beacon.variable; + else + variable = ext->u.s1g_beacon.variable; + } + + baselen = (u8 *) variable - (u8 *) mgmt; if (baselen > len) return; @@ -3932,10 +3991,10 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, rcu_read_unlock(); if (ifmgd->assoc_data && ifmgd->assoc_data->need_beacon && - ieee80211_rx_our_beacon(mgmt->bssid, ifmgd->assoc_data->bss)) { - ieee802_11_parse_elems(mgmt->u.beacon.variable, + ieee80211_rx_our_beacon(bssid, ifmgd->assoc_data->bss)) { + ieee802_11_parse_elems(variable, len - baselen, false, &elems, - mgmt->bssid, + bssid, ifmgd->assoc_data->bss->bssid); ieee80211_rx_bss_info(sdata, mgmt, len, rx_status); @@ -3968,7 +4027,7 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, } if (!ifmgd->associated || - !ieee80211_rx_our_beacon(mgmt->bssid, ifmgd->associated)) + !ieee80211_rx_our_beacon(bssid, ifmgd->associated)) return; bssid = ifmgd->associated->bssid; @@ -3988,8 +4047,14 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, */ ieee80211_sta_reset_beacon_monitor(sdata); - ncrc = crc32_be(0, (void *)&mgmt->u.beacon.beacon_int, 4); - ncrc = ieee802_11_parse_elems_crc(mgmt->u.beacon.variable, + /* TODO: CRC urrently not calculated on S1G Beacon Compatibility + * element (which carries the beacon interval). Don't forget to add a + * bit to care_about_ies[] above if mac80211 is interested in a + * changing S1G element. + */ + if (!ieee80211_is_s1g_beacon(hdr->frame_control)) + ncrc = crc32_be(0, (void *)&mgmt->u.beacon.beacon_int, 4); + ncrc = ieee802_11_parse_elems_crc(variable, len - baselen, false, &elems, care_about_ies, ncrc, mgmt->bssid, bssid); @@ -4023,7 +4088,7 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, struct ieee80211_p2p_noa_attr noa = {}; int ret; - ret = cfg80211_get_p2p_attr(mgmt->u.beacon.variable, + ret = cfg80211_get_p2p_attr(variable, len - baselen, IEEE80211_P2P_ATTR_ABSENCE_NOTICE, (u8 *) &noa, sizeof(noa)); @@ -4059,7 +4124,8 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, * the driver will use them. The synchronized view is currently * guaranteed only in certain callbacks. */ - if (ieee80211_hw_check(&local->hw, TIMING_BEACON_ONLY)) { + if (ieee80211_hw_check(&local->hw, TIMING_BEACON_ONLY) && + !ieee80211_is_s1g_beacon(hdr->frame_control)) { sdata->vif.bss_conf.sync_tsf = le64_to_cpu(mgmt->u.beacon.timestamp); sdata->vif.bss_conf.sync_device_ts = @@ -4067,7 +4133,8 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, sdata->vif.bss_conf.sync_dtim_count = elems.dtim_count; } - if (ncrc == ifmgd->beacon_crc && ifmgd->beacon_crc_valid) + if ((ncrc == ifmgd->beacon_crc && ifmgd->beacon_crc_valid) || + ieee80211_is_s1g_short_beacon(mgmt->frame_control)) return; ifmgd->beacon_crc = ncrc; ifmgd->beacon_crc_valid = true; @@ -4108,9 +4175,11 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, } else { erp_valid = false; } - changed |= ieee80211_handle_bss_capability(sdata, - le16_to_cpu(mgmt->u.beacon.capab_info), - erp_valid, erp_value); + + if (!ieee80211_is_s1g_beacon(hdr->frame_control)) + changed |= ieee80211_handle_bss_capability(sdata, + le16_to_cpu(mgmt->u.beacon.capab_info), + erp_valid, erp_value); mutex_lock(&local->sta_mtx); sta = sta_info_get(sdata, bssid); @@ -4120,7 +4189,7 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, if (ieee80211_config_bw(sdata, sta, elems.ht_cap_elem, elems.vht_cap_elem, elems.ht_operation, elems.vht_operation, elems.he_operation, - bssid, &changed)) { + elems.s1g_oper, bssid, &changed)) { mutex_unlock(&local->sta_mtx); sdata_info(sdata, "failed to follow AP %pM bandwidth change, disconnect\n", @@ -4148,6 +4217,26 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_sub_if_data *sdata, ieee80211_bss_info_change_notify(sdata, changed); } +void ieee80211_sta_rx_queued_ext(struct ieee80211_sub_if_data *sdata, + struct sk_buff *skb) +{ + struct ieee80211_rx_status *rx_status; + struct ieee80211_hdr *hdr; + u16 fc; + + rx_status = (struct ieee80211_rx_status *) skb->cb; + hdr = (struct ieee80211_hdr *) skb->data; + fc = le16_to_cpu(hdr->frame_control); + + sdata_lock(sdata); + switch (fc & IEEE80211_FCTL_STYPE) { + case IEEE80211_STYPE_S1G_BEACON: + ieee80211_rx_mgmt_beacon(sdata, hdr, skb->len, rx_status); + break; + } + sdata_unlock(sdata); +} + void ieee80211_sta_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb) { @@ -4165,7 +4254,8 @@ void ieee80211_sta_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata, switch (fc & IEEE80211_FCTL_STYPE) { case IEEE80211_STYPE_BEACON: - ieee80211_rx_mgmt_beacon(sdata, mgmt, skb->len, rx_status); + ieee80211_rx_mgmt_beacon(sdata, (void *)mgmt, + skb->len, rx_status); break; case IEEE80211_STYPE_PROBE_RESP: ieee80211_rx_mgmt_probe_resp(sdata, skb); @@ -4852,6 +4942,7 @@ static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata, const struct ieee80211_ht_operation *ht_oper = NULL; const struct ieee80211_vht_operation *vht_oper = NULL; const struct ieee80211_he_operation *he_oper = NULL; + const struct ieee80211_s1g_oper_ie *s1g_oper = NULL; struct ieee80211_supported_band *sband; struct cfg80211_chan_def chandef; bool is_6ghz = cbss->channel->band == NL80211_BAND_6GHZ; @@ -4955,10 +5046,23 @@ static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata, if (!have_80mhz) ifmgd->flags |= IEEE80211_STA_DISABLE_VHT; + if (sband->band == NL80211_BAND_S1GHZ) { + const u8 *s1g_oper_ie; + + s1g_oper_ie = ieee80211_bss_get_ie(cbss, + WLAN_EID_S1G_OPERATION); + if (s1g_oper_ie && s1g_oper_ie[1] >= sizeof(*s1g_oper)) + s1g_oper = (void *)(s1g_oper_ie + 2); + else + sdata_info(sdata, + "AP missing S1G operation element?\n"); + } + ifmgd->flags |= ieee80211_determine_chantype(sdata, sband, cbss->channel, bss->vht_cap_info, ht_oper, vht_oper, he_oper, + s1g_oper, &chandef, false); sdata->needed_rx_chains = min(ieee80211_ht_vht_rx_chains(sdata, cbss), @@ -5085,6 +5189,10 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata, const struct cfg80211_bss_ies *ies; int shift = ieee80211_vif_get_shift(&sdata->vif); + /* TODO: S1G Basic Rate Set is expressed elsewhere */ + if (cbss->channel->band == NL80211_BAND_S1GHZ) + goto skip_rates; + ieee80211_get_rates(sband, bss->supp_rates, bss->supp_rates_len, &rates, &basic_rates, @@ -5129,6 +5237,7 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata, else sdata->flags &= ~IEEE80211_SDATA_OPERATING_GMODE; +skip_rates: memcpy(ifmgd->bssid, cbss->bssid, ETH_ALEN); /* set timing information */ @@ -5456,6 +5565,10 @@ int ieee80211_mgd_assoc(struct ieee80211_sub_if_data *sdata, memcpy(&ifmgd->vht_capa_mask, &req->vht_capa_mask, sizeof(ifmgd->vht_capa_mask)); + memcpy(&ifmgd->s1g_capa, &req->s1g_capa, sizeof(ifmgd->s1g_capa)); + memcpy(&ifmgd->s1g_capa_mask, &req->s1g_capa_mask, + sizeof(ifmgd->s1g_capa_mask)); + if (req->ie && req->ie_len) { memcpy(assoc_data->ie, req->ie, req->ie_len); assoc_data->ie_len = req->ie_len; diff --git a/net/mac80211/offchannel.c b/net/mac80211/offchannel.c index 1ac7b8c374c9..853c9a369d72 100644 --- a/net/mac80211/offchannel.c +++ b/net/mac80211/offchannel.c @@ -26,8 +26,7 @@ static void ieee80211_offchannel_ps_enable(struct ieee80211_sub_if_data *sdata) { struct ieee80211_local *local = sdata->local; struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; - - local->offchannel_ps_enabled = false; + bool offchannel_ps_enabled = false; /* FIXME: what to do when local->pspolling is true? */ @@ -38,12 +37,12 @@ static void ieee80211_offchannel_ps_enable(struct ieee80211_sub_if_data *sdata) cancel_work_sync(&local->dynamic_ps_enable_work); if (local->hw.conf.flags & IEEE80211_CONF_PS) { - local->offchannel_ps_enabled = true; + offchannel_ps_enabled = true; local->hw.conf.flags &= ~IEEE80211_CONF_PS; ieee80211_hw_config(local, IEEE80211_CONF_CHANGE_PS); } - if (!local->offchannel_ps_enabled || + if (!offchannel_ps_enabled || !ieee80211_hw_check(&local->hw, PS_NULLFUNC_STACK)) /* * If power save was enabled, no need to send a nullfunc @@ -58,38 +57,19 @@ static void ieee80211_offchannel_ps_enable(struct ieee80211_sub_if_data *sdata) ieee80211_send_nullfunc(local, sdata, true); } -/* inform AP that we are awake again, unless power save is enabled */ +/* inform AP that we are awake again */ static void ieee80211_offchannel_ps_disable(struct ieee80211_sub_if_data *sdata) { struct ieee80211_local *local = sdata->local; if (!local->ps_sdata) ieee80211_send_nullfunc(local, sdata, false); - else if (local->offchannel_ps_enabled) { - /* - * In !IEEE80211_HW_PS_NULLFUNC_STACK case the hardware - * will send a nullfunc frame with the powersave bit set - * even though the AP already knows that we are sleeping. - * This could be avoided by sending a null frame with power - * save bit disabled before enabling the power save, but - * this doesn't gain anything. - * - * When IEEE80211_HW_PS_NULLFUNC_STACK is enabled, no need - * to send a nullfunc frame because AP already knows that - * we are sleeping, let's just enable power save mode in - * hardware. - */ - /* TODO: Only set hardware if CONF_PS changed? - * TODO: Should we set offchannel_ps_enabled to false? - */ - local->hw.conf.flags |= IEEE80211_CONF_PS; - ieee80211_hw_config(local, IEEE80211_CONF_CHANGE_PS); - } else if (local->hw.conf.dynamic_ps_timeout > 0) { + else if (local->hw.conf.dynamic_ps_timeout > 0) { /* - * If IEEE80211_CONF_PS was not set and the dynamic_ps_timer - * had been running before leaving the operating channel, - * restart the timer now and send a nullfunc frame to inform - * the AP that we are awake. + * the dynamic_ps_timer had been running before leaving the + * operating channel, restart the timer now and send a nullfunc + * frame to inform the AP that we are awake so that AP sends + * the buffered packets (if any). */ ieee80211_send_nullfunc(local, sdata, false); mod_timer(&local->dynamic_ps_timer, jiffies + diff --git a/net/mac80211/rate.c b/net/mac80211/rate.c index b051f125d3af..0cba7fed28cf 100644 --- a/net/mac80211/rate.c +++ b/net/mac80211/rate.c @@ -51,6 +51,12 @@ void rate_control_rate_init(struct sta_info *sta) sband = local->hw.wiphy->bands[chanctx_conf->def.chan->band]; + /* TODO: check for minstrel_s1g ? */ + if (sband->band == NL80211_BAND_S1GHZ) { + rcu_read_unlock(); + return; + } + spin_lock_bh(&sta->rate_ctrl_lock); ref->ops->rate_init(ref->priv, sband, &chanctx_conf->def, ista, priv_sta); @@ -266,10 +272,15 @@ void ieee80211_check_rate_mask(struct ieee80211_sub_if_data *sdata) if (WARN_ON(!sdata->vif.bss_conf.chandef.chan)) return; + band = sdata->vif.bss_conf.chandef.chan->band; + if (band == NL80211_BAND_S1GHZ) { + /* TODO */ + return; + } + if (WARN_ON_ONCE(!basic_rates)) return; - band = sdata->vif.bss_conf.chandef.chan->band; user_mask = sdata->rc_rateidx_mask[band]; sband = local->hw.wiphy->bands[band]; @@ -296,21 +307,29 @@ static bool rc_no_data_or_no_ack_use_min(struct ieee80211_tx_rate_control *txrc) !ieee80211_is_data(fc); } -static void rc_send_low_basicrate(s8 *idx, u32 basic_rates, +static void rc_send_low_basicrate(struct ieee80211_tx_rate *rate, + u32 basic_rates, struct ieee80211_supported_band *sband) { u8 i; + if (sband->band == NL80211_BAND_S1GHZ) { + /* TODO */ + rate->flags |= IEEE80211_TX_RC_S1G_MCS; + rate->idx = 0; + return; + } + if (basic_rates == 0) return; /* assume basic rates unknown and accept rate */ - if (*idx < 0) + if (rate->idx < 0) return; - if (basic_rates & (1 << *idx)) + if (basic_rates & (1 << rate->idx)) return; /* selected rate is a basic rate */ - for (i = *idx + 1; i <= sband->n_bitrates; i++) { + for (i = rate->idx + 1; i <= sband->n_bitrates; i++) { if (basic_rates & (1 << i)) { - *idx = i; + rate->idx = i; return; } } @@ -328,6 +347,12 @@ static void __rate_control_send_low(struct ieee80211_hw *hw, u32 rate_flags = ieee80211_chandef_rate_flags(&hw->conf.chandef); + if (sband->band == NL80211_BAND_S1GHZ) { + info->control.rates[0].flags |= IEEE80211_TX_RC_S1G_MCS; + info->control.rates[0].idx = 0; + return; + } + if ((sband->band == NL80211_BAND_2GHZ) && (info->flags & IEEE80211_TX_CTL_NO_CCK_RATE)) rate_flags |= IEEE80211_RATE_ERP_G; @@ -388,7 +413,7 @@ static bool rate_control_send_low(struct ieee80211_sta *pubsta, } if (use_basicrate) - rc_send_low_basicrate(&info->control.rates[0].idx, + rc_send_low_basicrate(&info->control.rates[0], txrc->bss_conf->basic_rates, sband); diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 7f88a1b2215c..1e2e5a406d58 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -42,51 +42,6 @@ static inline void ieee80211_rx_stats(struct net_device *dev, u32 len) u64_stats_update_end(&tstats->syncp); } -static u8 *ieee80211_get_bssid(struct ieee80211_hdr *hdr, size_t len, - enum nl80211_iftype type) -{ - __le16 fc = hdr->frame_control; - - if (ieee80211_is_data(fc)) { - if (len < 24) /* drop incorrect hdr len (data) */ - return NULL; - - if (ieee80211_has_a4(fc)) - return NULL; - if (ieee80211_has_tods(fc)) - return hdr->addr1; - if (ieee80211_has_fromds(fc)) - return hdr->addr2; - - return hdr->addr3; - } - - if (ieee80211_is_mgmt(fc)) { - if (len < 24) /* drop incorrect hdr len (mgmt) */ - return NULL; - return hdr->addr3; - } - - if (ieee80211_is_ctl(fc)) { - if (ieee80211_is_pspoll(fc)) - return hdr->addr1; - - if (ieee80211_is_back_req(fc)) { - switch (type) { - case NL80211_IFTYPE_STATION: - return hdr->addr2; - case NL80211_IFTYPE_AP: - case NL80211_IFTYPE_AP_VLAN: - return hdr->addr1; - default: - break; /* fall through to the return */ - } - } - } - - return NULL; -} - /* * monitor mode reception * @@ -1802,7 +1757,8 @@ ieee80211_rx_h_sta_process(struct ieee80211_rx_data *rx) } } else if (rx->sdata->vif.type == NL80211_IFTYPE_OCB) { sta->rx_stats.last_rx = jiffies; - } else if (!is_multicast_ether_addr(hdr->addr1)) { + } else if (!ieee80211_is_s1g_beacon(hdr->frame_control) && + is_multicast_ether_addr(hdr->addr1)) { /* * Mesh beacons will update last_rx when if they are found to * match the current local configuration when processed. @@ -1837,6 +1793,9 @@ ieee80211_rx_h_sta_process(struct ieee80211_rx_data *rx) } } + if (ieee80211_is_s1g_beacon(hdr->frame_control)) + return RX_CONTINUE; + /* * Change STA power saving mode only at the end of a frame * exchange sequence, and only for a data or management @@ -1947,6 +1906,9 @@ ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx) __le16 fc; const struct ieee80211_cipher_scheme *cs = NULL; + if (ieee80211_is_ext(hdr->frame_control)) + return RX_CONTINUE; + /* * Key selection 101 * @@ -2255,7 +2217,7 @@ ieee80211_rx_h_defragment(struct ieee80211_rx_data *rx) hdr = (struct ieee80211_hdr *)rx->skb->data; fc = hdr->frame_control; - if (ieee80211_is_ctl(fc)) + if (ieee80211_is_ctl(fc) || ieee80211_is_ext(fc)) return RX_CONTINUE; sc = le16_to_cpu(hdr->seq_ctrl); @@ -3129,6 +3091,9 @@ ieee80211_rx_h_mgmt_check(struct ieee80211_rx_data *rx) struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) rx->skb->data; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb); + if (ieee80211_is_s1g_beacon(mgmt->frame_control)) + return RX_CONTINUE; + /* * From here on, look only at management frames. * Data and control frames are already handled, @@ -3596,6 +3561,27 @@ ieee80211_rx_h_action_return(struct ieee80211_rx_data *rx) } static ieee80211_rx_result debug_noinline +ieee80211_rx_h_ext(struct ieee80211_rx_data *rx) +{ + struct ieee80211_sub_if_data *sdata = rx->sdata; + struct ieee80211_hdr *hdr = (void *)rx->skb->data; + + if (!ieee80211_is_ext(hdr->frame_control)) + return RX_CONTINUE; + + if (sdata->vif.type != NL80211_IFTYPE_STATION) + return RX_DROP_MONITOR; + + /* for now only beacons are ext, so queue them */ + skb_queue_tail(&sdata->skb_queue, rx->skb); + ieee80211_queue_work(&rx->local->hw, &sdata->work); + if (rx->sta) + rx->sta->rx_stats.packets++; + + return RX_QUEUED; +} + +static ieee80211_rx_result debug_noinline ieee80211_rx_h_mgmt(struct ieee80211_rx_data *rx) { struct ieee80211_sub_if_data *sdata = rx->sdata; @@ -3814,6 +3800,7 @@ static void ieee80211_rx_handlers(struct ieee80211_rx_data *rx, CALL_RXH(ieee80211_rx_h_userspace_mgmt); CALL_RXH(ieee80211_rx_h_action_post_userspace); CALL_RXH(ieee80211_rx_h_action_return); + CALL_RXH(ieee80211_rx_h_ext); CALL_RXH(ieee80211_rx_h_mgmt); rxh_next: @@ -3980,7 +3967,8 @@ static bool ieee80211_accept_frame(struct ieee80211_rx_data *rx) struct ieee80211_hdr *hdr = (void *)skb->data; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); u8 *bssid = ieee80211_get_bssid(hdr, skb->len, sdata->vif.type); - bool multicast = is_multicast_ether_addr(hdr->addr1); + bool multicast = is_multicast_ether_addr(hdr->addr1) || + ieee80211_is_s1g_beacon(hdr->frame_control); switch (sdata->vif.type) { case NL80211_IFTYPE_STATION: @@ -4578,7 +4566,8 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw, ieee80211_verify_alignment(&rx); if (unlikely(ieee80211_is_probe_resp(hdr->frame_control) || - ieee80211_is_beacon(hdr->frame_control))) + ieee80211_is_beacon(hdr->frame_control) || + ieee80211_is_s1g_beacon(hdr->frame_control))) ieee80211_scan_rx(local, skb); if (ieee80211_is_data(fc)) { diff --git a/net/mac80211/scan.c b/net/mac80211/scan.c index 5ac2785cdc7b..d4cc9ac2d703 100644 --- a/net/mac80211/scan.c +++ b/net/mac80211/scan.c @@ -9,7 +9,7 @@ * Copyright 2007, Michael Wu <flamingice@sourmilk.net> * Copyright 2013-2015 Intel Mobile Communications GmbH * Copyright 2016-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2019 Intel Corporation + * Copyright (C) 2018-2020 Intel Corporation */ #include <linux/if_arp.h> @@ -146,7 +146,8 @@ ieee80211_bss_info_update(struct ieee80211_local *local, struct ieee80211_mgmt *mgmt, size_t len, struct ieee80211_channel *channel) { - bool beacon = ieee80211_is_beacon(mgmt->frame_control); + bool beacon = ieee80211_is_beacon(mgmt->frame_control) || + ieee80211_is_s1g_beacon(mgmt->frame_control); struct cfg80211_bss *cbss, *non_tx_cbss; struct ieee80211_bss *bss, *non_tx_bss; struct cfg80211_inform_bss bss_meta = { @@ -195,6 +196,11 @@ ieee80211_bss_info_update(struct ieee80211_local *local, elements = mgmt->u.probe_resp.variable; baselen = offsetof(struct ieee80211_mgmt, u.probe_resp.variable); + } else if (ieee80211_is_s1g_beacon(mgmt->frame_control)) { + struct ieee80211_ext *ext = (void *) mgmt; + + baselen = offsetof(struct ieee80211_ext, u.s1g_beacon.variable); + elements = ext->u.s1g_beacon.variable; } else { baselen = offsetof(struct ieee80211_mgmt, u.beacon.variable); elements = mgmt->u.beacon.variable; @@ -246,9 +252,12 @@ void ieee80211_scan_rx(struct ieee80211_local *local, struct sk_buff *skb) struct ieee80211_bss *bss; struct ieee80211_channel *channel; - if (skb->len < 24 || - (!ieee80211_is_probe_resp(mgmt->frame_control) && - !ieee80211_is_beacon(mgmt->frame_control))) + if (ieee80211_is_s1g_beacon(mgmt->frame_control)) { + if (skb->len < 15) + return; + } else if (skb->len < 24 || + (!ieee80211_is_probe_resp(mgmt->frame_control) && + !ieee80211_is_beacon(mgmt->frame_control))) return; sdata1 = rcu_dereference(local->scan_sdata); @@ -712,6 +721,10 @@ static int __ieee80211_start_scan(struct ieee80211_sub_if_data *sdata, req->duration_mandatory; local->hw_scan_band = 0; + local->hw_scan_req->req.n_6ghz_params = req->n_6ghz_params; + local->hw_scan_req->req.scan_6ghz_params = + req->scan_6ghz_params; + local->hw_scan_req->req.scan_6ghz = req->scan_6ghz; /* * After allocating local->hw_scan_req, we must @@ -905,6 +918,17 @@ static void ieee80211_scan_state_set_channel(struct ieee80211_local *local, local->scan_chandef.center_freq1 = chan->center_freq; local->scan_chandef.freq1_offset = chan->freq_offset; local->scan_chandef.center_freq2 = 0; + + /* For scanning on the S1G band, ignore scan_width (which is constant + * across all channels) for now since channel width is specific to each + * channel. Detect the required channel width here and likely revisit + * later. Maybe scan_width could be used to build the channel scan list? + */ + if (chan->band == NL80211_BAND_S1GHZ) { + local->scan_chandef.width = ieee80211_s1g_channel_width(chan); + goto set_channel; + } + switch (scan_req->scan_width) { case NL80211_BSS_CHAN_WIDTH_5: local->scan_chandef.width = NL80211_CHAN_WIDTH_5; @@ -925,8 +949,14 @@ static void ieee80211_scan_state_set_channel(struct ieee80211_local *local, else local->scan_chandef.width = NL80211_CHAN_WIDTH_20_NOHT; break; + case NL80211_BSS_CHAN_WIDTH_1: + case NL80211_BSS_CHAN_WIDTH_2: + /* shouldn't get here, S1G handled above */ + WARN_ON(1); + break; } +set_channel: if (ieee80211_hw_config(local, IEEE80211_CONF_CHANGE_CHANNEL)) skip = 1; @@ -1124,7 +1154,8 @@ int ieee80211_request_ibss_scan(struct ieee80211_sub_if_data *sdata, int max_n; for (band = 0; band < NUM_NL80211_BANDS; band++) { - if (!local->hw.wiphy->bands[band]) + if (!local->hw.wiphy->bands[band] || + band == NL80211_BAND_6GHZ) continue; max_n = local->hw.wiphy->bands[band]->n_channels; diff --git a/net/mac80211/status.c b/net/mac80211/status.c index 7fe5bececfd9..6feb45135020 100644 --- a/net/mac80211/status.c +++ b/net/mac80211/status.c @@ -982,10 +982,6 @@ static void __ieee80211_tx_status(struct ieee80211_hw *hw, if (!(info->flags & IEEE80211_TX_CTL_INJECTED) && acked) ieee80211_frame_acked(sta, skb); - if ((sta->sdata->vif.type == NL80211_IFTYPE_STATION) && - ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS)) - ieee80211_sta_tx_notify(sta->sdata, (void *) skb->data, - acked, info->status.tx_time); } /* SNMP counters @@ -1120,11 +1116,18 @@ void ieee80211_tx_status_ext(struct ieee80211_hw *hw, noack_success = !!(info->flags & IEEE80211_TX_STAT_NOACK_TRANSMITTED); if (pubsta) { + struct ieee80211_sub_if_data *sdata = sta->sdata; + if (!acked && !noack_success) sta->status_stats.retry_failed++; sta->status_stats.retry_count += retry_count; if (ieee80211_hw_check(&local->hw, REPORTS_TX_ACK_STATUS)) { + if (sdata->vif.type == NL80211_IFTYPE_STATION && + skb && !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP)) + ieee80211_sta_tx_notify(sdata, (void *) skb->data, + acked, info->status.tx_time); + if (acked) { sta->status_stats.last_ack = jiffies; @@ -1134,6 +1137,11 @@ void ieee80211_tx_status_ext(struct ieee80211_hw *hw, /* Track when last packet was ACKed */ sta->status_stats.last_pkt_time = jiffies; + /* Reset connection monitor */ + if (sdata->vif.type == NL80211_IFTYPE_STATION && + unlikely(sdata->u.mgd.probe_send_count > 0)) + sdata->u.mgd.probe_send_count = 0; + if (info->status.is_valid_ack_signal) { sta->status_stats.last_ack_signal = (s8)info->status.ack_signal; diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index adc83d830691..8ba10a48ded4 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -82,6 +82,10 @@ static __le16 ieee80211_duration(struct ieee80211_tx_data *tx, erp = txrate->flags & IEEE80211_RATE_ERP_G; + /* device is expected to do this */ + if (sband->band == NL80211_BAND_S1GHZ) + return 0; + /* * data and mgmt (except PS Poll): * - during CFP: 32768 @@ -2473,7 +2477,9 @@ static u16 ieee80211_store_ack_skb(struct ieee80211_local *local, * @sdata: virtual interface to build the header for * @skb: the skb to build the header in * @info_flags: skb flags to set + * @sta: the station pointer * @ctrl_flags: info control flags to set + * @cookie: cookie pointer to fill (if not %NULL) * * This function takes the skb with 802.3 header and reformats the header to * the appropriate IEEE 802.11 header based on which interface the packet is diff --git a/net/mac80211/util.c b/net/mac80211/util.c index 8d3bfc0fe176..49342060490f 100644 --- a/net/mac80211/util.c +++ b/net/mac80211/util.c @@ -45,6 +45,58 @@ struct ieee80211_hw *wiphy_to_ieee80211_hw(struct wiphy *wiphy) } EXPORT_SYMBOL(wiphy_to_ieee80211_hw); +u8 *ieee80211_get_bssid(struct ieee80211_hdr *hdr, size_t len, + enum nl80211_iftype type) +{ + __le16 fc = hdr->frame_control; + + if (ieee80211_is_data(fc)) { + if (len < 24) /* drop incorrect hdr len (data) */ + return NULL; + + if (ieee80211_has_a4(fc)) + return NULL; + if (ieee80211_has_tods(fc)) + return hdr->addr1; + if (ieee80211_has_fromds(fc)) + return hdr->addr2; + + return hdr->addr3; + } + + if (ieee80211_is_s1g_beacon(fc)) { + struct ieee80211_ext *ext = (void *) hdr; + + return ext->u.s1g_beacon.sa; + } + + if (ieee80211_is_mgmt(fc)) { + if (len < 24) /* drop incorrect hdr len (mgmt) */ + return NULL; + return hdr->addr3; + } + + if (ieee80211_is_ctl(fc)) { + if (ieee80211_is_pspoll(fc)) + return hdr->addr1; + + if (ieee80211_is_back_req(fc)) { + switch (type) { + case NL80211_IFTYPE_STATION: + return hdr->addr2; + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_AP_VLAN: + return hdr->addr1; + default: + break; /* fall through to the return */ + } + } + } + + return NULL; +} +EXPORT_SYMBOL(ieee80211_get_bssid); + void ieee80211_tx_set_protected(struct ieee80211_tx_data *tx) { struct sk_buff *skb; @@ -733,6 +785,9 @@ static void __iterate_interfaces(struct ieee80211_local *local, if (!(iter_flags & IEEE80211_IFACE_ITER_RESUME_ALL) && active_only && !(sdata->flags & IEEE80211_SDATA_IN_DRIVER)) continue; + if ((iter_flags & IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER) && + !(sdata->flags & IEEE80211_SDATA_IN_DRIVER)) + continue; if (ieee80211_sdata_running(sdata) || !active_only) iterator(data, sdata->vif.addr, &sdata->vif); @@ -1003,6 +1058,11 @@ _ieee802_11_parse_elems_crc(const u8 *start, size_t len, bool action, case WLAN_EID_LINK_ID: case WLAN_EID_BSS_MAX_IDLE_PERIOD: case WLAN_EID_RSNX: + case WLAN_EID_S1G_BCN_COMPAT: + case WLAN_EID_S1G_CAPABILITIES: + case WLAN_EID_S1G_OPERATION: + case WLAN_EID_AID_RESPONSE: + case WLAN_EID_S1G_SHORT_BCN_INTERVAL: /* * not listing WLAN_EID_CHANNEL_SWITCH_WRAPPER -- it seems possible * that if the content gets bigger it might be needed more than once @@ -1288,6 +1348,30 @@ _ieee802_11_parse_elems_crc(const u8 *start, size_t len, bool action, &crc : NULL, elem, elems); break; + case WLAN_EID_S1G_CAPABILITIES: + if (elen == sizeof(*elems->s1g_capab)) + elems->s1g_capab = (void *)pos; + else + elem_parse_failed = true; + break; + case WLAN_EID_S1G_OPERATION: + if (elen == sizeof(*elems->s1g_oper)) + elems->s1g_oper = (void *)pos; + else + elem_parse_failed = true; + break; + case WLAN_EID_S1G_BCN_COMPAT: + if (elen == sizeof(*elems->s1g_bcn_compat)) + elems->s1g_bcn_compat = (void *)pos; + else + elem_parse_failed = true; + break; + case WLAN_EID_AID_RESPONSE: + if (elen == sizeof(struct ieee80211_aid_response_ie)) + elems->aid_resp = (void *)pos; + else + elem_parse_failed = true; + break; default: break; } @@ -3371,6 +3455,42 @@ bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_sub_if_data *sdata, *chandef = he_chandef; + return false; +} + +bool ieee80211_chandef_s1g_oper(const struct ieee80211_s1g_oper_ie *oper, + struct cfg80211_chan_def *chandef) +{ + u32 oper_freq; + + if (!oper) + return false; + + switch (FIELD_GET(S1G_OPER_CH_WIDTH_OPER, oper->ch_width)) { + case IEEE80211_S1G_CHANWIDTH_1MHZ: + chandef->width = NL80211_CHAN_WIDTH_1; + break; + case IEEE80211_S1G_CHANWIDTH_2MHZ: + chandef->width = NL80211_CHAN_WIDTH_2; + break; + case IEEE80211_S1G_CHANWIDTH_4MHZ: + chandef->width = NL80211_CHAN_WIDTH_4; + break; + case IEEE80211_S1G_CHANWIDTH_8MHZ: + chandef->width = NL80211_CHAN_WIDTH_8; + break; + case IEEE80211_S1G_CHANWIDTH_16MHZ: + chandef->width = NL80211_CHAN_WIDTH_16; + break; + default: + return false; + } + + oper_freq = ieee80211_channel_to_freq_khz(oper->oper_ch, + NL80211_BAND_S1GHZ); + chandef->center_freq1 = KHZ_TO_MHZ(oper_freq); + chandef->freq1_offset = oper_freq % 1000; + return true; } @@ -4277,6 +4397,58 @@ int ieee80211_max_num_channels(struct ieee80211_local *local) return max_num_different_channels; } +void ieee80211_add_s1g_capab_ie(struct ieee80211_sub_if_data *sdata, + struct ieee80211_sta_s1g_cap *caps, + struct sk_buff *skb) +{ + struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; + struct ieee80211_s1g_cap s1g_capab; + u8 *pos; + int i; + + if (WARN_ON(sdata->vif.type != NL80211_IFTYPE_STATION)) + return; + + if (!caps->s1g) + return; + + memcpy(s1g_capab.capab_info, caps->cap, sizeof(caps->cap)); + memcpy(s1g_capab.supp_mcs_nss, caps->nss_mcs, sizeof(caps->nss_mcs)); + + /* override the capability info */ + for (i = 0; i < sizeof(ifmgd->s1g_capa.capab_info); i++) { + u8 mask = ifmgd->s1g_capa_mask.capab_info[i]; + + s1g_capab.capab_info[i] &= ~mask; + s1g_capab.capab_info[i] |= ifmgd->s1g_capa.capab_info[i] & mask; + } + + /* then MCS and NSS set */ + for (i = 0; i < sizeof(ifmgd->s1g_capa.supp_mcs_nss); i++) { + u8 mask = ifmgd->s1g_capa_mask.supp_mcs_nss[i]; + + s1g_capab.supp_mcs_nss[i] &= ~mask; + s1g_capab.supp_mcs_nss[i] |= + ifmgd->s1g_capa.supp_mcs_nss[i] & mask; + } + + pos = skb_put(skb, 2 + sizeof(s1g_capab)); + *pos++ = WLAN_EID_S1G_CAPABILITIES; + *pos++ = sizeof(s1g_capab); + + memcpy(pos, &s1g_capab, sizeof(s1g_capab)); +} + +void ieee80211_add_aid_request_ie(struct ieee80211_sub_if_data *sdata, + struct sk_buff *skb) +{ + u8 *pos = skb_put(skb, 3); + + *pos++ = WLAN_EID_AID_REQUEST; + *pos++ = 1; + *pos++ = 0; +} + u8 *ieee80211_add_wmm_info_ie(u8 *buf, u8 qosinfo) { *buf++ = WLAN_EID_VENDOR_SPECIFIC; @@ -4319,3 +4491,24 @@ const u8 ieee80211_ac_to_qos_mask[IEEE80211_NUM_ACS] = { IEEE80211_WMM_IE_STA_QOSINFO_AC_BE, IEEE80211_WMM_IE_STA_QOSINFO_AC_BK }; + +u16 ieee80211_encode_usf(int listen_interval) +{ + static const int listen_int_usf[] = { 1, 10, 1000, 10000 }; + u16 ui, usf = 0; + + /* find greatest USF */ + while (usf < IEEE80211_MAX_USF) { + if (listen_interval % listen_int_usf[usf + 1]) + break; + usf += 1; + } + ui = listen_interval / listen_int_usf[usf]; + + /* error if there is a remainder. Should've been checked by user */ + WARN_ON_ONCE(ui > IEEE80211_MAX_UI); + listen_interval = FIELD_PREP(LISTEN_INT_USF, usf) | + FIELD_PREP(LISTEN_INT_UI, ui); + + return (u16) listen_interval; +} diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 4ec4c0f72f61..df675a8e1918 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -2186,13 +2186,35 @@ EXPORT_SYMBOL(__nlmsg_put); * It would be better to create kernel thread. */ +static int netlink_dump_done(struct netlink_sock *nlk, struct sk_buff *skb, + struct netlink_callback *cb, + struct netlink_ext_ack *extack) +{ + struct nlmsghdr *nlh; + + nlh = nlmsg_put_answer(skb, cb, NLMSG_DONE, sizeof(nlk->dump_done_errno), + NLM_F_MULTI | cb->answer_flags); + if (WARN_ON(!nlh)) + return -ENOBUFS; + + nl_dump_check_consistent(cb, nlh); + memcpy(nlmsg_data(nlh), &nlk->dump_done_errno, sizeof(nlk->dump_done_errno)); + + if (extack->_msg && nlk->flags & NETLINK_F_EXT_ACK) { + nlh->nlmsg_flags |= NLM_F_ACK_TLVS; + if (!nla_put_string(skb, NLMSGERR_ATTR_MSG, extack->_msg)) + nlmsg_end(skb, nlh); + } + + return 0; +} + static int netlink_dump(struct sock *sk) { struct netlink_sock *nlk = nlk_sk(sk); struct netlink_ext_ack extack = {}; struct netlink_callback *cb; struct sk_buff *skb = NULL; - struct nlmsghdr *nlh; struct module *module; int err = -ENOBUFS; int alloc_min_size; @@ -2258,22 +2280,19 @@ static int netlink_dump(struct sock *sk) return 0; } - nlh = nlmsg_put_answer(skb, cb, NLMSG_DONE, - sizeof(nlk->dump_done_errno), - NLM_F_MULTI | cb->answer_flags); - if (WARN_ON(!nlh)) + if (netlink_dump_done(nlk, skb, cb, &extack)) goto errout_skb; - nl_dump_check_consistent(cb, nlh); - - memcpy(nlmsg_data(nlh), &nlk->dump_done_errno, - sizeof(nlk->dump_done_errno)); - - if (extack._msg && nlk->flags & NETLINK_F_EXT_ACK) { - nlh->nlmsg_flags |= NLM_F_ACK_TLVS; - if (!nla_put_string(skb, NLMSGERR_ATTR_MSG, extack._msg)) - nlmsg_end(skb, nlh); +#ifdef CONFIG_COMPAT_NETLINK_MESSAGES + /* frag_list skb's data is used for compat tasks + * and the regular skb's data for normal (non-compat) tasks. + * See netlink_recvmsg(). + */ + if (unlikely(skb_shinfo(skb)->frag_list)) { + if (netlink_dump_done(nlk, skb_shinfo(skb)->frag_list, cb, &extack)) + goto errout_skb; } +#endif if (sk_filter(sk, skb)) kfree_skb(skb); diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c index 3a718e327515..4a95e85cf907 100644 --- a/net/netlink/genetlink.c +++ b/net/netlink/genetlink.c @@ -1079,7 +1079,7 @@ static int ctrl_dumppolicy(struct sk_buff *skb, struct netlink_callback *cb) if (err) return err; - while (netlink_policy_dump_loop(&cb->args[1])) { + while (netlink_policy_dump_loop(cb->args[1])) { void *hdr; struct nlattr *nest; @@ -1113,6 +1113,12 @@ nla_put_failure: return skb->len; } +static int ctrl_dumppolicy_done(struct netlink_callback *cb) +{ + netlink_policy_dump_free(cb->args[1]); + return 0; +} + static const struct genl_ops genl_ctrl_ops[] = { { .cmd = CTRL_CMD_GETFAMILY, @@ -1123,6 +1129,7 @@ static const struct genl_ops genl_ctrl_ops[] = { { .cmd = CTRL_CMD_GETPOLICY, .dumpit = ctrl_dumppolicy, + .done = ctrl_dumppolicy_done, }, }; diff --git a/net/netlink/policy.c b/net/netlink/policy.c index 62f977fa645a..7a9792442578 100644 --- a/net/netlink/policy.c +++ b/net/netlink/policy.c @@ -84,7 +84,6 @@ int netlink_policy_dump_start(const struct nla_policy *policy, unsigned int policy_idx; int err; - /* also returns 0 if "*_state" is our ERR_PTR() end marker */ if (*_state) return 0; @@ -140,21 +139,11 @@ static bool netlink_policy_dump_finished(struct nl_policy_dump *state) !state->policies[state->policy_idx].policy; } -bool netlink_policy_dump_loop(unsigned long *_state) +bool netlink_policy_dump_loop(unsigned long _state) { - struct nl_policy_dump *state = (void *)*_state; - - if (IS_ERR(state)) - return false; - - if (netlink_policy_dump_finished(state)) { - kfree(state); - /* store end marker instead of freed state */ - *_state = (unsigned long)ERR_PTR(-ENOENT); - return false; - } + struct nl_policy_dump *state = (void *)_state; - return true; + return !netlink_policy_dump_finished(state); } int netlink_policy_dump_write(struct sk_buff *skb, unsigned long _state) @@ -316,3 +305,10 @@ nla_put_failure: nla_nest_cancel(skb, policy); return -ENOBUFS; } + +void netlink_policy_dump_free(unsigned long _state) +{ + struct nl_policy_dump *state = (void *)_state; + + kfree(state); +} diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index 7b69ab1993ba..54209a18d7fe 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -79,7 +79,7 @@ struct tc_u_hnode { /* The 'ht' field MUST be the last field in structure to allow for * more entries allocated at end of structure. */ - struct tc_u_knode __rcu *ht[1]; + struct tc_u_knode __rcu *ht[]; }; struct tc_u_common { @@ -353,7 +353,7 @@ static int u32_init(struct tcf_proto *tp) void *key = tc_u_common_ptr(tp); struct tc_u_common *tp_c = tc_u_common_find(key); - root_ht = kzalloc(sizeof(*root_ht), GFP_KERNEL); + root_ht = kzalloc(struct_size(root_ht, ht, 1), GFP_KERNEL); if (root_ht == NULL) return -ENOBUFS; @@ -364,7 +364,7 @@ static int u32_init(struct tcf_proto *tp) idr_init(&root_ht->handle_idr); if (tp_c == NULL) { - tp_c = kzalloc(sizeof(*tp_c), GFP_KERNEL); + tp_c = kzalloc(struct_size(tp_c, hlist->ht, 1), GFP_KERNEL); if (tp_c == NULL) { kfree(root_ht); return -ENOBUFS; @@ -933,7 +933,7 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, NL_SET_ERR_MSG_MOD(extack, "Divisor can only be used on a hash table"); return -EINVAL; } - ht = kzalloc(sizeof(*ht) + divisor*sizeof(void *), GFP_KERNEL); + ht = kzalloc(struct_size(ht, ht, divisor + 1), GFP_KERNEL); if (ht == NULL) return -ENOBUFS; if (handle == 0) { diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index ed8f97166be9..e874d0e6267f 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -26,6 +26,7 @@ #include <linux/sched/signal.h> #include <linux/if_vlan.h> #include <linux/rcupdate_wait.h> +#include <linux/ctype.h> #include <net/sock.h> #include <net/tcp.h> @@ -448,6 +449,16 @@ static void smcr_conn_save_peer_info(struct smc_sock *smc, smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1); } +static bool smc_isascii(char *hostname) +{ + int i; + + for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++) + if (!isascii(hostname[i])) + return false; + return true; +} + static void smcd_conn_save_peer_info(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *clc) { @@ -459,6 +470,22 @@ static void smcd_conn_save_peer_info(struct smc_sock *smc, smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg); atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx; + if (clc->hdr.version > SMC_V1 && + (clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK)) { + struct smc_clc_msg_accept_confirm_v2 *clc_v2 = + (struct smc_clc_msg_accept_confirm_v2 *)clc; + struct smc_clc_first_contact_ext *fce = + (struct smc_clc_first_contact_ext *) + (((u8 *)clc_v2) + sizeof(*clc_v2)); + + memcpy(smc->conn.lgr->negotiated_eid, clc_v2->eid, + SMC_MAX_EID_LEN); + smc->conn.lgr->peer_os = fce->os_type; + smc->conn.lgr->peer_smc_release = fce->release; + if (smc_isascii(fce->hostname)) + memcpy(smc->conn.lgr->peer_hostname, fce->hostname, + SMC_MAX_HOSTNAME_LEN); + } } static void smc_conn_save_peer_info(struct smc_sock *smc, @@ -504,7 +531,8 @@ static int smc_connect_fallback(struct smc_sock *smc, int reason_code) } /* decline and fall back during connect */ -static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code) +static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code, + u8 version) { int rc; @@ -514,7 +542,7 @@ static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code) return reason_code; } if (reason_code != SMC_CLC_DECL_PEERDECL) { - rc = smc_clc_send_decline(smc, reason_code); + rc = smc_clc_send_decline(smc, reason_code, version); if (rc < 0) { if (smc->sk.sk_state == SMC_INIT) sock_put(&smc->sk); /* passive closing */ @@ -564,47 +592,121 @@ static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini) { /* Find ISM device with same PNETID as connecting interface */ smc_pnet_find_ism_resource(smc->clcsock->sk, ini); - if (!ini->ism_dev) + if (!ini->ism_dev[0]) return SMC_CLC_DECL_NOSMCDDEV; + else + ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]); return 0; } +/* determine possible V2 ISM devices (either without PNETID or with PNETID plus + * PNETID matching net_device) + */ +static int smc_find_ism_v2_device_clnt(struct smc_sock *smc, + struct smc_init_info *ini) +{ + int rc = SMC_CLC_DECL_NOSMCDDEV; + struct smcd_dev *smcd; + int i = 1; + + if (smcd_indicated(ini->smc_type_v1)) + rc = 0; /* already initialized for V1 */ + mutex_lock(&smcd_dev_list.mutex); + list_for_each_entry(smcd, &smcd_dev_list.list, list) { + if (smcd->going_away || smcd == ini->ism_dev[0]) + continue; + if (!smc_pnet_is_pnetid_set(smcd->pnetid) || + smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) { + ini->ism_dev[i] = smcd; + ini->ism_chid[i] = smc_ism_get_chid(ini->ism_dev[i]); + ini->is_smcd = true; + rc = 0; + i++; + if (i > SMC_MAX_ISM_DEVS) + break; + } + } + mutex_unlock(&smcd_dev_list.mutex); + ini->ism_offered_cnt = i - 1; + if (!ini->ism_dev[0] && !ini->ism_dev[1]) + ini->smcd_version = 0; + + return rc; +} + /* Check for VLAN ID and register it on ISM device just for CLC handshake */ static int smc_connect_ism_vlan_setup(struct smc_sock *smc, struct smc_init_info *ini) { - if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev, ini->vlan_id)) + if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id)) return SMC_CLC_DECL_ISMVLANERR; return 0; } +static int smc_find_proposal_devices(struct smc_sock *smc, + struct smc_init_info *ini) +{ + int rc = 0; + + /* check if there is an ism device available */ + if (ini->smcd_version & SMC_V1) { + if (smc_find_ism_device(smc, ini) || + smc_connect_ism_vlan_setup(smc, ini)) { + if (ini->smc_type_v1 == SMC_TYPE_B) + ini->smc_type_v1 = SMC_TYPE_R; + else + ini->smc_type_v1 = SMC_TYPE_N; + } /* else ISM V1 is supported for this connection */ + if (smc_find_rdma_device(smc, ini)) { + if (ini->smc_type_v1 == SMC_TYPE_B) + ini->smc_type_v1 = SMC_TYPE_D; + else + ini->smc_type_v1 = SMC_TYPE_N; + } /* else RDMA is supported for this connection */ + } + if (smc_ism_v2_capable && smc_find_ism_v2_device_clnt(smc, ini)) + ini->smc_type_v2 = SMC_TYPE_N; + + /* if neither ISM nor RDMA are supported, fallback */ + if (!smcr_indicated(ini->smc_type_v1) && + ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N) + rc = SMC_CLC_DECL_NOSMCDEV; + + return rc; +} + /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is * used, the VLAN ID will be registered again during the connection setup. */ -static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd, +static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, struct smc_init_info *ini) { - if (!is_smcd) + if (!smcd_indicated(ini->smc_type_v1)) return 0; - if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev, ini->vlan_id)) + if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id)) return SMC_CLC_DECL_CNFERR; return 0; } +#define SMC_CLC_MAX_ACCEPT_LEN \ + (sizeof(struct smc_clc_msg_accept_confirm_v2) + \ + sizeof(struct smc_clc_first_contact_ext) + \ + sizeof(struct smc_clc_msg_trail)) + /* CLC handshake during connect */ -static int smc_connect_clc(struct smc_sock *smc, int smc_type, - struct smc_clc_msg_accept_confirm *aclc, +static int smc_connect_clc(struct smc_sock *smc, + struct smc_clc_msg_accept_confirm_v2 *aclc2, struct smc_init_info *ini) { int rc = 0; /* do inband token exchange */ - rc = smc_clc_send_proposal(smc, smc_type, ini); + rc = smc_clc_send_proposal(smc, ini); if (rc) return rc; /* receive SMC Accept CLC message */ - return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT, - CLC_WAIT_TIME); + return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN, + SMC_CLC_ACCEPT, CLC_WAIT_TIME); } /* setup for RDMA connection of client */ @@ -618,7 +720,7 @@ static int smc_connect_rdma(struct smc_sock *smc, ini->is_smcd = false; ini->ib_lcl = &aclc->r0.lcl; ini->ib_clcqpn = ntoh24(aclc->r0.qpn); - ini->first_contact_peer = aclc->hdr.flag; + ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK; mutex_lock(&smc_client_lgr_pending); reason_code = smc_conn_create(smc, ini); @@ -678,7 +780,8 @@ static int smc_connect_rdma(struct smc_sock *smc, } smc_rmb_sync_sg_for_device(&smc->conn); - reason_code = smc_clc_send_confirm(smc); + reason_code = smc_clc_send_confirm(smc, ini->first_contact_local, + SMC_V1); if (reason_code) return smc_connect_abort(smc, reason_code, ini->first_contact_local); @@ -704,6 +807,25 @@ static int smc_connect_rdma(struct smc_sock *smc, return 0; } +/* The server has chosen one of the proposed ISM devices for the communication. + * Determine from the CHID of the received CLC ACCEPT the ISM device chosen. + */ +static int +smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc, + struct smc_init_info *ini) +{ + int i; + + for (i = 0; i < ini->ism_offered_cnt + 1; i++) { + if (ini->ism_chid[i] == ntohs(aclc->chid)) { + ini->ism_selected = i; + return 0; + } + } + + return -EPROTO; +} + /* setup for ISM connection of client */ static int smc_connect_ism(struct smc_sock *smc, struct smc_clc_msg_accept_confirm *aclc, @@ -712,8 +834,17 @@ static int smc_connect_ism(struct smc_sock *smc, int rc = 0; ini->is_smcd = true; - ini->ism_peer_gid = aclc->d0.gid; - ini->first_contact_peer = aclc->hdr.flag; + ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK; + + if (aclc->hdr.version == SMC_V2) { + struct smc_clc_msg_accept_confirm_v2 *aclc_v2 = + (struct smc_clc_msg_accept_confirm_v2 *)aclc; + + rc = smc_v2_determine_accepted_chid(aclc_v2, ini); + if (rc) + return rc; + } + ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid; /* there is only one lgr role for SMC-D; use server lock */ mutex_lock(&smc_server_lgr_pending); @@ -736,7 +867,8 @@ static int smc_connect_ism(struct smc_sock *smc, smc_rx_init(smc); smc_tx_init(smc); - rc = smc_clc_send_confirm(smc); + rc = smc_clc_send_confirm(smc, ini->first_contact_local, + aclc->hdr.version); if (rc) return smc_connect_abort(smc, rc, ini->first_contact_local); mutex_unlock(&smc_server_lgr_pending); @@ -749,13 +881,32 @@ static int smc_connect_ism(struct smc_sock *smc, return 0; } +/* check if received accept type and version matches a proposed one */ +static int smc_connect_check_aclc(struct smc_init_info *ini, + struct smc_clc_msg_accept_confirm *aclc) +{ + if ((aclc->hdr.typev1 == SMC_TYPE_R && + !smcr_indicated(ini->smc_type_v1)) || + (aclc->hdr.typev1 == SMC_TYPE_D && + ((!smcd_indicated(ini->smc_type_v1) && + !smcd_indicated(ini->smc_type_v2)) || + (aclc->hdr.version == SMC_V1 && + !smcd_indicated(ini->smc_type_v1)) || + (aclc->hdr.version == SMC_V2 && + !smcd_indicated(ini->smc_type_v2))))) + return SMC_CLC_DECL_MODEUNSUPP; + + return 0; +} + /* perform steps before actually connecting */ static int __smc_connect(struct smc_sock *smc) { - bool ism_supported = false, rdma_supported = false; - struct smc_clc_msg_accept_confirm aclc; - struct smc_init_info ini = {0}; - int smc_type; + u8 version = smc_ism_v2_capable ? SMC_V2 : SMC_V1; + struct smc_clc_msg_accept_confirm_v2 *aclc2; + struct smc_clc_msg_accept_confirm *aclc; + struct smc_init_info *ini = NULL; + u8 *buf = NULL; int rc = 0; if (smc->use_fallback) @@ -765,58 +916,73 @@ static int __smc_connect(struct smc_sock *smc) if (!tcp_sk(smc->clcsock->sk)->syn_smc) return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC); - /* IPSec connections opt out of SMC-R optimizations */ + /* IPSec connections opt out of SMC optimizations */ if (using_ipsec(smc)) - return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC); + return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC, + version); - /* get vlan id from IP device */ - if (smc_vlan_by_tcpsk(smc->clcsock, &ini)) - return smc_connect_decline_fallback(smc, - SMC_CLC_DECL_GETVLANERR); + ini = kzalloc(sizeof(*ini), GFP_KERNEL); + if (!ini) + return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM, + version); - /* check if there is an ism device available */ - if (!smc_find_ism_device(smc, &ini) && - !smc_connect_ism_vlan_setup(smc, &ini)) { - /* ISM is supported for this connection */ - ism_supported = true; - smc_type = SMC_TYPE_D; - } - - /* check if there is a rdma device available */ - if (!smc_find_rdma_device(smc, &ini)) { - /* RDMA is supported for this connection */ - rdma_supported = true; - if (ism_supported) - smc_type = SMC_TYPE_B; /* both */ - else - smc_type = SMC_TYPE_R; /* only RDMA */ + ini->smcd_version = SMC_V1; + ini->smcd_version |= smc_ism_v2_capable ? SMC_V2 : 0; + ini->smc_type_v1 = SMC_TYPE_B; + ini->smc_type_v2 = smc_ism_v2_capable ? SMC_TYPE_D : SMC_TYPE_N; + + /* get vlan id from IP device */ + if (smc_vlan_by_tcpsk(smc->clcsock, ini)) { + ini->smcd_version &= ~SMC_V1; + ini->smc_type_v1 = SMC_TYPE_N; + if (!ini->smcd_version) { + rc = SMC_CLC_DECL_GETVLANERR; + goto fallback; + } } - /* if neither ISM nor RDMA are supported, fallback */ - if (!rdma_supported && !ism_supported) - return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV); + rc = smc_find_proposal_devices(smc, ini); + if (rc) + goto fallback; - /* perform CLC handshake */ - rc = smc_connect_clc(smc, smc_type, &aclc, &ini); - if (rc) { - smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini); - return smc_connect_decline_fallback(smc, rc); + buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL); + if (!buf) { + rc = SMC_CLC_DECL_MEM; + goto fallback; } + aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf; + aclc = (struct smc_clc_msg_accept_confirm *)aclc2; + + /* perform CLC handshake */ + rc = smc_connect_clc(smc, aclc2, ini); + if (rc) + goto vlan_cleanup; + + /* check if smc modes and versions of CLC proposal and accept match */ + rc = smc_connect_check_aclc(ini, aclc); + version = aclc->hdr.version == SMC_V1 ? SMC_V1 : version; + if (rc) + goto vlan_cleanup; /* depending on previous steps, connect using rdma or ism */ - if (rdma_supported && aclc.hdr.path == SMC_TYPE_R) - rc = smc_connect_rdma(smc, &aclc, &ini); - else if (ism_supported && aclc.hdr.path == SMC_TYPE_D) - rc = smc_connect_ism(smc, &aclc, &ini); - else - rc = SMC_CLC_DECL_MODEUNSUPP; - if (rc) { - smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini); - return smc_connect_decline_fallback(smc, rc); - } + if (aclc->hdr.typev1 == SMC_TYPE_R) + rc = smc_connect_rdma(smc, aclc, ini); + else if (aclc->hdr.typev1 == SMC_TYPE_D) + rc = smc_connect_ism(smc, aclc, ini); + if (rc) + goto vlan_cleanup; - smc_connect_ism_vlan_cleanup(smc, ism_supported, &ini); + smc_connect_ism_vlan_cleanup(smc, ini); + kfree(buf); + kfree(ini); return 0; + +vlan_cleanup: + smc_connect_ism_vlan_cleanup(smc, ini); + kfree(buf); +fallback: + kfree(ini); + return smc_connect_decline_fallback(smc, rc, version); } static void smc_connect_work(struct work_struct *work) @@ -1132,10 +1298,10 @@ static void smc_listen_out_err(struct smc_sock *new_smc) /* listen worker: decline and fall back if possible */ static void smc_listen_decline(struct smc_sock *new_smc, int reason_code, - bool local_first) + struct smc_init_info *ini, u8 version) { /* RDMA setup failed, switch back to TCP */ - if (local_first) + if (ini->first_contact_local) smc_lgr_cleanup_early(&new_smc->conn); else smc_conn_free(&new_smc->conn); @@ -1146,7 +1312,7 @@ static void smc_listen_decline(struct smc_sock *new_smc, int reason_code, smc_switch_to_fallback(new_smc); new_smc->fallback_rsn = reason_code; if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) { - if (smc_clc_send_decline(new_smc, reason_code) < 0) { + if (smc_clc_send_decline(new_smc, reason_code, version) < 0) { smc_listen_out_err(new_smc); return; } @@ -1154,6 +1320,47 @@ static void smc_listen_decline(struct smc_sock *new_smc, int reason_code, smc_listen_out_connected(new_smc); } +/* listen worker: version checking */ +static int smc_listen_v2_check(struct smc_sock *new_smc, + struct smc_clc_msg_proposal *pclc, + struct smc_init_info *ini) +{ + struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext; + struct smc_clc_v2_extension *pclc_v2_ext; + + ini->smc_type_v1 = pclc->hdr.typev1; + ini->smc_type_v2 = pclc->hdr.typev2; + ini->smcd_version = ini->smc_type_v1 != SMC_TYPE_N ? SMC_V1 : 0; + if (pclc->hdr.version > SMC_V1) + ini->smcd_version |= + ini->smc_type_v2 != SMC_TYPE_N ? SMC_V2 : 0; + if (!smc_ism_v2_capable) { + ini->smcd_version &= ~SMC_V2; + goto out; + } + pclc_v2_ext = smc_get_clc_v2_ext(pclc); + if (!pclc_v2_ext) { + ini->smcd_version &= ~SMC_V2; + goto out; + } + pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext); + if (!pclc_smcd_v2_ext) + ini->smcd_version &= ~SMC_V2; + +out: + if (!ini->smcd_version) { + if (pclc->hdr.typev1 == SMC_TYPE_B || + pclc->hdr.typev2 == SMC_TYPE_B) + return SMC_CLC_DECL_NOSMCDEV; + if (pclc->hdr.typev1 == SMC_TYPE_D || + pclc->hdr.typev2 == SMC_TYPE_D) + return SMC_CLC_DECL_NOSMCDDEV; + return SMC_CLC_DECL_NOSMCRDEV; + } + + return 0; +} + /* listen worker: check prefixes */ static int smc_listen_prfx_check(struct smc_sock *new_smc, struct smc_clc_msg_proposal *pclc) @@ -1161,6 +1368,8 @@ static int smc_listen_prfx_check(struct smc_sock *new_smc, struct smc_clc_msg_proposal_prefix *pclc_prfx; struct socket *newclcsock = new_smc->clcsock; + if (pclc->hdr.typev1 == SMC_TYPE_N) + return 0; pclc_prfx = smc_clc_proposal_get_prefix(pclc); if (smc_clc_prfx_match(newclcsock, pclc_prfx)) return SMC_CLC_DECL_DIFFPREFIX; @@ -1188,7 +1397,6 @@ static int smc_listen_rdma_init(struct smc_sock *new_smc, /* listen worker: initialize connection and buffers for SMC-D */ static int smc_listen_ism_init(struct smc_sock *new_smc, - struct smc_clc_msg_proposal *pclc, struct smc_init_info *ini) { int rc; @@ -1211,6 +1419,125 @@ static int smc_listen_ism_init(struct smc_sock *new_smc, return 0; } +static bool smc_is_already_selected(struct smcd_dev *smcd, + struct smc_init_info *ini, + int matches) +{ + int i; + + for (i = 0; i < matches; i++) + if (smcd == ini->ism_dev[i]) + return true; + + return false; +} + +/* check for ISM devices matching proposed ISM devices */ +static void smc_check_ism_v2_match(struct smc_init_info *ini, + u16 proposed_chid, u64 proposed_gid, + unsigned int *matches) +{ + struct smcd_dev *smcd; + + list_for_each_entry(smcd, &smcd_dev_list.list, list) { + if (smcd->going_away) + continue; + if (smc_is_already_selected(smcd, ini, *matches)) + continue; + if (smc_ism_get_chid(smcd) == proposed_chid && + !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) { + ini->ism_peer_gid[*matches] = proposed_gid; + ini->ism_dev[*matches] = smcd; + (*matches)++; + break; + } + } +} + +static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc, + struct smc_clc_msg_proposal *pclc, + struct smc_init_info *ini) +{ + struct smc_clc_smcd_v2_extension *smcd_v2_ext; + struct smc_clc_v2_extension *smc_v2_ext; + struct smc_clc_msg_smcd *pclc_smcd; + unsigned int matches = 0; + u8 *eid = NULL; + int i; + + if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2)) + return; + + pclc_smcd = smc_get_clc_msg_smcd(pclc); + smc_v2_ext = smc_get_clc_v2_ext(pclc); + smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext); + if (!smcd_v2_ext || + !smc_v2_ext->hdr.flag.seid) /* no system EID support for SMCD */ + goto not_found; + + mutex_lock(&smcd_dev_list.mutex); + if (pclc_smcd->ism.chid) + /* check for ISM device matching proposed native ISM device */ + smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid), + ntohll(pclc_smcd->ism.gid), &matches); + for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) { + /* check for ISM devices matching proposed non-native ISM + * devices + */ + smc_check_ism_v2_match(ini, + ntohs(smcd_v2_ext->gidchid[i - 1].chid), + ntohll(smcd_v2_ext->gidchid[i - 1].gid), + &matches); + } + mutex_unlock(&smcd_dev_list.mutex); + + if (ini->ism_dev[0]) { + smc_ism_get_system_eid(ini->ism_dev[0], &eid); + if (memcmp(eid, smcd_v2_ext->system_eid, SMC_MAX_EID_LEN)) + goto not_found; + } else { + goto not_found; + } + + /* separate - outside the smcd_dev_list.lock */ + for (i = 0; i < matches; i++) { + ini->smcd_version = SMC_V2; + ini->is_smcd = true; + ini->ism_selected = i; + if (smc_listen_ism_init(new_smc, ini)) + /* try next active ISM device */ + continue; + return; /* matching and usable V2 ISM device found */ + } + +not_found: + ini->smcd_version &= ~SMC_V2; + ini->ism_dev[0] = NULL; + ini->is_smcd = false; +} + +static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc, + struct smc_clc_msg_proposal *pclc, + struct smc_init_info *ini) +{ + struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc); + + /* check if ISM V1 is available */ + if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1)) + goto not_found; + ini->is_smcd = true; /* prepare ISM check */ + ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid); + if (smc_find_ism_device(new_smc, ini)) + goto not_found; + ini->ism_selected = 0; + if (!smc_listen_ism_init(new_smc, ini)) + return; /* V1 ISM device found */ + +not_found: + ini->ism_dev[0] = NULL; + ini->is_smcd = false; +} + /* listen worker: register buffers */ static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first) { @@ -1225,6 +1552,67 @@ static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first) return 0; } +static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc, + struct smc_clc_msg_proposal *pclc, + struct smc_init_info *ini) +{ + int rc; + + if (!smcr_indicated(ini->smc_type_v1)) + return SMC_CLC_DECL_NOSMCDEV; + + /* prepare RDMA check */ + ini->ib_lcl = &pclc->lcl; + rc = smc_find_rdma_device(new_smc, ini); + if (rc) { + /* no RDMA device found */ + if (ini->smc_type_v1 == SMC_TYPE_B) + /* neither ISM nor RDMA device found */ + rc = SMC_CLC_DECL_NOSMCDEV; + return rc; + } + rc = smc_listen_rdma_init(new_smc, ini); + if (rc) + return rc; + return smc_listen_rdma_reg(new_smc, ini->first_contact_local); +} + +/* determine the local device matching to proposal */ +static int smc_listen_find_device(struct smc_sock *new_smc, + struct smc_clc_msg_proposal *pclc, + struct smc_init_info *ini) +{ + int rc; + + /* check for ISM device matching V2 proposed device */ + smc_find_ism_v2_device_serv(new_smc, pclc, ini); + if (ini->ism_dev[0]) + return 0; + + if (!(ini->smcd_version & SMC_V1)) + return SMC_CLC_DECL_NOSMCDEV; + + /* check for matching IP prefix and subnet length */ + rc = smc_listen_prfx_check(new_smc, pclc); + if (rc) + return rc; + + /* get vlan id from IP device */ + if (smc_vlan_by_tcpsk(new_smc->clcsock, ini)) + return SMC_CLC_DECL_GETVLANERR; + + /* check for ISM device matching V1 proposed device */ + smc_find_ism_v1_device_serv(new_smc, pclc, ini); + if (ini->ism_dev[0]) + return 0; + + if (pclc->hdr.typev1 == SMC_TYPE_D) + return SMC_CLC_DECL_NOSMCDDEV; /* skip RDMA and decline */ + + /* check if RDMA is available */ + return smc_find_rdma_v1_device_serv(new_smc, pclc, ini); +} + /* listen worker: finish RDMA setup */ static int smc_listen_rdma_finish(struct smc_sock *new_smc, struct smc_clc_msg_accept_confirm *cclc, @@ -1250,17 +1638,18 @@ static int smc_listen_rdma_finish(struct smc_sock *new_smc, return reason_code; } -/* setup for RDMA connection of server */ +/* setup for connection of server */ static void smc_listen_work(struct work_struct *work) { struct smc_sock *new_smc = container_of(work, struct smc_sock, smc_listen_work); + u8 version = smc_ism_v2_capable ? SMC_V2 : SMC_V1; struct socket *newclcsock = new_smc->clcsock; - struct smc_clc_msg_accept_confirm cclc; + struct smc_clc_msg_accept_confirm_v2 *cclc2; + struct smc_clc_msg_accept_confirm *cclc; struct smc_clc_msg_proposal_area *buf; struct smc_clc_msg_proposal *pclc; - struct smc_init_info ini = {0}; - bool ism_supported = false; + struct smc_init_info *ini = NULL; int rc = 0; if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN) @@ -1292,101 +1681,76 @@ static void smc_listen_work(struct work_struct *work) SMC_CLC_PROPOSAL, CLC_WAIT_TIME); if (rc) goto out_decl; + version = pclc->hdr.version == SMC_V1 ? SMC_V1 : version; - /* IPSec connections opt out of SMC-R optimizations */ + /* IPSec connections opt out of SMC optimizations */ if (using_ipsec(new_smc)) { rc = SMC_CLC_DECL_IPSEC; goto out_decl; } - /* check for matching IP prefix and subnet length */ - rc = smc_listen_prfx_check(new_smc, pclc); - if (rc) + ini = kzalloc(sizeof(*ini), GFP_KERNEL); + if (!ini) { + rc = SMC_CLC_DECL_MEM; goto out_decl; + } - /* get vlan id from IP device */ - if (smc_vlan_by_tcpsk(new_smc->clcsock, &ini)) { - rc = SMC_CLC_DECL_GETVLANERR; + /* initial version checking */ + rc = smc_listen_v2_check(new_smc, pclc, ini); + if (rc) goto out_decl; - } mutex_lock(&smc_server_lgr_pending); smc_close_init(new_smc); smc_rx_init(new_smc); smc_tx_init(new_smc); - /* check if ISM is available */ - if (pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) { - struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc); - - ini.is_smcd = true; /* prepare ISM check */ - ini.ism_peer_gid = pclc_smcd->gid; - rc = smc_find_ism_device(new_smc, &ini); - if (!rc) - rc = smc_listen_ism_init(new_smc, pclc, &ini); - if (!rc) - ism_supported = true; - else if (pclc->hdr.path == SMC_TYPE_D) - goto out_unlock; /* skip RDMA and decline */ - } - - /* check if RDMA is available */ - if (!ism_supported) { /* SMC_TYPE_R or SMC_TYPE_B */ - /* prepare RDMA check */ - ini.is_smcd = false; - ini.ism_dev = NULL; - ini.ib_lcl = &pclc->lcl; - rc = smc_find_rdma_device(new_smc, &ini); - if (rc) { - /* no RDMA device found */ - if (pclc->hdr.path == SMC_TYPE_B) - /* neither ISM nor RDMA device found */ - rc = SMC_CLC_DECL_NOSMCDEV; - goto out_unlock; - } - rc = smc_listen_rdma_init(new_smc, &ini); - if (rc) - goto out_unlock; - rc = smc_listen_rdma_reg(new_smc, ini.first_contact_local); - if (rc) - goto out_unlock; - } + /* determine ISM or RoCE device used for connection */ + rc = smc_listen_find_device(new_smc, pclc, ini); + if (rc) + goto out_unlock; /* send SMC Accept CLC message */ - rc = smc_clc_send_accept(new_smc, ini.first_contact_local); + rc = smc_clc_send_accept(new_smc, ini->first_contact_local, + ini->smcd_version == SMC_V2 ? SMC_V2 : SMC_V1); if (rc) goto out_unlock; /* SMC-D does not need this lock any more */ - if (ism_supported) + if (ini->is_smcd) mutex_unlock(&smc_server_lgr_pending); /* receive SMC Confirm CLC message */ - rc = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc), + cclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf; + cclc = (struct smc_clc_msg_accept_confirm *)cclc2; + memset(buf, 0, sizeof(struct smc_clc_msg_proposal_area)); + rc = smc_clc_wait_msg(new_smc, cclc2, + sizeof(struct smc_clc_msg_proposal_area), SMC_CLC_CONFIRM, CLC_WAIT_TIME); if (rc) { - if (!ism_supported) + if (!ini->is_smcd) goto out_unlock; goto out_decl; } /* finish worker */ - if (!ism_supported) { - rc = smc_listen_rdma_finish(new_smc, &cclc, - ini.first_contact_local); + if (!ini->is_smcd) { + rc = smc_listen_rdma_finish(new_smc, cclc, + ini->first_contact_local); if (rc) goto out_unlock; mutex_unlock(&smc_server_lgr_pending); } - smc_conn_save_peer_info(new_smc, &cclc); + smc_conn_save_peer_info(new_smc, cclc); smc_listen_out_connected(new_smc); goto out_free; out_unlock: mutex_unlock(&smc_server_lgr_pending); out_decl: - smc_listen_decline(new_smc, rc, ini.first_contact_local); + smc_listen_decline(new_smc, rc, ini, version); out_free: + kfree(ini); kfree(buf); } @@ -2092,6 +2456,9 @@ static int __init smc_init(void) if (rc) return rc; + smc_ism_init(); + smc_clc_init(); + rc = smc_pnet_init(); if (rc) goto out_pernet_subsys; diff --git a/net/smc/smc.h b/net/smc/smc.h index 2bd57e57b7e7..d65e15f0c944 100644 --- a/net/smc/smc.h +++ b/net/smc/smc.h @@ -19,10 +19,19 @@ #include "smc_ib.h" #define SMC_V1 1 /* SMC version V1 */ +#define SMC_V2 2 /* SMC version V2 */ +#define SMC_RELEASE 0 #define SMCPROTO_SMC 0 /* SMC protocol, IPv4 */ #define SMCPROTO_SMC6 1 /* SMC protocol, IPv6 */ +#define SMC_MAX_ISM_DEVS 8 /* max # of proposed non-native ISM + * devices + */ + +#define SMC_MAX_HOSTNAME_LEN 32 +#define SMC_MAX_EID_LEN 32 + extern struct proto smc_proto; extern struct proto smc_proto6; @@ -246,6 +255,9 @@ extern struct workqueue_struct *smc_close_wq; /* wq for close work */ extern u8 local_systemid[SMC_SYSTEMID_LEN]; /* unique system identifier */ +#define ntohll(x) be64_to_cpu(x) +#define htonll(x) cpu_to_be64(x) + /* convert an u32 value into network byte order, store it into a 3 byte field */ static inline void hton24(u8 *net, u32 host) { diff --git a/net/smc/smc_clc.c b/net/smc/smc_clc.c index bd116e1949b9..696d89c2dce4 100644 --- a/net/smc/smc_clc.c +++ b/net/smc/smc_clc.c @@ -14,6 +14,8 @@ #include <linux/inetdevice.h> #include <linux/if_ether.h> #include <linux/sched/signal.h> +#include <linux/utsname.h> +#include <linux/ctype.h> #include <net/addrconf.h> #include <net/sock.h> @@ -27,6 +29,7 @@ #define SMCR_CLC_ACCEPT_CONFIRM_LEN 68 #define SMCD_CLC_ACCEPT_CONFIRM_LEN 48 +#define SMCD_CLC_ACCEPT_CONFIRM_LEN_V2 78 #define SMC_CLC_RECV_BUF_LEN 100 /* eye catcher "SMCR" EBCDIC for CLC messages */ @@ -34,13 +37,88 @@ static const char SMC_EYECATCHER[4] = {'\xe2', '\xd4', '\xc3', '\xd9'}; /* eye catcher "SMCD" EBCDIC for CLC messages */ static const char SMCD_EYECATCHER[4] = {'\xe2', '\xd4', '\xc3', '\xc4'}; +static u8 smc_hostname[SMC_MAX_HOSTNAME_LEN]; + +/* check arriving CLC proposal */ +static bool smc_clc_msg_prop_valid(struct smc_clc_msg_proposal *pclc) +{ + struct smc_clc_msg_proposal_prefix *pclc_prfx; + struct smc_clc_smcd_v2_extension *smcd_v2_ext; + struct smc_clc_msg_hdr *hdr = &pclc->hdr; + struct smc_clc_v2_extension *v2_ext; + + v2_ext = smc_get_clc_v2_ext(pclc); + pclc_prfx = smc_clc_proposal_get_prefix(pclc); + if (hdr->version == SMC_V1) { + if (hdr->typev1 == SMC_TYPE_N) + return false; + if (ntohs(hdr->length) != + sizeof(*pclc) + ntohs(pclc->iparea_offset) + + sizeof(*pclc_prfx) + + pclc_prfx->ipv6_prefixes_cnt * + sizeof(struct smc_clc_ipv6_prefix) + + sizeof(struct smc_clc_msg_trail)) + return false; + } else { + if (ntohs(hdr->length) != + sizeof(*pclc) + + sizeof(struct smc_clc_msg_smcd) + + (hdr->typev1 != SMC_TYPE_N ? + sizeof(*pclc_prfx) + + pclc_prfx->ipv6_prefixes_cnt * + sizeof(struct smc_clc_ipv6_prefix) : 0) + + (hdr->typev2 != SMC_TYPE_N ? + sizeof(*v2_ext) + + v2_ext->hdr.eid_cnt * SMC_MAX_EID_LEN : 0) + + (smcd_indicated(hdr->typev2) ? + sizeof(*smcd_v2_ext) + v2_ext->hdr.ism_gid_cnt * + sizeof(struct smc_clc_smcd_gid_chid) : + 0) + + sizeof(struct smc_clc_msg_trail)) + return false; + } + return true; +} + +/* check arriving CLC accept or confirm */ +static bool +smc_clc_msg_acc_conf_valid(struct smc_clc_msg_accept_confirm_v2 *clc_v2) +{ + struct smc_clc_msg_hdr *hdr = &clc_v2->hdr; + + if (hdr->typev1 != SMC_TYPE_R && hdr->typev1 != SMC_TYPE_D) + return false; + if (hdr->version == SMC_V1) { + if ((hdr->typev1 == SMC_TYPE_R && + ntohs(hdr->length) != SMCR_CLC_ACCEPT_CONFIRM_LEN) || + (hdr->typev1 == SMC_TYPE_D && + ntohs(hdr->length) != SMCD_CLC_ACCEPT_CONFIRM_LEN)) + return false; + } else { + if (hdr->typev1 == SMC_TYPE_D && + ntohs(hdr->length) != SMCD_CLC_ACCEPT_CONFIRM_LEN_V2 && + (ntohs(hdr->length) != SMCD_CLC_ACCEPT_CONFIRM_LEN_V2 + + sizeof(struct smc_clc_first_contact_ext))) + return false; + } + return true; +} + +static void smc_clc_fill_fce(struct smc_clc_first_contact_ext *fce, int *len) +{ + memset(fce, 0, sizeof(*fce)); + fce->os_type = SMC_CLC_OS_LINUX; + fce->release = SMC_RELEASE; + memcpy(fce->hostname, smc_hostname, sizeof(smc_hostname)); + (*len) += sizeof(*fce); +} + /* check if received message has a correct header length and contains valid * heading and trailing eyecatchers */ static bool smc_clc_msg_hdr_valid(struct smc_clc_msg_hdr *clcm, bool check_trl) { - struct smc_clc_msg_proposal_prefix *pclc_prfx; - struct smc_clc_msg_accept_confirm *clc; + struct smc_clc_msg_accept_confirm_v2 *clc_v2; struct smc_clc_msg_proposal *pclc; struct smc_clc_msg_decline *dclc; struct smc_clc_msg_trail *trl; @@ -51,29 +129,19 @@ static bool smc_clc_msg_hdr_valid(struct smc_clc_msg_hdr *clcm, bool check_trl) switch (clcm->type) { case SMC_CLC_PROPOSAL: pclc = (struct smc_clc_msg_proposal *)clcm; - pclc_prfx = smc_clc_proposal_get_prefix(pclc); - if (ntohs(pclc->hdr.length) < - sizeof(*pclc) + ntohs(pclc->iparea_offset) + - sizeof(*pclc_prfx) + - pclc_prfx->ipv6_prefixes_cnt * - sizeof(struct smc_clc_ipv6_prefix) + - sizeof(*trl)) + if (!smc_clc_msg_prop_valid(pclc)) return false; trl = (struct smc_clc_msg_trail *) ((u8 *)pclc + ntohs(pclc->hdr.length) - sizeof(*trl)); break; case SMC_CLC_ACCEPT: case SMC_CLC_CONFIRM: - if (clcm->path != SMC_TYPE_R && clcm->path != SMC_TYPE_D) - return false; - clc = (struct smc_clc_msg_accept_confirm *)clcm; - if ((clcm->path == SMC_TYPE_R && - ntohs(clc->hdr.length) != SMCR_CLC_ACCEPT_CONFIRM_LEN) || - (clcm->path == SMC_TYPE_D && - ntohs(clc->hdr.length) != SMCD_CLC_ACCEPT_CONFIRM_LEN)) + clc_v2 = (struct smc_clc_msg_accept_confirm_v2 *)clcm; + if (!smc_clc_msg_acc_conf_valid(clc_v2)) return false; trl = (struct smc_clc_msg_trail *) - ((u8 *)clc + ntohs(clc->hdr.length) - sizeof(*trl)); + ((u8 *)clc_v2 + ntohs(clc_v2->hdr.length) - + sizeof(*trl)); break; case SMC_CLC_DECLINE: dclc = (struct smc_clc_msg_decline *)clcm; @@ -327,9 +395,6 @@ int smc_clc_wait_msg(struct smc_sock *smc, void *buf, int buflen, goto out; } - if (clcm->type == SMC_CLC_PROPOSAL && clcm->path == SMC_TYPE_N) - reason_code = SMC_CLC_DECL_VERSMISMAT; /* just V2 offered */ - /* receive the complete CLC message */ memset(&msg, 0, sizeof(struct msghdr)); if (datlen > buflen) { @@ -365,7 +430,8 @@ int smc_clc_wait_msg(struct smc_sock *smc, void *buf, int buflen, dclc = (struct smc_clc_msg_decline *)clcm; reason_code = SMC_CLC_DECL_PEERDECL; smc->peer_diagnosis = ntohl(dclc->peer_diagnosis); - if (((struct smc_clc_msg_decline *)buf)->hdr.flag) { + if (((struct smc_clc_msg_decline *)buf)->hdr.typev2 & + SMC_FIRST_CONTACT_MASK) { smc->conn.lgr->sync_err = 1; smc_lgr_terminate_sched(smc->conn.lgr); } @@ -377,7 +443,7 @@ out: } /* send CLC DECLINE message across internal TCP socket */ -int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info) +int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info, u8 version) { struct smc_clc_msg_decline dclc; struct msghdr msg; @@ -388,8 +454,10 @@ int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info) memcpy(dclc.hdr.eyecatcher, SMC_EYECATCHER, sizeof(SMC_EYECATCHER)); dclc.hdr.type = SMC_CLC_DECLINE; dclc.hdr.length = htons(sizeof(struct smc_clc_msg_decline)); - dclc.hdr.version = SMC_V1; - dclc.hdr.flag = (peer_diag_info == SMC_CLC_DECL_SYNCERR) ? 1 : 0; + dclc.hdr.version = version; + dclc.os_type = version == SMC_V1 ? 0 : SMC_CLC_OS_LINUX; + dclc.hdr.typev2 = (peer_diag_info == SMC_CLC_DECL_SYNCERR) ? + SMC_FIRST_CONTACT_MASK : 0; if ((!smc->conn.lgr || !smc->conn.lgr->is_smcd) && smc_ib_is_valid_local_systemid()) memcpy(dclc.id_for_peer, local_systemid, @@ -408,18 +476,20 @@ int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info) } /* send CLC PROPOSAL message across internal TCP socket */ -int smc_clc_send_proposal(struct smc_sock *smc, int smc_type, - struct smc_init_info *ini) +int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini) { + struct smc_clc_smcd_v2_extension *smcd_v2_ext; struct smc_clc_msg_proposal_prefix *pclc_prfx; struct smc_clc_msg_proposal *pclc_base; + struct smc_clc_smcd_gid_chid *gidchids; struct smc_clc_msg_proposal_area *pclc; struct smc_clc_ipv6_prefix *ipv6_prfx; + struct smc_clc_v2_extension *v2_ext; struct smc_clc_msg_smcd *pclc_smcd; struct smc_clc_msg_trail *trl; int len, i, plen, rc; int reason_code = 0; - struct kvec vec[5]; + struct kvec vec[8]; struct msghdr msg; pclc = kzalloc(sizeof(*pclc), GFP_KERNEL); @@ -430,56 +500,121 @@ int smc_clc_send_proposal(struct smc_sock *smc, int smc_type, pclc_smcd = &pclc->pclc_smcd; pclc_prfx = &pclc->pclc_prfx; ipv6_prfx = pclc->pclc_prfx_ipv6; + v2_ext = &pclc->pclc_v2_ext; + smcd_v2_ext = &pclc->pclc_smcd_v2_ext; + gidchids = pclc->pclc_gidchids; trl = &pclc->pclc_trl; + pclc_base->hdr.version = SMC_V2; + pclc_base->hdr.typev1 = ini->smc_type_v1; + pclc_base->hdr.typev2 = ini->smc_type_v2; + plen = sizeof(*pclc_base) + sizeof(*pclc_smcd) + sizeof(*trl); + /* retrieve ip prefixes for CLC proposal msg */ - rc = smc_clc_prfx_set(smc->clcsock, pclc_prfx, ipv6_prfx); - if (rc) { - kfree(pclc); - return SMC_CLC_DECL_CNFERR; /* configuration error */ + if (ini->smc_type_v1 != SMC_TYPE_N) { + rc = smc_clc_prfx_set(smc->clcsock, pclc_prfx, ipv6_prfx); + if (rc) { + if (ini->smc_type_v2 == SMC_TYPE_N) { + kfree(pclc); + return SMC_CLC_DECL_CNFERR; + } + pclc_base->hdr.typev1 = SMC_TYPE_N; + } else { + pclc_base->iparea_offset = htons(sizeof(*pclc_smcd)); + plen += sizeof(*pclc_prfx) + + pclc_prfx->ipv6_prefixes_cnt * + sizeof(ipv6_prfx[0]); + } } - /* send SMC Proposal CLC message */ - plen = sizeof(*pclc_base) + sizeof(*pclc_prfx) + - (pclc_prfx->ipv6_prefixes_cnt * sizeof(ipv6_prfx[0])) + - sizeof(*trl); + /* build SMC Proposal CLC message */ memcpy(pclc_base->hdr.eyecatcher, SMC_EYECATCHER, sizeof(SMC_EYECATCHER)); pclc_base->hdr.type = SMC_CLC_PROPOSAL; - pclc_base->hdr.version = SMC_V1; /* SMC version */ - pclc_base->hdr.path = smc_type; - if (smc_type == SMC_TYPE_R || smc_type == SMC_TYPE_B) { + if (smcr_indicated(ini->smc_type_v1)) { /* add SMC-R specifics */ memcpy(pclc_base->lcl.id_for_peer, local_systemid, sizeof(local_systemid)); memcpy(pclc_base->lcl.gid, ini->ib_gid, SMC_GID_SIZE); memcpy(pclc_base->lcl.mac, &ini->ib_dev->mac[ini->ib_port - 1], ETH_ALEN); - pclc_base->iparea_offset = htons(0); } - if (smc_type == SMC_TYPE_D || smc_type == SMC_TYPE_B) { + if (smcd_indicated(ini->smc_type_v1)) { /* add SMC-D specifics */ - plen += sizeof(*pclc_smcd); - pclc_base->iparea_offset = htons(sizeof(*pclc_smcd)); - pclc_smcd->gid = ini->ism_dev->local_gid; + if (ini->ism_dev[0]) { + pclc_smcd->ism.gid = htonll(ini->ism_dev[0]->local_gid); + pclc_smcd->ism.chid = + htons(smc_ism_get_chid(ini->ism_dev[0])); + } + } + if (ini->smc_type_v2 == SMC_TYPE_N) { + pclc_smcd->v2_ext_offset = 0; + } else { + u16 v2_ext_offset; + u8 *eid = NULL; + + v2_ext_offset = sizeof(*pclc_smcd) - + offsetofend(struct smc_clc_msg_smcd, v2_ext_offset); + if (ini->smc_type_v1 != SMC_TYPE_N) + v2_ext_offset += sizeof(*pclc_prfx) + + pclc_prfx->ipv6_prefixes_cnt * + sizeof(ipv6_prfx[0]); + pclc_smcd->v2_ext_offset = htons(v2_ext_offset); + v2_ext->hdr.eid_cnt = 0; + v2_ext->hdr.ism_gid_cnt = ini->ism_offered_cnt; + v2_ext->hdr.flag.release = SMC_RELEASE; + v2_ext->hdr.flag.seid = 1; + v2_ext->hdr.smcd_v2_ext_offset = htons(sizeof(*v2_ext) - + offsetofend(struct smc_clnt_opts_area_hdr, + smcd_v2_ext_offset) + + v2_ext->hdr.eid_cnt * SMC_MAX_EID_LEN); + if (ini->ism_dev[0]) + smc_ism_get_system_eid(ini->ism_dev[0], &eid); + else + smc_ism_get_system_eid(ini->ism_dev[1], &eid); + if (eid) + memcpy(smcd_v2_ext->system_eid, eid, SMC_MAX_EID_LEN); + plen += sizeof(*v2_ext) + sizeof(*smcd_v2_ext); + if (ini->ism_offered_cnt) { + for (i = 1; i <= ini->ism_offered_cnt; i++) { + gidchids[i - 1].gid = + htonll(ini->ism_dev[i]->local_gid); + gidchids[i - 1].chid = + htons(smc_ism_get_chid(ini->ism_dev[i])); + } + plen += ini->ism_offered_cnt * + sizeof(struct smc_clc_smcd_gid_chid); + } } pclc_base->hdr.length = htons(plen); - memcpy(trl->eyecatcher, SMC_EYECATCHER, sizeof(SMC_EYECATCHER)); + + /* send SMC Proposal CLC message */ memset(&msg, 0, sizeof(msg)); i = 0; vec[i].iov_base = pclc_base; vec[i++].iov_len = sizeof(*pclc_base); - if (smc_type == SMC_TYPE_D || smc_type == SMC_TYPE_B) { - vec[i].iov_base = pclc_smcd; - vec[i++].iov_len = sizeof(*pclc_smcd); + vec[i].iov_base = pclc_smcd; + vec[i++].iov_len = sizeof(*pclc_smcd); + if (ini->smc_type_v1 != SMC_TYPE_N) { + vec[i].iov_base = pclc_prfx; + vec[i++].iov_len = sizeof(*pclc_prfx); + if (pclc_prfx->ipv6_prefixes_cnt > 0) { + vec[i].iov_base = ipv6_prfx; + vec[i++].iov_len = pclc_prfx->ipv6_prefixes_cnt * + sizeof(ipv6_prfx[0]); + } } - vec[i].iov_base = pclc_prfx; - vec[i++].iov_len = sizeof(*pclc_prfx); - if (pclc_prfx->ipv6_prefixes_cnt > 0) { - vec[i].iov_base = ipv6_prfx; - vec[i++].iov_len = pclc_prfx->ipv6_prefixes_cnt * - sizeof(ipv6_prfx[0]); + if (ini->smc_type_v2 != SMC_TYPE_N) { + vec[i].iov_base = v2_ext; + vec[i++].iov_len = sizeof(*v2_ext); + vec[i].iov_base = smcd_v2_ext; + vec[i++].iov_len = sizeof(*smcd_v2_ext); + if (ini->ism_offered_cnt) { + vec[i].iov_base = gidchids; + vec[i++].iov_len = ini->ism_offered_cnt * + sizeof(struct smc_clc_smcd_gid_chid); + } } vec[i].iov_base = trl; vec[i++].iov_len = sizeof(*trl); @@ -499,29 +634,47 @@ int smc_clc_send_proposal(struct smc_sock *smc, int smc_type, /* build and send CLC CONFIRM / ACCEPT message */ static int smc_clc_send_confirm_accept(struct smc_sock *smc, - struct smc_clc_msg_accept_confirm *clc, - int first_contact) + struct smc_clc_msg_accept_confirm_v2 *clc_v2, + int first_contact, u8 version) { struct smc_connection *conn = &smc->conn; + struct smc_clc_msg_accept_confirm *clc; + struct smc_clc_first_contact_ext fce; + struct smc_clc_msg_trail trl; + struct kvec vec[3]; struct msghdr msg; - struct kvec vec; + int i, len; /* send SMC Confirm CLC msg */ - clc->hdr.version = SMC_V1; /* SMC version */ + clc = (struct smc_clc_msg_accept_confirm *)clc_v2; + clc->hdr.version = version; /* SMC version */ if (first_contact) - clc->hdr.flag = 1; + clc->hdr.typev2 |= SMC_FIRST_CONTACT_MASK; if (conn->lgr->is_smcd) { /* SMC-D specific settings */ memcpy(clc->hdr.eyecatcher, SMCD_EYECATCHER, sizeof(SMCD_EYECATCHER)); - clc->hdr.path = SMC_TYPE_D; - clc->hdr.length = htons(SMCD_CLC_ACCEPT_CONFIRM_LEN); + clc->hdr.typev1 = SMC_TYPE_D; clc->d0.gid = conn->lgr->smcd->local_gid; clc->d0.token = conn->rmb_desc->token; clc->d0.dmbe_size = conn->rmbe_size_short; clc->d0.dmbe_idx = 0; memcpy(&clc->d0.linkid, conn->lgr->id, SMC_LGR_ID_SIZE); - memcpy(clc->d0.smcd_trl.eyecatcher, SMCD_EYECATCHER, + if (version == SMC_V1) { + clc->hdr.length = htons(SMCD_CLC_ACCEPT_CONFIRM_LEN); + } else { + u8 *eid = NULL; + + clc_v2->chid = htons(smc_ism_get_chid(conn->lgr->smcd)); + smc_ism_get_system_eid(conn->lgr->smcd, &eid); + if (eid) + memcpy(clc_v2->eid, eid, SMC_MAX_EID_LEN); + len = SMCD_CLC_ACCEPT_CONFIRM_LEN_V2; + if (first_contact) + smc_clc_fill_fce(&fce, &len); + clc_v2->hdr.length = htons(len); + } + memcpy(trl.eyecatcher, SMCD_EYECATCHER, sizeof(SMCD_EYECATCHER)); } else { struct smc_link *link = conn->lnk; @@ -530,7 +683,7 @@ static int smc_clc_send_confirm_accept(struct smc_sock *smc, link = conn->lnk; memcpy(clc->hdr.eyecatcher, SMC_EYECATCHER, sizeof(SMC_EYECATCHER)); - clc->hdr.path = SMC_TYPE_R; + clc->hdr.typev1 = SMC_TYPE_R; clc->hdr.length = htons(SMCR_CLC_ACCEPT_CONFIRM_LEN); memcpy(clc->r0.lcl.id_for_peer, local_systemid, sizeof(local_systemid)); @@ -554,29 +707,43 @@ static int smc_clc_send_confirm_accept(struct smc_sock *smc, clc->r0.rmb_dma_addr = cpu_to_be64((u64)sg_dma_address (conn->rmb_desc->sgt[link->link_idx].sgl)); hton24(clc->r0.psn, link->psn_initial); - memcpy(clc->r0.smcr_trl.eyecatcher, SMC_EYECATCHER, - sizeof(SMC_EYECATCHER)); + memcpy(trl.eyecatcher, SMC_EYECATCHER, sizeof(SMC_EYECATCHER)); } memset(&msg, 0, sizeof(msg)); - vec.iov_base = clc; - vec.iov_len = ntohs(clc->hdr.length); - return kernel_sendmsg(smc->clcsock, &msg, &vec, 1, + i = 0; + vec[i].iov_base = clc_v2; + if (version > SMC_V1) + vec[i++].iov_len = SMCD_CLC_ACCEPT_CONFIRM_LEN_V2 - sizeof(trl); + else + vec[i++].iov_len = (clc->hdr.typev1 == SMC_TYPE_D ? + SMCD_CLC_ACCEPT_CONFIRM_LEN : + SMCR_CLC_ACCEPT_CONFIRM_LEN) - + sizeof(trl); + if (version > SMC_V1 && first_contact) { + vec[i].iov_base = &fce; + vec[i++].iov_len = sizeof(fce); + } + vec[i].iov_base = &trl; + vec[i++].iov_len = sizeof(trl); + return kernel_sendmsg(smc->clcsock, &msg, vec, 1, ntohs(clc->hdr.length)); } /* send CLC CONFIRM message across internal TCP socket */ -int smc_clc_send_confirm(struct smc_sock *smc) +int smc_clc_send_confirm(struct smc_sock *smc, bool clnt_first_contact, + u8 version) { - struct smc_clc_msg_accept_confirm cclc; + struct smc_clc_msg_accept_confirm_v2 cclc_v2; int reason_code = 0; int len; /* send SMC Confirm CLC msg */ - memset(&cclc, 0, sizeof(cclc)); - cclc.hdr.type = SMC_CLC_CONFIRM; - len = smc_clc_send_confirm_accept(smc, &cclc, 0); - if (len < ntohs(cclc.hdr.length)) { + memset(&cclc_v2, 0, sizeof(cclc_v2)); + cclc_v2.hdr.type = SMC_CLC_CONFIRM; + len = smc_clc_send_confirm_accept(smc, &cclc_v2, clnt_first_contact, + version); + if (len < ntohs(cclc_v2.hdr.length)) { if (len >= 0) { reason_code = -ENETUNREACH; smc->sk.sk_err = -reason_code; @@ -589,16 +756,28 @@ int smc_clc_send_confirm(struct smc_sock *smc) } /* send CLC ACCEPT message across internal TCP socket */ -int smc_clc_send_accept(struct smc_sock *new_smc, bool srv_first_contact) +int smc_clc_send_accept(struct smc_sock *new_smc, bool srv_first_contact, + u8 version) { - struct smc_clc_msg_accept_confirm aclc; + struct smc_clc_msg_accept_confirm_v2 aclc_v2; int len; - memset(&aclc, 0, sizeof(aclc)); - aclc.hdr.type = SMC_CLC_ACCEPT; - len = smc_clc_send_confirm_accept(new_smc, &aclc, srv_first_contact); - if (len < ntohs(aclc.hdr.length)) + memset(&aclc_v2, 0, sizeof(aclc_v2)); + aclc_v2.hdr.type = SMC_CLC_ACCEPT; + len = smc_clc_send_confirm_accept(new_smc, &aclc_v2, srv_first_contact, + version); + if (len < ntohs(aclc_v2.hdr.length)) len = len >= 0 ? -EPROTO : -new_smc->clcsock->sk->sk_err; return len > 0 ? 0 : len; } + +void __init smc_clc_init(void) +{ + struct new_utsname *u; + + memset(smc_hostname, _S, sizeof(smc_hostname)); /* ASCII blanks */ + u = utsname(); + memcpy(smc_hostname, u->nodename, + min_t(size_t, strlen(u->nodename), sizeof(smc_hostname))); +} diff --git a/net/smc/smc_clc.h b/net/smc/smc_clc.h index fcd8521c7737..b3f46ab79e47 100644 --- a/net/smc/smc_clc.h +++ b/net/smc/smc_clc.h @@ -54,19 +54,19 @@ #define SMC_CLC_DECL_ERR_RDYLNK 0x09990002 /* ib ready link failed */ #define SMC_CLC_DECL_ERR_REGRMB 0x09990003 /* reg rmb failed */ +#define SMC_FIRST_CONTACT_MASK 0b10 /* first contact bit within typev2 */ + struct smc_clc_msg_hdr { /* header1 of clc messages */ u8 eyecatcher[4]; /* eye catcher */ u8 type; /* proposal / accept / confirm / decline */ __be16 length; #if defined(__BIG_ENDIAN_BITFIELD) u8 version : 4, - flag : 1, - rsvd : 1, - path : 2; + typev2 : 2, + typev1 : 2; #elif defined(__LITTLE_ENDIAN_BITFIELD) - u8 path : 2, - rsvd : 1, - flag : 1, + u8 typev1 : 2, + typev2 : 2, version : 4; #endif } __packed; /* format defined in RFC7609 */ @@ -81,8 +81,6 @@ struct smc_clc_msg_local { /* header2 of clc messages */ u8 mac[6]; /* mac of ib_device port */ }; -#define SMC_CLC_MAX_V6_PREFIX 8 - /* Struct would be 4 byte aligned, but it is used in an array that is sent * to peers and must conform to RFC7609, hence we need to use packed here. */ @@ -91,6 +89,44 @@ struct smc_clc_ipv6_prefix { u8 prefix_len; } __packed; /* format defined in RFC7609 */ +#if defined(__BIG_ENDIAN_BITFIELD) +struct smc_clc_v2_flag { + u8 release : 4, + rsvd : 3, + seid : 1; +}; +#elif defined(__LITTLE_ENDIAN_BITFIELD) +struct smc_clc_v2_flag { + u8 seid : 1, + rsvd : 3, + release : 4; +}; +#endif + +struct smc_clnt_opts_area_hdr { + u8 eid_cnt; /* number of user defined EIDs */ + u8 ism_gid_cnt; /* number of ISMv2 GIDs */ + u8 reserved1; + struct smc_clc_v2_flag flag; + u8 reserved2[2]; + __be16 smcd_v2_ext_offset; /* SMC-Dv2 Extension Offset */ +}; + +struct smc_clc_smcd_gid_chid { + __be64 gid; /* ISM GID */ + __be16 chid; /* ISMv2 CHID */ +} __packed; /* format defined in + * IBM Shared Memory Communications Version 2 + * (https://www.ibm.com/support/pages/node/6326337) + */ + +struct smc_clc_v2_extension { + struct smc_clnt_opts_area_hdr hdr; + u8 roce[16]; /* RoCEv2 GID */ + u8 reserved[16]; + u8 user_eids[0][SMC_MAX_EID_LEN]; +}; + struct smc_clc_msg_proposal_prefix { /* prefix part of clc proposal message*/ __be32 outgoing_subnet; /* subnet mask */ u8 prefix_len; /* number of significant bits in mask */ @@ -99,8 +135,15 @@ struct smc_clc_msg_proposal_prefix { /* prefix part of clc proposal message*/ } __aligned(4); struct smc_clc_msg_smcd { /* SMC-D GID information */ - u64 gid; /* ISM GID of requestor */ - u8 res[32]; + struct smc_clc_smcd_gid_chid ism; /* ISM native GID+CHID of requestor */ + __be16 v2_ext_offset; /* SMC Version 2 Extension Offset */ + u8 reserved[28]; +}; + +struct smc_clc_smcd_v2_extension { + u8 system_eid[SMC_MAX_EID_LEN]; + u8 reserved[16]; + struct smc_clc_smcd_gid_chid gidchid[0]; }; struct smc_clc_msg_proposal { /* clc proposal message sent by Linux */ @@ -109,11 +152,16 @@ struct smc_clc_msg_proposal { /* clc proposal message sent by Linux */ __be16 iparea_offset; /* offset to IP address information area */ } __aligned(4); +#define SMC_CLC_MAX_V6_PREFIX 8 + struct smc_clc_msg_proposal_area { struct smc_clc_msg_proposal pclc_base; struct smc_clc_msg_smcd pclc_smcd; struct smc_clc_msg_proposal_prefix pclc_prfx; struct smc_clc_ipv6_prefix pclc_prfx_ipv6[SMC_CLC_MAX_V6_PREFIX]; + struct smc_clc_v2_extension pclc_v2_ext; + struct smc_clc_smcd_v2_extension pclc_smcd_v2_ext; + struct smc_clc_smcd_gid_chid pclc_gidchids[SMC_MAX_ISM_DEVS]; struct smc_clc_msg_trail pclc_trl; }; @@ -134,11 +182,9 @@ struct smcr_clc_msg_accept_confirm { /* SMCR accept/confirm */ __be64 rmb_dma_addr; /* RMB virtual address */ u8 reserved2; u8 psn[3]; /* packet sequence number */ - struct smc_clc_msg_trail smcr_trl; - /* eye catcher "SMCR" EBCDIC */ } __packed; -struct smcd_clc_msg_accept_confirm { /* SMCD accept/confirm */ +struct smcd_clc_msg_accept_confirm_common { /* SMCD accept/confirm */ u64 gid; /* Sender GID */ u64 token; /* DMB token */ u8 dmbe_idx; /* DMBE index */ @@ -150,26 +196,63 @@ struct smcd_clc_msg_accept_confirm { /* SMCD accept/confirm */ dmbe_size : 4; #endif u16 reserved4; - u32 linkid; /* Link identifier */ - u32 reserved5[3]; - struct smc_clc_msg_trail smcd_trl; - /* eye catcher "SMCD" EBCDIC */ + __be32 linkid; /* Link identifier */ } __packed; +#define SMC_CLC_OS_ZOS 1 +#define SMC_CLC_OS_LINUX 2 +#define SMC_CLC_OS_AIX 3 + +struct smc_clc_first_contact_ext { + u8 reserved1; +#if defined(__BIG_ENDIAN_BITFIELD) + u8 os_type : 4, + release : 4; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u8 release : 4, + os_type : 4; +#endif + u8 reserved2[2]; + u8 hostname[SMC_MAX_HOSTNAME_LEN]; +}; + struct smc_clc_msg_accept_confirm { /* clc accept / confirm message */ struct smc_clc_msg_hdr hdr; union { struct smcr_clc_msg_accept_confirm r0; /* SMC-R */ - struct smcd_clc_msg_accept_confirm d0; /* SMC-D */ + struct { /* SMC-D */ + struct smcd_clc_msg_accept_confirm_common d0; + u32 reserved5[3]; + }; }; } __packed; /* format defined in RFC7609 */ +struct smc_clc_msg_accept_confirm_v2 { /* clc accept / confirm message */ + struct smc_clc_msg_hdr hdr; + union { + struct smcr_clc_msg_accept_confirm r0; /* SMC-R */ + struct { /* SMC-D */ + struct smcd_clc_msg_accept_confirm_common d0; + __be16 chid; + u8 eid[SMC_MAX_EID_LEN]; + u8 reserved5[8]; + }; + }; +}; + struct smc_clc_msg_decline { /* clc decline message */ struct smc_clc_msg_hdr hdr; u8 id_for_peer[SMC_SYSTEMID_LEN]; /* sender peer_id */ __be32 peer_diagnosis; /* diagnosis information */ - u8 reserved2[4]; - struct smc_clc_msg_trail trl; /* eye catcher "SMCR" EBCDIC */ +#if defined(__BIG_ENDIAN_BITFIELD) + u8 os_type : 4, + reserved : 4; +#elif defined(__LITTLE_ENDIAN_BITFIELD) + u8 reserved : 4, + os_type : 4; +#endif + u8 reserved2[3]; + struct smc_clc_msg_trail trl; /* eye catcher "SMCD" or "SMCR" EBCDIC */ } __aligned(4); /* determine start of the prefix area within the proposal message */ @@ -180,16 +263,58 @@ smc_clc_proposal_get_prefix(struct smc_clc_msg_proposal *pclc) ((u8 *)pclc + sizeof(*pclc) + ntohs(pclc->iparea_offset)); } +static inline bool smcr_indicated(int smc_type) +{ + return smc_type == SMC_TYPE_R || smc_type == SMC_TYPE_B; +} + +static inline bool smcd_indicated(int smc_type) +{ + return smc_type == SMC_TYPE_D || smc_type == SMC_TYPE_B; +} + /* get SMC-D info from proposal message */ static inline struct smc_clc_msg_smcd * smc_get_clc_msg_smcd(struct smc_clc_msg_proposal *prop) { - if (ntohs(prop->iparea_offset) != sizeof(struct smc_clc_msg_smcd)) + if (smcd_indicated(prop->hdr.typev1) && + ntohs(prop->iparea_offset) != sizeof(struct smc_clc_msg_smcd)) return NULL; return (struct smc_clc_msg_smcd *)(prop + 1); } +static inline struct smc_clc_v2_extension * +smc_get_clc_v2_ext(struct smc_clc_msg_proposal *prop) +{ + struct smc_clc_msg_smcd *prop_smcd = smc_get_clc_msg_smcd(prop); + + if (!prop_smcd || !ntohs(prop_smcd->v2_ext_offset)) + return NULL; + + return (struct smc_clc_v2_extension *) + ((u8 *)prop_smcd + + offsetof(struct smc_clc_msg_smcd, v2_ext_offset) + + sizeof(prop_smcd->v2_ext_offset) + + ntohs(prop_smcd->v2_ext_offset)); +} + +static inline struct smc_clc_smcd_v2_extension * +smc_get_clc_smcd_v2_ext(struct smc_clc_v2_extension *prop_v2ext) +{ + if (!prop_v2ext) + return NULL; + if (!ntohs(prop_v2ext->hdr.smcd_v2_ext_offset)) + return NULL; + + return (struct smc_clc_smcd_v2_extension *) + ((u8 *)prop_v2ext + + offsetof(struct smc_clc_v2_extension, hdr) + + offsetof(struct smc_clnt_opts_area_hdr, smcd_v2_ext_offset) + + sizeof(prop_v2ext->hdr.smcd_v2_ext_offset) + + ntohs(prop_v2ext->hdr.smcd_v2_ext_offset)); +} + struct smcd_dev; struct smc_init_info; @@ -197,10 +322,12 @@ int smc_clc_prfx_match(struct socket *clcsock, struct smc_clc_msg_proposal_prefix *prop); int smc_clc_wait_msg(struct smc_sock *smc, void *buf, int buflen, u8 expected_type, unsigned long timeout); -int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info); -int smc_clc_send_proposal(struct smc_sock *smc, int smc_type, - struct smc_init_info *ini); -int smc_clc_send_confirm(struct smc_sock *smc); -int smc_clc_send_accept(struct smc_sock *smc, bool srv_first_contact); +int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info, u8 version); +int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini); +int smc_clc_send_confirm(struct smc_sock *smc, bool clnt_first_contact, + u8 version); +int smc_clc_send_accept(struct smc_sock *smc, bool srv_first_contact, + u8 version); +void smc_clc_init(void) __init; #endif diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c index c811ae1a8add..f1dbb5025c0b 100644 --- a/net/smc/smc_core.c +++ b/net/smc/smc_core.c @@ -375,7 +375,8 @@ static int smc_lgr_create(struct smc_sock *smc, struct smc_init_info *ini) int i; if (ini->is_smcd && ini->vlan_id) { - if (smc_ism_get_vlan(ini->ism_dev, ini->vlan_id)) { + if (smc_ism_get_vlan(ini->ism_dev[ini->ism_selected], + ini->vlan_id)) { rc = SMC_CLC_DECL_ISMVLANERR; goto out; } @@ -412,13 +413,14 @@ static int smc_lgr_create(struct smc_sock *smc, struct smc_init_info *ini) lgr->conns_all = RB_ROOT; if (ini->is_smcd) { /* SMC-D specific settings */ - get_device(&ini->ism_dev->dev); - lgr->peer_gid = ini->ism_peer_gid; - lgr->smcd = ini->ism_dev; - lgr_list = &ini->ism_dev->lgr_list; + get_device(&ini->ism_dev[ini->ism_selected]->dev); + lgr->peer_gid = ini->ism_peer_gid[ini->ism_selected]; + lgr->smcd = ini->ism_dev[ini->ism_selected]; + lgr_list = &ini->ism_dev[ini->ism_selected]->lgr_list; lgr_lock = &lgr->smcd->lgr_lock; + lgr->smc_version = ini->smcd_version; lgr->peer_shutdown = 0; - atomic_inc(&ini->ism_dev->lgr_cnt); + atomic_inc(&ini->ism_dev[ini->ism_selected]->lgr_cnt); } else { /* SMC-R specific settings */ lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT; @@ -449,7 +451,7 @@ free_lgr: kfree(lgr); ism_put_vlan: if (ini->is_smcd && ini->vlan_id) - smc_ism_put_vlan(ini->ism_dev, ini->vlan_id); + smc_ism_put_vlan(ini->ism_dev[ini->ism_selected], ini->vlan_id); out: if (rc < 0) { if (rc == -ENOMEM) @@ -1288,8 +1290,10 @@ int smc_conn_create(struct smc_sock *smc, struct smc_init_info *ini) spinlock_t *lgr_lock; int rc = 0; - lgr_list = ini->is_smcd ? &ini->ism_dev->lgr_list : &smc_lgr_list.list; - lgr_lock = ini->is_smcd ? &ini->ism_dev->lgr_lock : &smc_lgr_list.lock; + lgr_list = ini->is_smcd ? &ini->ism_dev[ini->ism_selected]->lgr_list : + &smc_lgr_list.list; + lgr_lock = ini->is_smcd ? &ini->ism_dev[ini->ism_selected]->lgr_lock : + &smc_lgr_list.lock; ini->first_contact_local = 1; role = smc->listen_smc ? SMC_SERV : SMC_CLNT; if (role == SMC_CLNT && ini->first_contact_peer) @@ -1301,7 +1305,8 @@ int smc_conn_create(struct smc_sock *smc, struct smc_init_info *ini) list_for_each_entry(lgr, lgr_list, list) { write_lock_bh(&lgr->conns_lock); if ((ini->is_smcd ? - smcd_lgr_match(lgr, ini->ism_dev, ini->ism_peer_gid) : + smcd_lgr_match(lgr, ini->ism_dev[ini->ism_selected], + ini->ism_peer_gid[ini->ism_selected]) : smcr_lgr_match(lgr, ini->ib_lcl, role, ini->ib_clcqpn)) && !lgr->sync_err && lgr->vlan_id == ini->vlan_id && diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h index 37a5903789b0..f1e867ce2e63 100644 --- a/net/smc/smc_core.h +++ b/net/smc/smc_core.h @@ -231,6 +231,11 @@ struct smc_link_group { u8 freeing : 1; /* lgr is being freed */ bool is_smcd; /* SMC-R or SMC-D */ + u8 smc_version; + u8 negotiated_eid[SMC_MAX_EID_LEN]; + u8 peer_os; /* peer operating system */ + u8 peer_smc_release; + u8 peer_hostname[SMC_MAX_HOSTNAME_LEN]; union { struct { /* SMC-R */ enum smc_lgr_role role; @@ -291,6 +296,8 @@ struct smc_clc_msg_local; struct smc_init_info { u8 is_smcd; + u8 smc_type_v1; + u8 smc_type_v2; u8 first_contact_peer; u8 first_contact_local; unsigned short vlan_id; @@ -301,8 +308,12 @@ struct smc_init_info { u8 ib_port; u32 ib_clcqpn; /* SMC-D */ - u64 ism_peer_gid; - struct smcd_dev *ism_dev; + u64 ism_peer_gid[SMC_MAX_ISM_DEVS + 1]; + struct smcd_dev *ism_dev[SMC_MAX_ISM_DEVS + 1]; + u16 ism_chid[SMC_MAX_ISM_DEVS + 1]; + u8 ism_offered_cnt; /* # of ISM devices offered */ + u8 ism_selected; /* index of selected ISM dev*/ + u8 smcd_version; }; /* Find the connection associated with the given alert token in the link group. diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c index 998c525de785..e9a6487a42cb 100644 --- a/net/smc/smc_ism.c +++ b/net/smc/smc_ism.c @@ -21,7 +21,9 @@ struct smcd_dev_list smcd_dev_list = { .mutex = __MUTEX_INITIALIZER(smcd_dev_list.mutex) }; -/* Test if an ISM communication is possible. */ +bool smc_ism_v2_capable; + +/* Test if an ISM communication is possible - same CPC */ int smc_ism_cantalk(u64 peer_gid, unsigned short vlan_id, struct smcd_dev *smcd) { return smcd->ops->query_remote_gid(smcd, peer_gid, vlan_id ? 1 : 0, @@ -39,6 +41,16 @@ int smc_ism_write(struct smcd_dev *smcd, const struct smc_ism_position *pos, return rc < 0 ? rc : 0; } +void smc_ism_get_system_eid(struct smcd_dev *smcd, u8 **eid) +{ + smcd->ops->get_system_eid(smcd, eid); +} + +u16 smc_ism_get_chid(struct smcd_dev *smcd) +{ + return smcd->ops->get_chid(smcd); +} + /* Set a connection using this DMBE. */ void smc_ism_set_conn(struct smc_connection *conn) { @@ -319,7 +331,18 @@ EXPORT_SYMBOL_GPL(smcd_alloc_dev); int smcd_register_dev(struct smcd_dev *smcd) { mutex_lock(&smcd_dev_list.mutex); - list_add_tail(&smcd->list, &smcd_dev_list.list); + if (list_empty(&smcd_dev_list.list)) { + u8 *system_eid = NULL; + + smc_ism_get_system_eid(smcd, &system_eid); + if ((*system_eid) + 24 != '0' || (*system_eid) + 28 != '0') + smc_ism_v2_capable = true; + } + /* sort list: devices without pnetid before devices with pnetid */ + if (smcd->pnetid[0]) + list_add_tail(&smcd->list, &smcd_dev_list.list); + else + list_add(&smcd->list, &smcd_dev_list.list); mutex_unlock(&smcd_dev_list.mutex); pr_warn_ratelimited("smc: adding smcd device %s with pnetid %.16s%s\n", @@ -399,3 +422,8 @@ void smcd_handle_irq(struct smcd_dev *smcd, unsigned int dmbno) spin_unlock_irqrestore(&smcd->lock, flags); } EXPORT_SYMBOL_GPL(smcd_handle_irq); + +void __init smc_ism_init(void) +{ + smc_ism_v2_capable = false; +} diff --git a/net/smc/smc_ism.h b/net/smc/smc_ism.h index 81cc4537efd3..8048e09ddcf8 100644 --- a/net/smc/smc_ism.h +++ b/net/smc/smc_ism.h @@ -19,7 +19,10 @@ struct smcd_dev_list { /* List of SMCD devices */ struct mutex mutex; /* Protects list of devices */ }; -extern struct smcd_dev_list smcd_dev_list; /* list of smcd devices */ +extern struct smcd_dev_list smcd_dev_list; /* list of smcd devices */ +extern bool smc_ism_v2_capable; /* HW supports ISM V2 and thus + * System EID is defined + */ struct smc_ism_vlanid { /* VLAN id set on ISM device */ struct list_head list; @@ -47,4 +50,7 @@ int smc_ism_unregister_dmb(struct smcd_dev *dev, struct smc_buf_desc *dmb_desc); int smc_ism_write(struct smcd_dev *dev, const struct smc_ism_position *pos, void *data, size_t len); int smc_ism_signal_shutdown(struct smc_link_group *lgr); +void smc_ism_get_system_eid(struct smcd_dev *dev, u8 **eid); +u16 smc_ism_get_chid(struct smcd_dev *dev); +void smc_ism_init(void); #endif diff --git a/net/smc/smc_netns.h b/net/smc/smc_netns.h index e7a8fc4ae02f..0f4f35aa43ad 100644 --- a/net/smc/smc_netns.h +++ b/net/smc/smc_netns.h @@ -16,5 +16,6 @@ extern unsigned int smc_net_id; /* per-network namespace private data */ struct smc_net { struct smc_pnettable pnettable; + struct smc_pnetids_ndev pnetids_ndev; }; #endif diff --git a/net/smc/smc_pnet.c b/net/smc/smc_pnet.c index 70684c49510e..f3c18b991d35 100644 --- a/net/smc/smc_pnet.c +++ b/net/smc/smc_pnet.c @@ -29,8 +29,7 @@ #include "smc_ism.h" #include "smc_core.h" -#define SMC_ASCII_BLANK 32 - +static struct net_device *__pnet_find_base_ndev(struct net_device *ndev); static struct net_device *pnet_find_base_ndev(struct net_device *ndev); static const struct nla_policy smc_pnet_policy[SMC_PNETID_MAX + 1] = { @@ -73,14 +72,22 @@ struct smc_pnetentry { }; }; +/* Check if the pnetid is set */ +bool smc_pnet_is_pnetid_set(u8 *pnetid) +{ + if (pnetid[0] == 0 || pnetid[0] == _S) + return false; + return true; +} + /* Check if two given pnetids match */ static bool smc_pnet_match(u8 *pnetid1, u8 *pnetid2) { int i; for (i = 0; i < SMC_MAX_PNETID_LEN; i++) { - if ((pnetid1[i] == 0 || pnetid1[i] == SMC_ASCII_BLANK) && - (pnetid2[i] == 0 || pnetid2[i] == SMC_ASCII_BLANK)) + if ((pnetid1[i] == 0 || pnetid1[i] == _S) && + (pnetid2[i] == 0 || pnetid2[i] == _S)) break; if (pnetid1[i] != pnetid2[i]) return false; @@ -238,11 +245,10 @@ static int smc_pnet_remove_by_ndev(struct net_device *ndev) static bool smc_pnet_apply_ib(struct smc_ib_device *ib_dev, u8 ib_port, char *pnet_name) { - u8 pnet_null[SMC_MAX_PNETID_LEN] = {0}; bool applied = false; mutex_lock(&smc_ib_devices.mutex); - if (smc_pnet_match(ib_dev->pnetid[ib_port - 1], pnet_null)) { + if (!smc_pnet_is_pnetid_set(ib_dev->pnetid[ib_port - 1])) { memcpy(ib_dev->pnetid[ib_port - 1], pnet_name, SMC_MAX_PNETID_LEN); ib_dev->pnetid_by_user[ib_port - 1] = true; @@ -256,11 +262,10 @@ static bool smc_pnet_apply_ib(struct smc_ib_device *ib_dev, u8 ib_port, */ static bool smc_pnet_apply_smcd(struct smcd_dev *smcd_dev, char *pnet_name) { - u8 pnet_null[SMC_MAX_PNETID_LEN] = {0}; bool applied = false; mutex_lock(&smcd_dev_list.mutex); - if (smc_pnet_match(smcd_dev->pnetid, pnet_null)) { + if (!smc_pnet_is_pnetid_set(smcd_dev->pnetid)) { memcpy(smcd_dev->pnetid, pnet_name, SMC_MAX_PNETID_LEN); smcd_dev->pnetid_by_user = true; applied = true; @@ -708,10 +713,115 @@ static struct genl_family smc_pnet_nl_family __ro_after_init = { .n_ops = ARRAY_SIZE(smc_pnet_ops) }; +bool smc_pnet_is_ndev_pnetid(struct net *net, u8 *pnetid) +{ + struct smc_net *sn = net_generic(net, smc_net_id); + struct smc_pnetids_ndev_entry *pe; + bool rc = false; + + read_lock(&sn->pnetids_ndev.lock); + list_for_each_entry(pe, &sn->pnetids_ndev.list, list) { + if (smc_pnet_match(pnetid, pe->pnetid)) { + rc = true; + goto unlock; + } + } + +unlock: + read_unlock(&sn->pnetids_ndev.lock); + return rc; +} + +static int smc_pnet_add_pnetid(struct net *net, u8 *pnetid) +{ + struct smc_net *sn = net_generic(net, smc_net_id); + struct smc_pnetids_ndev_entry *pe, *pi; + + pe = kzalloc(sizeof(*pe), GFP_KERNEL); + if (!pe) + return -ENOMEM; + + write_lock(&sn->pnetids_ndev.lock); + list_for_each_entry(pi, &sn->pnetids_ndev.list, list) { + if (smc_pnet_match(pnetid, pe->pnetid)) { + refcount_inc(&pi->refcnt); + kfree(pe); + goto unlock; + } + } + refcount_set(&pe->refcnt, 1); + memcpy(pe->pnetid, pnetid, SMC_MAX_PNETID_LEN); + list_add_tail(&pe->list, &sn->pnetids_ndev.list); + +unlock: + write_unlock(&sn->pnetids_ndev.lock); + return 0; +} + +static void smc_pnet_remove_pnetid(struct net *net, u8 *pnetid) +{ + struct smc_net *sn = net_generic(net, smc_net_id); + struct smc_pnetids_ndev_entry *pe, *pe2; + + write_lock(&sn->pnetids_ndev.lock); + list_for_each_entry_safe(pe, pe2, &sn->pnetids_ndev.list, list) { + if (smc_pnet_match(pnetid, pe->pnetid)) { + if (refcount_dec_and_test(&pe->refcnt)) { + list_del(&pe->list); + kfree(pe); + } + break; + } + } + write_unlock(&sn->pnetids_ndev.lock); +} + +static void smc_pnet_add_base_pnetid(struct net *net, struct net_device *dev, + u8 *ndev_pnetid) +{ + struct net_device *base_dev; + + base_dev = __pnet_find_base_ndev(dev); + if (base_dev->flags & IFF_UP && + !smc_pnetid_by_dev_port(base_dev->dev.parent, base_dev->dev_port, + ndev_pnetid)) { + /* add to PNETIDs list */ + smc_pnet_add_pnetid(net, ndev_pnetid); + } +} + +/* create initial list of netdevice pnetids */ +static void smc_pnet_create_pnetids_list(struct net *net) +{ + u8 ndev_pnetid[SMC_MAX_PNETID_LEN]; + struct net_device *dev; + + rtnl_lock(); + for_each_netdev(net, dev) + smc_pnet_add_base_pnetid(net, dev, ndev_pnetid); + rtnl_unlock(); +} + +/* clean up list of netdevice pnetids */ +static void smc_pnet_destroy_pnetids_list(struct net *net) +{ + struct smc_net *sn = net_generic(net, smc_net_id); + struct smc_pnetids_ndev_entry *pe, *temp_pe; + + write_lock(&sn->pnetids_ndev.lock); + list_for_each_entry_safe(pe, temp_pe, &sn->pnetids_ndev.list, list) { + list_del(&pe->list); + kfree(pe); + } + write_unlock(&sn->pnetids_ndev.lock); +} + static int smc_pnet_netdev_event(struct notifier_block *this, unsigned long event, void *ptr) { struct net_device *event_dev = netdev_notifier_info_to_dev(ptr); + struct net *net = dev_net(event_dev); + u8 ndev_pnetid[SMC_MAX_PNETID_LEN]; switch (event) { case NETDEV_REBOOT: @@ -721,6 +831,17 @@ static int smc_pnet_netdev_event(struct notifier_block *this, case NETDEV_REGISTER: smc_pnet_add_by_ndev(event_dev); return NOTIFY_OK; + case NETDEV_UP: + smc_pnet_add_base_pnetid(net, event_dev, ndev_pnetid); + return NOTIFY_OK; + case NETDEV_DOWN: + event_dev = __pnet_find_base_ndev(event_dev); + if (!smc_pnetid_by_dev_port(event_dev->dev.parent, + event_dev->dev_port, ndev_pnetid)) { + /* remove from PNETIDs list */ + smc_pnet_remove_pnetid(net, ndev_pnetid); + } + return NOTIFY_OK; default: return NOTIFY_DONE; } @@ -735,9 +856,14 @@ int smc_pnet_net_init(struct net *net) { struct smc_net *sn = net_generic(net, smc_net_id); struct smc_pnettable *pnettable = &sn->pnettable; + struct smc_pnetids_ndev *pnetids_ndev = &sn->pnetids_ndev; INIT_LIST_HEAD(&pnettable->pnetlist); rwlock_init(&pnettable->lock); + INIT_LIST_HEAD(&pnetids_ndev->list); + rwlock_init(&pnetids_ndev->lock); + + smc_pnet_create_pnetids_list(net); return 0; } @@ -752,6 +878,7 @@ int __init smc_pnet_init(void) rc = register_netdevice_notifier(&smc_netdev_notifier); if (rc) genl_unregister_family(&smc_pnet_nl_family); + return rc; } @@ -760,6 +887,7 @@ void smc_pnet_net_exit(struct net *net) { /* flush pnet table */ smc_pnet_remove_by_pnetid(net, NULL); + smc_pnet_destroy_pnetids_list(net); } void smc_pnet_exit(void) @@ -768,16 +896,11 @@ void smc_pnet_exit(void) genl_unregister_family(&smc_pnet_nl_family); } -/* Determine one base device for stacked net devices. - * If the lower device level contains more than one devices - * (for instance with bonding slaves), just the first device - * is used to reach a base device. - */ -static struct net_device *pnet_find_base_ndev(struct net_device *ndev) +static struct net_device *__pnet_find_base_ndev(struct net_device *ndev) { int i, nest_lvl; - rtnl_lock(); + ASSERT_RTNL(); nest_lvl = ndev->lower_level; for (i = 0; i < nest_lvl; i++) { struct list_head *lower = &ndev->adj_list.lower; @@ -787,6 +910,18 @@ static struct net_device *pnet_find_base_ndev(struct net_device *ndev) lower = lower->next; ndev = netdev_lower_get_next(ndev, &lower); } + return ndev; +} + +/* Determine one base device for stacked net devices. + * If the lower device level contains more than one devices + * (for instance with bonding slaves), just the first device + * is used to reach a base device. + */ +static struct net_device *pnet_find_base_ndev(struct net_device *ndev) +{ + rtnl_lock(); + ndev = __pnet_find_base_ndev(ndev); rtnl_unlock(); return ndev; } @@ -929,10 +1064,10 @@ static void smc_pnet_find_ism_by_pnetid(struct net_device *ndev, list_for_each_entry(ismdev, &smcd_dev_list.list, list) { if (smc_pnet_match(ismdev->pnetid, ndev_pnetid) && !ismdev->going_away && - (!ini->ism_peer_gid || - !smc_ism_cantalk(ini->ism_peer_gid, ini->vlan_id, + (!ini->ism_peer_gid[0] || + !smc_ism_cantalk(ini->ism_peer_gid[0], ini->vlan_id, ismdev))) { - ini->ism_dev = ismdev; + ini->ism_dev[0] = ismdev; break; } } @@ -966,7 +1101,7 @@ void smc_pnet_find_ism_resource(struct sock *sk, struct smc_init_info *ini) { struct dst_entry *dst = sk_dst_get(sk); - ini->ism_dev = NULL; + ini->ism_dev[0] = NULL; if (!dst) goto out; if (!dst->dev) diff --git a/net/smc/smc_pnet.h b/net/smc/smc_pnet.h index 811a65986691..14039272f7e4 100644 --- a/net/smc/smc_pnet.h +++ b/net/smc/smc_pnet.h @@ -12,6 +12,8 @@ #ifndef _SMC_PNET_H #define _SMC_PNET_H +#include <net/smc.h> + #if IS_ENABLED(CONFIG_HAVE_PNETID) #include <asm/pnet.h> #endif @@ -31,6 +33,17 @@ struct smc_pnettable { struct list_head pnetlist; }; +struct smc_pnetids_ndev { /* list of pnetids for net devices in UP state*/ + struct list_head list; + rwlock_t lock; +}; + +struct smc_pnetids_ndev_entry { + struct list_head list; + u8 pnetid[SMC_MAX_PNETID_LEN]; + refcount_t refcnt; +}; + static inline int smc_pnetid_by_dev_port(struct device *dev, unsigned short port, u8 *pnetid) { @@ -52,4 +65,6 @@ int smc_pnetid_by_table_smcd(struct smcd_dev *smcd); void smc_pnet_find_alt_roce(struct smc_link_group *lgr, struct smc_init_info *ini, struct smc_ib_device *known_dev); +bool smc_pnet_is_ndev_pnetid(struct net *net, u8 *pnetid); +bool smc_pnet_is_pnetid_set(u8 *pnetid); #endif diff --git a/net/wireless/core.c b/net/wireless/core.c index 354b0ccbdc24..9f23923e8d29 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -236,7 +236,9 @@ void cfg80211_stop_p2p_device(struct cfg80211_registered_device *rdev, rdev->opencount--; if (rdev->scan_req && rdev->scan_req->wdev == wdev) { - if (WARN_ON(!rdev->scan_req->notified)) + if (WARN_ON(!rdev->scan_req->notified && + (!rdev->int_scan_req || + !rdev->int_scan_req->notified))) rdev->scan_req->info.aborted = true; ___cfg80211_scan_done(rdev, false); } @@ -1336,7 +1338,9 @@ static int cfg80211_netdev_notifier_call(struct notifier_block *nb, case NETDEV_DOWN: cfg80211_update_iface_num(rdev, wdev->iftype, -1); if (rdev->scan_req && rdev->scan_req->wdev == wdev) { - if (WARN_ON(!rdev->scan_req->notified)) + if (WARN_ON(!rdev->scan_req->notified && + (!rdev->int_scan_req || + !rdev->int_scan_req->notified))) rdev->scan_req->info.aborted = true; ___cfg80211_scan_done(rdev, false); } diff --git a/net/wireless/core.h b/net/wireless/core.h index 2ebc2a66680d..e1ec9ac8e608 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -3,7 +3,7 @@ * Wireless configuration interface internals. * * Copyright 2006-2010 Johannes Berg <johannes@sipsolutions.net> - * Copyright (C) 2018-2019 Intel Corporation + * Copyright (C) 2018-2020 Intel Corporation */ #ifndef __NET_WIRELESS_CORE_H #define __NET_WIRELESS_CORE_H @@ -72,6 +72,7 @@ struct cfg80211_registered_device { u32 bss_generation; u32 bss_entries; struct cfg80211_scan_request *scan_req; /* protected by RTNL */ + struct cfg80211_scan_request *int_scan_req; struct sk_buff *scan_msg; struct list_head sched_scan_req_list; time64_t suspend_at; @@ -457,6 +458,8 @@ void cfg80211_process_wdev_events(struct wireless_dev *wdev); bool cfg80211_does_bw_fit_range(const struct ieee80211_freq_range *freq_range, u32 center_freq_khz, u32 bw_khz); +int cfg80211_scan(struct cfg80211_registered_device *rdev); + extern struct work_struct cfg80211_disconnect_work; /** diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c index db7333e20dd7..0ac820780437 100644 --- a/net/wireless/mlme.c +++ b/net/wireless/mlme.c @@ -30,6 +30,15 @@ void cfg80211_rx_assoc_resp(struct net_device *dev, struct cfg80211_bss *bss, struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)buf; struct cfg80211_connect_resp_params cr; + const u8 *resp_ie = mgmt->u.assoc_resp.variable; + size_t resp_ie_len = len - offsetof(struct ieee80211_mgmt, + u.assoc_resp.variable); + + if (bss->channel->band == NL80211_BAND_S1GHZ) { + resp_ie = (u8 *)&mgmt->u.s1g_assoc_resp.variable; + resp_ie_len = len - offsetof(struct ieee80211_mgmt, + u.s1g_assoc_resp.variable); + } memset(&cr, 0, sizeof(cr)); cr.status = (int)le16_to_cpu(mgmt->u.assoc_resp.status_code); @@ -37,9 +46,8 @@ void cfg80211_rx_assoc_resp(struct net_device *dev, struct cfg80211_bss *bss, cr.bss = bss; cr.req_ie = req_ies; cr.req_ie_len = req_ies_len; - cr.resp_ie = mgmt->u.assoc_resp.variable; - cr.resp_ie_len = - len - offsetof(struct ieee80211_mgmt, u.assoc_resp.variable); + cr.resp_ie = resp_ie; + cr.resp_ie_len = resp_ie_len; cr.timeout_reason = NL80211_TIMEOUT_UNSPECIFIED; trace_cfg80211_send_rx_assoc(dev, bss); diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 1a212db7a300..91d4550677d0 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -329,6 +329,13 @@ he_obss_pd_policy[NL80211_HE_OBSS_PD_ATTR_MAX + 1] = { NLA_POLICY_RANGE(NLA_U8, 1, 20), [NL80211_HE_OBSS_PD_ATTR_MAX_OFFSET] = NLA_POLICY_RANGE(NLA_U8, 1, 20), + [NL80211_HE_OBSS_PD_ATTR_NON_SRG_MAX_OFFSET] = + NLA_POLICY_RANGE(NLA_U8, 1, 20), + [NL80211_HE_OBSS_PD_ATTR_BSS_COLOR_BITMAP] = + NLA_POLICY_EXACT_LEN(8), + [NL80211_HE_OBSS_PD_ATTR_PARTIAL_BSSID_BITMAP] = + NLA_POLICY_EXACT_LEN(8), + [NL80211_HE_OBSS_PD_ATTR_SR_CTRL] = { .type = NLA_U8 }, }; static const struct nla_policy @@ -704,6 +711,10 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { NLA_POLICY_NESTED(nl80211_fils_discovery_policy), [NL80211_ATTR_UNSOL_BCAST_PROBE_RESP] = NLA_POLICY_NESTED(nl80211_unsol_bcast_probe_resp_policy), + [NL80211_ATTR_S1G_CAPABILITY] = + NLA_POLICY_EXACT_LEN(IEEE80211_S1G_CAPABILITY_LEN), + [NL80211_ATTR_S1G_CAPABILITY_MASK] = + NLA_POLICY_EXACT_LEN(IEEE80211_S1G_CAPABILITY_LEN), }; /* policy for the key attributes */ @@ -973,6 +984,8 @@ static int nl80211_msg_put_channel(struct sk_buff *msg, struct wiphy *wiphy, if (!large && chan->flags & (IEEE80211_CHAN_NO_10MHZ | IEEE80211_CHAN_NO_20MHZ)) return 0; + if (!large && chan->freq_offset) + return 0; if (nla_put_u32(msg, NL80211_FREQUENCY_ATTR_FREQ, chan->center_freq)) @@ -1665,7 +1678,8 @@ nl80211_send_iftype_data(struct sk_buff *msg, } static int nl80211_send_band_rateinfo(struct sk_buff *msg, - struct ieee80211_supported_band *sband) + struct ieee80211_supported_band *sband, + bool large) { struct nlattr *nl_rates, *nl_rate; struct ieee80211_rate *rate; @@ -1693,7 +1707,7 @@ static int nl80211_send_band_rateinfo(struct sk_buff *msg, sband->vht_cap.cap))) return -ENOBUFS; - if (sband->n_iftype_data) { + if (large && sband->n_iftype_data) { struct nlattr *nl_iftype_data = nla_nest_start_noflag(msg, NL80211_BAND_ATTR_IFTYPE_DATA); @@ -1721,7 +1735,7 @@ static int nl80211_send_band_rateinfo(struct sk_buff *msg, } /* add EDMG info */ - if (sband->edmg_cap.channels && + if (large && sband->edmg_cap.channels && (nla_put_u8(msg, NL80211_BAND_ATTR_EDMG_CHANNELS, sband->edmg_cap.channels) || nla_put_u8(msg, NL80211_BAND_ATTR_EDMG_BW_CONFIG, @@ -2139,13 +2153,7 @@ static int nl80211_send_wiphy(struct cfg80211_registered_device *rdev, nla_put_u16(msg, NL80211_ATTR_MAX_SCHED_SCAN_IE_LEN, rdev->wiphy.max_sched_scan_ie_len) || nla_put_u8(msg, NL80211_ATTR_MAX_MATCH_SETS, - rdev->wiphy.max_match_sets) || - nla_put_u32(msg, NL80211_ATTR_MAX_NUM_SCHED_SCAN_PLANS, - rdev->wiphy.max_sched_scan_plans) || - nla_put_u32(msg, NL80211_ATTR_MAX_SCAN_PLAN_INTERVAL, - rdev->wiphy.max_sched_scan_plan_interval) || - nla_put_u32(msg, NL80211_ATTR_MAX_SCAN_PLAN_ITERATIONS, - rdev->wiphy.max_sched_scan_plan_iterations)) + rdev->wiphy.max_match_sets)) goto nla_put_failure; if ((rdev->wiphy.flags & WIPHY_FLAG_IBSS_RSN) && @@ -2235,6 +2243,10 @@ static int nl80211_send_wiphy(struct cfg80211_registered_device *rdev, band < NUM_NL80211_BANDS; band++) { struct ieee80211_supported_band *sband; + /* omit higher bands for ancient software */ + if (band > NL80211_BAND_5GHZ && !state->split) + break; + sband = rdev->wiphy.bands[band]; if (!sband) @@ -2246,7 +2258,8 @@ static int nl80211_send_wiphy(struct cfg80211_registered_device *rdev, switch (state->chan_start) { case 0: - if (nl80211_send_band_rateinfo(msg, sband)) + if (nl80211_send_band_rateinfo(msg, sband, + state->split)) goto nla_put_failure; state->chan_start++; if (state->split) @@ -2348,8 +2361,6 @@ static int nl80211_send_wiphy(struct cfg80211_registered_device *rdev, nla_put_flag(msg, NL80211_ATTR_OFFCHANNEL_TX_OK)) goto nla_put_failure; - if (nl80211_send_mgmt_stypes(msg, mgmt_stypes)) - goto nla_put_failure; state->split_start++; if (state->split) break; @@ -2417,9 +2428,23 @@ static int nl80211_send_wiphy(struct cfg80211_registered_device *rdev, * case we'll continue with more data in the next round, * but break unconditionally so unsplit data stops here. */ - state->split_start++; + if (state->split) + state->split_start++; + else + state->split_start = 0; break; case 9: + if (nl80211_send_mgmt_stypes(msg, mgmt_stypes)) + goto nla_put_failure; + + if (nla_put_u32(msg, NL80211_ATTR_MAX_NUM_SCHED_SCAN_PLANS, + rdev->wiphy.max_sched_scan_plans) || + nla_put_u32(msg, NL80211_ATTR_MAX_SCAN_PLAN_INTERVAL, + rdev->wiphy.max_sched_scan_plan_interval) || + nla_put_u32(msg, NL80211_ATTR_MAX_SCAN_PLAN_ITERATIONS, + rdev->wiphy.max_sched_scan_plan_iterations)) + goto nla_put_failure; + if (rdev->wiphy.extended_capabilities && (nla_put(msg, NL80211_ATTR_EXT_CAPA, rdev->wiphy.extended_capabilities_len, @@ -4853,18 +4878,34 @@ static int nl80211_parse_he_obss_pd(struct nlattr *attrs, if (err) return err; - if (!tb[NL80211_HE_OBSS_PD_ATTR_MIN_OFFSET] || - !tb[NL80211_HE_OBSS_PD_ATTR_MAX_OFFSET]) + if (!tb[NL80211_HE_OBSS_PD_ATTR_SR_CTRL]) return -EINVAL; - he_obss_pd->min_offset = - nla_get_u32(tb[NL80211_HE_OBSS_PD_ATTR_MIN_OFFSET]); - he_obss_pd->max_offset = - nla_get_u32(tb[NL80211_HE_OBSS_PD_ATTR_MAX_OFFSET]); + he_obss_pd->sr_ctrl = nla_get_u8(tb[NL80211_HE_OBSS_PD_ATTR_SR_CTRL]); - if (he_obss_pd->min_offset >= he_obss_pd->max_offset) + if (tb[NL80211_HE_OBSS_PD_ATTR_MIN_OFFSET]) + he_obss_pd->min_offset = + nla_get_u8(tb[NL80211_HE_OBSS_PD_ATTR_MIN_OFFSET]); + if (tb[NL80211_HE_OBSS_PD_ATTR_MAX_OFFSET]) + he_obss_pd->max_offset = + nla_get_u8(tb[NL80211_HE_OBSS_PD_ATTR_MAX_OFFSET]); + if (tb[NL80211_HE_OBSS_PD_ATTR_NON_SRG_MAX_OFFSET]) + he_obss_pd->non_srg_max_offset = + nla_get_u8(tb[NL80211_HE_OBSS_PD_ATTR_NON_SRG_MAX_OFFSET]); + + if (he_obss_pd->min_offset > he_obss_pd->max_offset) return -EINVAL; + if (tb[NL80211_HE_OBSS_PD_ATTR_BSS_COLOR_BITMAP]) + memcpy(he_obss_pd->bss_color_bitmap, + nla_data(tb[NL80211_HE_OBSS_PD_ATTR_BSS_COLOR_BITMAP]), + sizeof(he_obss_pd->bss_color_bitmap)); + + if (tb[NL80211_HE_OBSS_PD_ATTR_PARTIAL_BSSID_BITMAP]) + memcpy(he_obss_pd->partial_bssid_bitmap, + nla_data(tb[NL80211_HE_OBSS_PD_ATTR_PARTIAL_BSSID_BITMAP]), + sizeof(he_obss_pd->partial_bssid_bitmap)); + he_obss_pd->enable = true; return 0; @@ -8236,7 +8277,7 @@ static int nl80211_trigger_scan(struct sk_buff *skb, struct genl_info *info) request->scan_start = jiffies; rdev->scan_req = request; - err = rdev_scan(rdev, request); + err = cfg80211_scan(rdev); if (err) goto out_free; @@ -9315,6 +9356,11 @@ static int nl80211_send_survey(struct sk_buff *msg, u32 portid, u32 seq, survey->channel->center_freq)) goto nla_put_failure; + if (survey->channel && survey->channel->freq_offset && + nla_put_u32(msg, NL80211_SURVEY_INFO_FREQUENCY_OFFSET, + survey->channel->freq_offset)) + goto nla_put_failure; + if ((survey->filled & SURVEY_INFO_NOISE_DBM) && nla_put_u8(msg, NL80211_SURVEY_INFO_NOISE, survey->noise)) goto nla_put_failure; @@ -9792,6 +9838,22 @@ static int nl80211_associate(struct sk_buff *skb, struct genl_info *info) nla_data(info->attrs[NL80211_ATTR_FILS_NONCES]); } + if (info->attrs[NL80211_ATTR_S1G_CAPABILITY_MASK]) { + if (!info->attrs[NL80211_ATTR_S1G_CAPABILITY]) + return -EINVAL; + memcpy(&req.s1g_capa_mask, + nla_data(info->attrs[NL80211_ATTR_S1G_CAPABILITY_MASK]), + sizeof(req.s1g_capa_mask)); + } + + if (info->attrs[NL80211_ATTR_S1G_CAPABILITY]) { + if (!info->attrs[NL80211_ATTR_S1G_CAPABILITY_MASK]) + return -EINVAL; + memcpy(&req.s1g_capa, + nla_data(info->attrs[NL80211_ATTR_S1G_CAPABILITY]), + sizeof(req.s1g_capa)); + } + err = nl80211_crypto_settings(rdev, info, &req.crypto, 1); if (!err) { wdev_lock(dev->ieee80211_ptr); @@ -15518,6 +15580,7 @@ static int nl80211_add_scan_req(struct sk_buff *msg, struct cfg80211_scan_request *req = rdev->scan_req; struct nlattr *nest; int i; + struct cfg80211_scan_info *info; if (WARN_ON(!req)) return 0; @@ -15561,11 +15624,13 @@ static int nl80211_add_scan_req(struct sk_buff *msg, nla_put_u32(msg, NL80211_ATTR_SCAN_FLAGS, req->flags)) goto nla_put_failure; - if (req->info.scan_start_tsf && + info = rdev->int_scan_req ? &rdev->int_scan_req->info : + &rdev->scan_req->info; + if (info->scan_start_tsf && (nla_put_u64_64bit(msg, NL80211_ATTR_SCAN_START_TIME_TSF, - req->info.scan_start_tsf, NL80211_BSS_PAD) || + info->scan_start_tsf, NL80211_BSS_PAD) || nla_put(msg, NL80211_ATTR_SCAN_START_TIME_TSF_BSSID, ETH_ALEN, - req->info.tsf_bssid))) + info->tsf_bssid))) goto nla_put_failure; return 0; diff --git a/net/wireless/radiotap.c b/net/wireless/radiotap.c index d5e28239e030..36f1b59a78bf 100644 --- a/net/wireless/radiotap.c +++ b/net/wireless/radiotap.c @@ -59,6 +59,7 @@ static const struct ieee80211_radiotap_namespace radiotap_ns = { * @iterator: radiotap_iterator to initialize * @radiotap_header: radiotap header to parse * @max_length: total length we can parse into (eg, whole packet length) + * @vns: vendor namespaces to parse * * Returns: 0 or a negative error code if there is a problem. * diff --git a/net/wireless/reg.c b/net/wireless/reg.c index 6043a9d33d61..3dab859641e1 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -1616,8 +1616,8 @@ static const struct ieee80211_reg_rule * __freq_reg_info(struct wiphy *wiphy, u32 center_freq, u32 min_bw) { const struct ieee80211_regdomain *regd = reg_get_regdomain(wiphy); - const struct ieee80211_reg_rule *reg_rule = NULL; const u32 bws[] = {0, 1, 2, 4, 5, 8, 10, 16, 20}; + const struct ieee80211_reg_rule *reg_rule; int i = ARRAY_SIZE(bws) - 1; u32 bw; diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 84fc8ab16dd2..8d0e49c46db3 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -5,7 +5,7 @@ * Copyright 2008 Johannes Berg <johannes@sipsolutions.net> * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2016 Intel Deutschland GmbH - * Copyright (C) 2018-2019 Intel Corporation + * Copyright (C) 2018-2020 Intel Corporation */ #include <linux/kernel.h> #include <linux/slab.h> @@ -14,6 +14,8 @@ #include <linux/wireless.h> #include <linux/nl80211.h> #include <linux/etherdevice.h> +#include <linux/crc32.h> +#include <linux/bitfield.h> #include <net/arp.h> #include <net/cfg80211.h> #include <net/cfg80211-wext.h> @@ -74,6 +76,43 @@ MODULE_PARM_DESC(bss_entries_limit, #define IEEE80211_SCAN_RESULT_EXPIRE (30 * HZ) +/** + * struct cfg80211_colocated_ap - colocated AP information + * + * @list: linked list to all colocated aPS + * @bssid: BSSID of the reported AP + * @ssid: SSID of the reported AP + * @ssid_len: length of the ssid + * @center_freq: frequency the reported AP is on + * @unsolicited_probe: the reported AP is part of an ESS, where all the APs + * that operate in the same channel as the reported AP and that might be + * detected by a STA receiving this frame, are transmitting unsolicited + * Probe Response frames every 20 TUs + * @oct_recommended: OCT is recommended to exchange MMPDUs with the reported AP + * @same_ssid: the reported AP has the same SSID as the reporting AP + * @multi_bss: the reported AP is part of a multiple BSSID set + * @transmitted_bssid: the reported AP is the transmitting BSSID + * @colocated_ess: all the APs that share the same ESS as the reported AP are + * colocated and can be discovered via legacy bands. + * @short_ssid_valid: short_ssid is valid and can be used + * @short_ssid: the short SSID for this SSID + */ +struct cfg80211_colocated_ap { + struct list_head list; + u8 bssid[ETH_ALEN]; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + size_t ssid_len; + u32 short_ssid; + u32 center_freq; + u8 unsolicited_probe:1, + oct_recommended:1, + same_ssid:1, + multi_bss:1, + transmitted_bssid:1, + colocated_ess:1, + short_ssid_valid:1; +}; + static void bss_free(struct cfg80211_internal_bss *bss) { struct cfg80211_bss_ies *ies; @@ -448,10 +487,433 @@ static bool cfg80211_bss_expire_oldest(struct cfg80211_registered_device *rdev) return ret; } +static u8 cfg80211_parse_bss_param(u8 data, + struct cfg80211_colocated_ap *coloc_ap) +{ + coloc_ap->oct_recommended = + u8_get_bits(data, IEEE80211_RNR_TBTT_PARAMS_OCT_RECOMMENDED); + coloc_ap->same_ssid = + u8_get_bits(data, IEEE80211_RNR_TBTT_PARAMS_SAME_SSID); + coloc_ap->multi_bss = + u8_get_bits(data, IEEE80211_RNR_TBTT_PARAMS_MULTI_BSSID); + coloc_ap->transmitted_bssid = + u8_get_bits(data, IEEE80211_RNR_TBTT_PARAMS_TRANSMITTED_BSSID); + coloc_ap->unsolicited_probe = + u8_get_bits(data, IEEE80211_RNR_TBTT_PARAMS_PROBE_ACTIVE); + coloc_ap->colocated_ess = + u8_get_bits(data, IEEE80211_RNR_TBTT_PARAMS_COLOC_ESS); + + return u8_get_bits(data, IEEE80211_RNR_TBTT_PARAMS_COLOC_AP); +} + +static int cfg80211_calc_short_ssid(const struct cfg80211_bss_ies *ies, + const struct element **elem, u32 *s_ssid) +{ + + *elem = cfg80211_find_elem(WLAN_EID_SSID, ies->data, ies->len); + if (!*elem || (*elem)->datalen > IEEE80211_MAX_SSID_LEN) + return -EINVAL; + + *s_ssid = ~crc32_le(~0, (*elem)->data, (*elem)->datalen); + return 0; +} + +static void cfg80211_free_coloc_ap_list(struct list_head *coloc_ap_list) +{ + struct cfg80211_colocated_ap *ap, *tmp_ap; + + list_for_each_entry_safe(ap, tmp_ap, coloc_ap_list, list) { + list_del(&ap->list); + kfree(ap); + } +} + +static int cfg80211_parse_ap_info(struct cfg80211_colocated_ap *entry, + const u8 *pos, u8 length, + const struct element *ssid_elem, + int s_ssid_tmp) +{ + /* skip the TBTT offset */ + pos++; + + memcpy(entry->bssid, pos, ETH_ALEN); + pos += ETH_ALEN; + + if (length == IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM) { + memcpy(&entry->short_ssid, pos, + sizeof(entry->short_ssid)); + entry->short_ssid_valid = true; + pos += 4; + } + + /* skip non colocated APs */ + if (!cfg80211_parse_bss_param(*pos, entry)) + return -EINVAL; + pos++; + + if (length == IEEE80211_TBTT_INFO_OFFSET_BSSID_BSS_PARAM) { + /* + * no information about the short ssid. Consider the entry valid + * for now. It would later be dropped in case there are explicit + * SSIDs that need to be matched + */ + if (!entry->same_ssid) + return 0; + } + + if (entry->same_ssid) { + entry->short_ssid = s_ssid_tmp; + entry->short_ssid_valid = true; + + /* + * This is safe because we validate datalen in + * cfg80211_parse_colocated_ap(), before calling this + * function. + */ + memcpy(&entry->ssid, &ssid_elem->data, + ssid_elem->datalen); + entry->ssid_len = ssid_elem->datalen; + } + return 0; +} + +static int cfg80211_parse_colocated_ap(const struct cfg80211_bss_ies *ies, + struct list_head *list) +{ + struct ieee80211_neighbor_ap_info *ap_info; + const struct element *elem, *ssid_elem; + const u8 *pos, *end; + u32 s_ssid_tmp; + int n_coloc = 0, ret; + LIST_HEAD(ap_list); + + elem = cfg80211_find_elem(WLAN_EID_REDUCED_NEIGHBOR_REPORT, ies->data, + ies->len); + if (!elem || elem->datalen > IEEE80211_MAX_SSID_LEN) + return 0; + + pos = elem->data; + end = pos + elem->datalen; + + ret = cfg80211_calc_short_ssid(ies, &ssid_elem, &s_ssid_tmp); + if (ret) + return ret; + + /* RNR IE may contain more than one NEIGHBOR_AP_INFO */ + while (pos + sizeof(*ap_info) <= end) { + enum nl80211_band band; + int freq; + u8 length, i, count; + + ap_info = (void *)pos; + count = u8_get_bits(ap_info->tbtt_info_hdr, + IEEE80211_AP_INFO_TBTT_HDR_COUNT) + 1; + length = ap_info->tbtt_info_len; + + pos += sizeof(*ap_info); + + if (!ieee80211_operating_class_to_band(ap_info->op_class, + &band)) + break; + + freq = ieee80211_channel_to_frequency(ap_info->channel, band); + + if (end - pos < count * ap_info->tbtt_info_len) + break; + + /* + * TBTT info must include bss param + BSSID + + * (short SSID or same_ssid bit to be set). + * ignore other options, and move to the + * next AP info + */ + if (band != NL80211_BAND_6GHZ || + (length != IEEE80211_TBTT_INFO_OFFSET_BSSID_BSS_PARAM && + length < IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM)) { + pos += count * ap_info->tbtt_info_len; + continue; + } + + for (i = 0; i < count; i++) { + struct cfg80211_colocated_ap *entry; + + entry = kzalloc(sizeof(*entry) + IEEE80211_MAX_SSID_LEN, + GFP_ATOMIC); + + if (!entry) + break; + + entry->center_freq = freq; + + if (!cfg80211_parse_ap_info(entry, pos, length, + ssid_elem, s_ssid_tmp)) { + n_coloc++; + list_add_tail(&entry->list, &ap_list); + } else { + kfree(entry); + } + + pos += ap_info->tbtt_info_len; + } + } + + if (pos != end) { + cfg80211_free_coloc_ap_list(&ap_list); + return 0; + } + + list_splice_tail(&ap_list, list); + return n_coloc; +} + +static void cfg80211_scan_req_add_chan(struct cfg80211_scan_request *request, + struct ieee80211_channel *chan, + bool add_to_6ghz) +{ + int i; + u32 n_channels = request->n_channels; + struct cfg80211_scan_6ghz_params *params = + &request->scan_6ghz_params[request->n_6ghz_params]; + + for (i = 0; i < n_channels; i++) { + if (request->channels[i] == chan) { + if (add_to_6ghz) + params->channel_idx = i; + return; + } + } + + request->channels[n_channels] = chan; + if (add_to_6ghz) + request->scan_6ghz_params[request->n_6ghz_params].channel_idx = + n_channels; + + request->n_channels++; +} + +static bool cfg80211_find_ssid_match(struct cfg80211_colocated_ap *ap, + struct cfg80211_scan_request *request) +{ + u8 i; + u32 s_ssid; + + for (i = 0; i < request->n_ssids; i++) { + /* wildcard ssid in the scan request */ + if (!request->ssids[i].ssid_len) + return true; + + if (ap->ssid_len && + ap->ssid_len == request->ssids[i].ssid_len) { + if (!memcmp(request->ssids[i].ssid, ap->ssid, + ap->ssid_len)) + return true; + } else if (ap->short_ssid_valid) { + s_ssid = ~crc32_le(~0, request->ssids[i].ssid, + request->ssids[i].ssid_len); + + if (ap->short_ssid == s_ssid) + return true; + } + } + + return false; +} + +static int cfg80211_scan_6ghz(struct cfg80211_registered_device *rdev) +{ + u8 i; + struct cfg80211_colocated_ap *ap; + int n_channels, count = 0, err; + struct cfg80211_scan_request *request, *rdev_req = rdev->scan_req; + LIST_HEAD(coloc_ap_list); + bool need_scan_psc; + const struct ieee80211_sband_iftype_data *iftd; + + rdev_req->scan_6ghz = true; + + if (!rdev->wiphy.bands[NL80211_BAND_6GHZ]) + return -EOPNOTSUPP; + + iftd = ieee80211_get_sband_iftype_data(rdev->wiphy.bands[NL80211_BAND_6GHZ], + rdev_req->wdev->iftype); + if (!iftd || !iftd->he_cap.has_he) + return -EOPNOTSUPP; + + n_channels = rdev->wiphy.bands[NL80211_BAND_6GHZ]->n_channels; + + if (rdev_req->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) { + struct cfg80211_internal_bss *intbss; + + spin_lock_bh(&rdev->bss_lock); + list_for_each_entry(intbss, &rdev->bss_list, list) { + struct cfg80211_bss *res = &intbss->pub; + const struct cfg80211_bss_ies *ies; + + ies = rcu_access_pointer(res->ies); + count += cfg80211_parse_colocated_ap(ies, + &coloc_ap_list); + } + spin_unlock_bh(&rdev->bss_lock); + } + + request = kzalloc(struct_size(request, channels, n_channels) + + sizeof(*request->scan_6ghz_params) * count, + GFP_KERNEL); + if (!request) { + cfg80211_free_coloc_ap_list(&coloc_ap_list); + return -ENOMEM; + } + + *request = *rdev_req; + request->n_channels = 0; + request->scan_6ghz_params = + (void *)&request->channels[n_channels]; + + /* + * PSC channels should not be scanned if all the reported co-located APs + * are indicating that all APs in the same ESS are co-located + */ + if (count) { + need_scan_psc = false; + + list_for_each_entry(ap, &coloc_ap_list, list) { + if (!ap->colocated_ess) { + need_scan_psc = true; + break; + } + } + } else { + need_scan_psc = true; + } + + /* + * add to the scan request the channels that need to be scanned + * regardless of the collocated APs (PSC channels or all channels + * in case that NL80211_SCAN_FLAG_COLOCATED_6GHZ is not set) + */ + for (i = 0; i < rdev_req->n_channels; i++) { + if (rdev_req->channels[i]->band == NL80211_BAND_6GHZ && + ((need_scan_psc && + cfg80211_channel_is_psc(rdev_req->channels[i])) || + !(rdev_req->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ))) { + cfg80211_scan_req_add_chan(request, + rdev_req->channels[i], + false); + } + } + + if (!(rdev_req->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ)) + goto skip; + + list_for_each_entry(ap, &coloc_ap_list, list) { + bool found = false; + struct cfg80211_scan_6ghz_params *scan_6ghz_params = + &request->scan_6ghz_params[request->n_6ghz_params]; + struct ieee80211_channel *chan = + ieee80211_get_channel(&rdev->wiphy, ap->center_freq); + + if (!chan || chan->flags & IEEE80211_CHAN_DISABLED) + continue; + + for (i = 0; i < rdev_req->n_channels; i++) { + if (rdev_req->channels[i] == chan) + found = true; + } + + if (!found) + continue; + + if (request->n_ssids > 0 && + !cfg80211_find_ssid_match(ap, request)) + continue; + + cfg80211_scan_req_add_chan(request, chan, true); + memcpy(scan_6ghz_params->bssid, ap->bssid, ETH_ALEN); + scan_6ghz_params->short_ssid = ap->short_ssid; + scan_6ghz_params->short_ssid_valid = ap->short_ssid_valid; + scan_6ghz_params->unsolicited_probe = ap->unsolicited_probe; + + /* + * If a PSC channel is added to the scan and 'need_scan_psc' is + * set to false, then all the APs that the scan logic is + * interested with on the channel are collocated and thus there + * is no need to perform the initial PSC channel listen. + */ + if (cfg80211_channel_is_psc(chan) && !need_scan_psc) + scan_6ghz_params->psc_no_listen = true; + + request->n_6ghz_params++; + } + +skip: + cfg80211_free_coloc_ap_list(&coloc_ap_list); + + if (request->n_channels) { + struct cfg80211_scan_request *old = rdev->int_scan_req; + + rdev->int_scan_req = request; + + /* + * If this scan follows a previous scan, save the scan start + * info from the first part of the scan + */ + if (old) + rdev->int_scan_req->info = old->info; + + err = rdev_scan(rdev, request); + if (err) { + rdev->int_scan_req = old; + kfree(request); + } else { + kfree(old); + } + + return err; + } + + kfree(request); + return -EINVAL; +} + +int cfg80211_scan(struct cfg80211_registered_device *rdev) +{ + struct cfg80211_scan_request *request; + struct cfg80211_scan_request *rdev_req = rdev->scan_req; + u32 n_channels = 0, idx, i; + + if (!(rdev->wiphy.flags & WIPHY_FLAG_SPLIT_SCAN_6GHZ)) + return rdev_scan(rdev, rdev_req); + + for (i = 0; i < rdev_req->n_channels; i++) { + if (rdev_req->channels[i]->band != NL80211_BAND_6GHZ) + n_channels++; + } + + if (!n_channels) + return cfg80211_scan_6ghz(rdev); + + request = kzalloc(struct_size(request, channels, n_channels), + GFP_KERNEL); + if (!request) + return -ENOMEM; + + *request = *rdev_req; + request->n_channels = n_channels; + + for (i = idx = 0; i < rdev_req->n_channels; i++) { + if (rdev_req->channels[i]->band != NL80211_BAND_6GHZ) + request->channels[idx++] = rdev_req->channels[i]; + } + + rdev_req->scan_6ghz = false; + rdev->int_scan_req = request; + return rdev_scan(rdev, request); +} + void ___cfg80211_scan_done(struct cfg80211_registered_device *rdev, bool send_message) { - struct cfg80211_scan_request *request; + struct cfg80211_scan_request *request, *rdev_req; struct wireless_dev *wdev; struct sk_buff *msg; #ifdef CONFIG_CFG80211_WEXT @@ -466,11 +928,18 @@ void ___cfg80211_scan_done(struct cfg80211_registered_device *rdev, return; } - request = rdev->scan_req; - if (!request) + rdev_req = rdev->scan_req; + if (!rdev_req) return; - wdev = request->wdev; + wdev = rdev_req->wdev; + request = rdev->int_scan_req ? rdev->int_scan_req : rdev_req; + + if (wdev_running(wdev) && + (rdev->wiphy.flags & WIPHY_FLAG_SPLIT_SCAN_6GHZ) && + !rdev_req->scan_6ghz && !request->info.aborted && + !cfg80211_scan_6ghz(rdev)) + return; /* * This must be before sending the other events! @@ -501,8 +970,11 @@ void ___cfg80211_scan_done(struct cfg80211_registered_device *rdev, if (wdev->netdev) dev_put(wdev->netdev); + kfree(rdev->int_scan_req); + rdev->int_scan_req = NULL; + + kfree(rdev->scan_req); rdev->scan_req = NULL; - kfree(request); if (!send_message) rdev->scan_msg = msg; @@ -525,10 +997,25 @@ void __cfg80211_scan_done(struct work_struct *wk) void cfg80211_scan_done(struct cfg80211_scan_request *request, struct cfg80211_scan_info *info) { + struct cfg80211_scan_info old_info = request->info; + trace_cfg80211_scan_done(request, info); - WARN_ON(request != wiphy_to_rdev(request->wiphy)->scan_req); + WARN_ON(request != wiphy_to_rdev(request->wiphy)->scan_req && + request != wiphy_to_rdev(request->wiphy)->int_scan_req); request->info = *info; + + /* + * In case the scan is split, the scan_start_tsf and tsf_bssid should + * be of the first part. In such a case old_info.scan_start_tsf should + * be non zero. + */ + if (request->scan_6ghz && old_info.scan_start_tsf) { + request->info.scan_start_tsf = old_info.scan_start_tsf; + memcpy(request->info.tsf_bssid, old_info.tsf_bssid, + sizeof(request->info.tsf_bssid)); + } + request->notified = true; queue_work(cfg80211_wq, &wiphy_to_rdev(request->wiphy)->scan_done_wk); } @@ -1315,15 +1802,24 @@ cfg80211_get_bss_channel(struct wiphy *wiphy, const u8 *ie, size_t ielen, int channel_number = -1; struct ieee80211_channel *alt_channel; - tmp = cfg80211_find_ie(WLAN_EID_DS_PARAMS, ie, ielen); - if (tmp && tmp[1] == 1) { - channel_number = tmp[2]; + if (channel->band == NL80211_BAND_S1GHZ) { + tmp = cfg80211_find_ie(WLAN_EID_S1G_OPERATION, ie, ielen); + if (tmp && tmp[1] >= sizeof(struct ieee80211_s1g_oper_ie)) { + struct ieee80211_s1g_oper_ie *s1gop = (void *)(tmp + 2); + + channel_number = s1gop->primary_ch; + } } else { - tmp = cfg80211_find_ie(WLAN_EID_HT_OPERATION, ie, ielen); - if (tmp && tmp[1] >= sizeof(struct ieee80211_ht_operation)) { - struct ieee80211_ht_operation *htop = (void *)(tmp + 2); + tmp = cfg80211_find_ie(WLAN_EID_DS_PARAMS, ie, ielen); + if (tmp && tmp[1] == 1) { + channel_number = tmp[2]; + } else { + tmp = cfg80211_find_ie(WLAN_EID_HT_OPERATION, ie, ielen); + if (tmp && tmp[1] >= sizeof(struct ieee80211_ht_operation)) { + struct ieee80211_ht_operation *htop = (void *)(tmp + 2); - channel_number = htop->primary_chan; + channel_number = htop->primary_chan; + } } } @@ -1807,8 +2303,11 @@ cfg80211_inform_single_bss_frame_data(struct wiphy *wiphy, struct cfg80211_bss_ies *ies; struct ieee80211_channel *channel; bool signal_valid; - size_t ielen = len - offsetof(struct ieee80211_mgmt, - u.probe_resp.variable); + struct ieee80211_ext *ext = NULL; + u8 *bssid, *variable; + u16 capability, beacon_int; + size_t ielen, min_hdr_len = offsetof(struct ieee80211_mgmt, + u.probe_resp.variable); int bss_type; BUILD_BUG_ON(offsetof(struct ieee80211_mgmt, u.probe_resp.variable) != @@ -1826,21 +2325,57 @@ cfg80211_inform_single_bss_frame_data(struct wiphy *wiphy, (data->signal < 0 || data->signal > 100))) return NULL; - if (WARN_ON(len < offsetof(struct ieee80211_mgmt, u.probe_resp.variable))) + if (ieee80211_is_s1g_beacon(mgmt->frame_control)) { + ext = (void *) mgmt; + min_hdr_len = offsetof(struct ieee80211_ext, u.s1g_beacon); + if (ieee80211_is_s1g_short_beacon(mgmt->frame_control)) + min_hdr_len = offsetof(struct ieee80211_ext, + u.s1g_short_beacon.variable); + } + + if (WARN_ON(len < min_hdr_len)) return NULL; - channel = cfg80211_get_bss_channel(wiphy, mgmt->u.beacon.variable, + ielen = len - min_hdr_len; + variable = mgmt->u.probe_resp.variable; + if (ext) { + if (ieee80211_is_s1g_short_beacon(mgmt->frame_control)) + variable = ext->u.s1g_short_beacon.variable; + else + variable = ext->u.s1g_beacon.variable; + } + + channel = cfg80211_get_bss_channel(wiphy, variable, ielen, data->chan, data->scan_width); if (!channel) return NULL; + if (ext) { + struct ieee80211_s1g_bcn_compat_ie *compat; + u8 *ie; + + ie = (void *)cfg80211_find_ie(WLAN_EID_S1G_BCN_COMPAT, + variable, ielen); + if (!ie) + return NULL; + compat = (void *)(ie + 2); + bssid = ext->u.s1g_beacon.sa; + capability = le16_to_cpu(compat->compat_info); + beacon_int = le16_to_cpu(compat->beacon_int); + } else { + bssid = mgmt->bssid; + beacon_int = le16_to_cpu(mgmt->u.probe_resp.beacon_int); + capability = le16_to_cpu(mgmt->u.probe_resp.capab_info); + } + ies = kzalloc(sizeof(*ies) + ielen, gfp); if (!ies) return NULL; ies->len = ielen; ies->tsf = le64_to_cpu(mgmt->u.probe_resp.timestamp); - ies->from_beacon = ieee80211_is_beacon(mgmt->frame_control); - memcpy(ies->data, mgmt->u.probe_resp.variable, ielen); + ies->from_beacon = ieee80211_is_beacon(mgmt->frame_control) || + ieee80211_is_s1g_beacon(mgmt->frame_control); + memcpy(ies->data, variable, ielen); if (ieee80211_is_probe_resp(mgmt->frame_control)) rcu_assign_pointer(tmp.pub.proberesp_ies, ies); @@ -1848,12 +2383,12 @@ cfg80211_inform_single_bss_frame_data(struct wiphy *wiphy, rcu_assign_pointer(tmp.pub.beacon_ies, ies); rcu_assign_pointer(tmp.pub.ies, ies); - memcpy(tmp.pub.bssid, mgmt->bssid, ETH_ALEN); + memcpy(tmp.pub.bssid, bssid, ETH_ALEN); + tmp.pub.beacon_interval = beacon_int; + tmp.pub.capability = capability; tmp.pub.channel = channel; tmp.pub.scan_width = data->scan_width; tmp.pub.signal = data->signal; - tmp.pub.beacon_interval = le16_to_cpu(mgmt->u.probe_resp.beacon_int); - tmp.pub.capability = le16_to_cpu(mgmt->u.probe_resp.capab_info); tmp.ts_boottime = data->boottime_ns; tmp.parent_tsf = data->parent_tsf; tmp.pub.chains = data->chains; diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 3895697f8540..ba4dfb17c666 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -703,6 +703,7 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) xs->pool = xp_create_and_assign_umem(xs, umem_xs->umem); if (!xs->pool) { + err = -ENOMEM; sockfd_put(sock); goto out_unlock; } diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h index 2d883f631c85..dc1dd5ef70d1 100644 --- a/net/xdp/xsk_queue.h +++ b/net/xdp/xsk_queue.h @@ -96,7 +96,7 @@ struct xsk_queue { * seen and read by the consumer. * * The consumer peeks into the ring to see if the producer has written - * any new entries. If so, the producer can then read these entries + * any new entries. If so, the consumer can then read these entries * and when it is done reading them release them back to the producer * so that the producer can use these slots to fill in new entries. * diff --git a/net/xfrm/Kconfig b/net/xfrm/Kconfig index 5b9a5ab48111..3adf31a83a79 100644 --- a/net/xfrm/Kconfig +++ b/net/xfrm/Kconfig @@ -28,6 +28,17 @@ config XFRM_USER If unsure, say Y. +config XFRM_USER_COMPAT + tristate "Compatible ABI support" + depends on XFRM_USER && COMPAT_FOR_U64_ALIGNMENT && \ + HAVE_EFFICIENT_UNALIGNED_ACCESS + select WANT_COMPAT_NETLINK_MESSAGES + help + Transformation(XFRM) user configuration interface like IPsec + used by compatible Linux applications. + + If unsure, say N. + config XFRM_INTERFACE tristate "Transformation virtual interface" depends on XFRM && IPV6 diff --git a/net/xfrm/Makefile b/net/xfrm/Makefile index 2d4bb4b9f75e..494aa744bfb9 100644 --- a/net/xfrm/Makefile +++ b/net/xfrm/Makefile @@ -9,6 +9,7 @@ obj-$(CONFIG_XFRM) := xfrm_policy.o xfrm_state.o xfrm_hash.o \ obj-$(CONFIG_XFRM_STATISTICS) += xfrm_proc.o obj-$(CONFIG_XFRM_ALGO) += xfrm_algo.o obj-$(CONFIG_XFRM_USER) += xfrm_user.o +obj-$(CONFIG_XFRM_USER_COMPAT) += xfrm_compat.o obj-$(CONFIG_XFRM_IPCOMP) += xfrm_ipcomp.o obj-$(CONFIG_XFRM_INTERFACE) += xfrm_interface.o obj-$(CONFIG_XFRM_ESPINTCP) += espintcp.o diff --git a/net/xfrm/xfrm_compat.c b/net/xfrm/xfrm_compat.c new file mode 100644 index 000000000000..e28f0c9ecd6a --- /dev/null +++ b/net/xfrm/xfrm_compat.c @@ -0,0 +1,625 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * XFRM compat layer + * Author: Dmitry Safonov <dima@arista.com> + * Based on code and translator idea by: Florian Westphal <fw@strlen.de> + */ +#include <linux/compat.h> +#include <linux/xfrm.h> +#include <net/xfrm.h> + +struct compat_xfrm_lifetime_cfg { + compat_u64 soft_byte_limit, hard_byte_limit; + compat_u64 soft_packet_limit, hard_packet_limit; + compat_u64 soft_add_expires_seconds, hard_add_expires_seconds; + compat_u64 soft_use_expires_seconds, hard_use_expires_seconds; +}; /* same size on 32bit, but only 4 byte alignment required */ + +struct compat_xfrm_lifetime_cur { + compat_u64 bytes, packets, add_time, use_time; +}; /* same size on 32bit, but only 4 byte alignment required */ + +struct compat_xfrm_userpolicy_info { + struct xfrm_selector sel; + struct compat_xfrm_lifetime_cfg lft; + struct compat_xfrm_lifetime_cur curlft; + __u32 priority, index; + u8 dir, action, flags, share; + /* 4 bytes additional padding on 64bit */ +}; + +struct compat_xfrm_usersa_info { + struct xfrm_selector sel; + struct xfrm_id id; + xfrm_address_t saddr; + struct compat_xfrm_lifetime_cfg lft; + struct compat_xfrm_lifetime_cur curlft; + struct xfrm_stats stats; + __u32 seq, reqid; + u16 family; + u8 mode, replay_window, flags; + /* 4 bytes additional padding on 64bit */ +}; + +struct compat_xfrm_user_acquire { + struct xfrm_id id; + xfrm_address_t saddr; + struct xfrm_selector sel; + struct compat_xfrm_userpolicy_info policy; + /* 4 bytes additional padding on 64bit */ + __u32 aalgos, ealgos, calgos, seq; +}; + +struct compat_xfrm_userspi_info { + struct compat_xfrm_usersa_info info; + /* 4 bytes additional padding on 64bit */ + __u32 min, max; +}; + +struct compat_xfrm_user_expire { + struct compat_xfrm_usersa_info state; + /* 8 bytes additional padding on 64bit */ + u8 hard; +}; + +struct compat_xfrm_user_polexpire { + struct compat_xfrm_userpolicy_info pol; + /* 8 bytes additional padding on 64bit */ + u8 hard; +}; + +#define XMSGSIZE(type) sizeof(struct type) + +static const int compat_msg_min[XFRM_NR_MSGTYPES] = { + [XFRM_MSG_NEWSA - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_usersa_info), + [XFRM_MSG_DELSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_id), + [XFRM_MSG_GETSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_id), + [XFRM_MSG_NEWPOLICY - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_userpolicy_info), + [XFRM_MSG_DELPOLICY - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_id), + [XFRM_MSG_GETPOLICY - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_id), + [XFRM_MSG_ALLOCSPI - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_userspi_info), + [XFRM_MSG_ACQUIRE - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_user_acquire), + [XFRM_MSG_EXPIRE - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_user_expire), + [XFRM_MSG_UPDPOLICY - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_userpolicy_info), + [XFRM_MSG_UPDSA - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_usersa_info), + [XFRM_MSG_POLEXPIRE - XFRM_MSG_BASE] = XMSGSIZE(compat_xfrm_user_polexpire), + [XFRM_MSG_FLUSHSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_flush), + [XFRM_MSG_FLUSHPOLICY - XFRM_MSG_BASE] = 0, + [XFRM_MSG_NEWAE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_aevent_id), + [XFRM_MSG_GETAE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_aevent_id), + [XFRM_MSG_REPORT - XFRM_MSG_BASE] = XMSGSIZE(xfrm_user_report), + [XFRM_MSG_MIGRATE - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_id), + [XFRM_MSG_NEWSADINFO - XFRM_MSG_BASE] = sizeof(u32), + [XFRM_MSG_GETSADINFO - XFRM_MSG_BASE] = sizeof(u32), + [XFRM_MSG_NEWSPDINFO - XFRM_MSG_BASE] = sizeof(u32), + [XFRM_MSG_GETSPDINFO - XFRM_MSG_BASE] = sizeof(u32), + [XFRM_MSG_MAPPING - XFRM_MSG_BASE] = XMSGSIZE(xfrm_user_mapping) +}; + +static const struct nla_policy compat_policy[XFRMA_MAX+1] = { + [XFRMA_SA] = { .len = XMSGSIZE(compat_xfrm_usersa_info)}, + [XFRMA_POLICY] = { .len = XMSGSIZE(compat_xfrm_userpolicy_info)}, + [XFRMA_LASTUSED] = { .type = NLA_U64}, + [XFRMA_ALG_AUTH_TRUNC] = { .len = sizeof(struct xfrm_algo_auth)}, + [XFRMA_ALG_AEAD] = { .len = sizeof(struct xfrm_algo_aead) }, + [XFRMA_ALG_AUTH] = { .len = sizeof(struct xfrm_algo) }, + [XFRMA_ALG_CRYPT] = { .len = sizeof(struct xfrm_algo) }, + [XFRMA_ALG_COMP] = { .len = sizeof(struct xfrm_algo) }, + [XFRMA_ENCAP] = { .len = sizeof(struct xfrm_encap_tmpl) }, + [XFRMA_TMPL] = { .len = sizeof(struct xfrm_user_tmpl) }, + [XFRMA_SEC_CTX] = { .len = sizeof(struct xfrm_sec_ctx) }, + [XFRMA_LTIME_VAL] = { .len = sizeof(struct xfrm_lifetime_cur) }, + [XFRMA_REPLAY_VAL] = { .len = sizeof(struct xfrm_replay_state) }, + [XFRMA_REPLAY_THRESH] = { .type = NLA_U32 }, + [XFRMA_ETIMER_THRESH] = { .type = NLA_U32 }, + [XFRMA_SRCADDR] = { .len = sizeof(xfrm_address_t) }, + [XFRMA_COADDR] = { .len = sizeof(xfrm_address_t) }, + [XFRMA_POLICY_TYPE] = { .len = sizeof(struct xfrm_userpolicy_type)}, + [XFRMA_MIGRATE] = { .len = sizeof(struct xfrm_user_migrate) }, + [XFRMA_KMADDRESS] = { .len = sizeof(struct xfrm_user_kmaddress) }, + [XFRMA_MARK] = { .len = sizeof(struct xfrm_mark) }, + [XFRMA_TFCPAD] = { .type = NLA_U32 }, + [XFRMA_REPLAY_ESN_VAL] = { .len = sizeof(struct xfrm_replay_state_esn) }, + [XFRMA_SA_EXTRA_FLAGS] = { .type = NLA_U32 }, + [XFRMA_PROTO] = { .type = NLA_U8 }, + [XFRMA_ADDRESS_FILTER] = { .len = sizeof(struct xfrm_address_filter) }, + [XFRMA_OFFLOAD_DEV] = { .len = sizeof(struct xfrm_user_offload) }, + [XFRMA_SET_MARK] = { .type = NLA_U32 }, + [XFRMA_SET_MARK_MASK] = { .type = NLA_U32 }, + [XFRMA_IF_ID] = { .type = NLA_U32 }, +}; + +static struct nlmsghdr *xfrm_nlmsg_put_compat(struct sk_buff *skb, + const struct nlmsghdr *nlh_src, u16 type) +{ + int payload = compat_msg_min[type]; + int src_len = xfrm_msg_min[type]; + struct nlmsghdr *nlh_dst; + + /* Compat messages are shorter or equal to native (+padding) */ + if (WARN_ON_ONCE(src_len < payload)) + return ERR_PTR(-EMSGSIZE); + + nlh_dst = nlmsg_put(skb, nlh_src->nlmsg_pid, nlh_src->nlmsg_seq, + nlh_src->nlmsg_type, payload, nlh_src->nlmsg_flags); + if (!nlh_dst) + return ERR_PTR(-EMSGSIZE); + + memset(nlmsg_data(nlh_dst), 0, payload); + + switch (nlh_src->nlmsg_type) { + /* Compat message has the same layout as native */ + case XFRM_MSG_DELSA: + case XFRM_MSG_DELPOLICY: + case XFRM_MSG_FLUSHSA: + case XFRM_MSG_FLUSHPOLICY: + case XFRM_MSG_NEWAE: + case XFRM_MSG_REPORT: + case XFRM_MSG_MIGRATE: + case XFRM_MSG_NEWSADINFO: + case XFRM_MSG_NEWSPDINFO: + case XFRM_MSG_MAPPING: + WARN_ON_ONCE(src_len != payload); + memcpy(nlmsg_data(nlh_dst), nlmsg_data(nlh_src), src_len); + break; + /* 4 byte alignment for trailing u64 on native, but not on compat */ + case XFRM_MSG_NEWSA: + case XFRM_MSG_NEWPOLICY: + case XFRM_MSG_UPDSA: + case XFRM_MSG_UPDPOLICY: + WARN_ON_ONCE(src_len != payload + 4); + memcpy(nlmsg_data(nlh_dst), nlmsg_data(nlh_src), payload); + break; + case XFRM_MSG_EXPIRE: { + const struct xfrm_user_expire *src_ue = nlmsg_data(nlh_src); + struct compat_xfrm_user_expire *dst_ue = nlmsg_data(nlh_dst); + + /* compat_xfrm_user_expire has 4-byte smaller state */ + memcpy(dst_ue, src_ue, sizeof(dst_ue->state)); + dst_ue->hard = src_ue->hard; + break; + } + case XFRM_MSG_ACQUIRE: { + const struct xfrm_user_acquire *src_ua = nlmsg_data(nlh_src); + struct compat_xfrm_user_acquire *dst_ua = nlmsg_data(nlh_dst); + + memcpy(dst_ua, src_ua, offsetof(struct compat_xfrm_user_acquire, aalgos)); + dst_ua->aalgos = src_ua->aalgos; + dst_ua->ealgos = src_ua->ealgos; + dst_ua->calgos = src_ua->calgos; + dst_ua->seq = src_ua->seq; + break; + } + case XFRM_MSG_POLEXPIRE: { + const struct xfrm_user_polexpire *src_upe = nlmsg_data(nlh_src); + struct compat_xfrm_user_polexpire *dst_upe = nlmsg_data(nlh_dst); + + /* compat_xfrm_user_polexpire has 4-byte smaller state */ + memcpy(dst_upe, src_upe, sizeof(dst_upe->pol)); + dst_upe->hard = src_upe->hard; + break; + } + case XFRM_MSG_ALLOCSPI: { + const struct xfrm_userspi_info *src_usi = nlmsg_data(nlh_src); + struct compat_xfrm_userspi_info *dst_usi = nlmsg_data(nlh_dst); + + /* compat_xfrm_user_polexpire has 4-byte smaller state */ + memcpy(dst_usi, src_usi, sizeof(src_usi->info)); + dst_usi->min = src_usi->min; + dst_usi->max = src_usi->max; + break; + } + /* Not being sent by kernel */ + case XFRM_MSG_GETSA: + case XFRM_MSG_GETPOLICY: + case XFRM_MSG_GETAE: + case XFRM_MSG_GETSADINFO: + case XFRM_MSG_GETSPDINFO: + default: + WARN_ONCE(1, "unsupported nlmsg_type %d", nlh_src->nlmsg_type); + return ERR_PTR(-EOPNOTSUPP); + } + + return nlh_dst; +} + +static int xfrm_nla_cpy(struct sk_buff *dst, const struct nlattr *src, int len) +{ + return nla_put(dst, src->nla_type, len, nla_data(src)); +} + +static int xfrm_xlate64_attr(struct sk_buff *dst, const struct nlattr *src) +{ + switch (src->nla_type) { + case XFRMA_PAD: + /* Ignore */ + return 0; + case XFRMA_ALG_AUTH: + case XFRMA_ALG_CRYPT: + case XFRMA_ALG_COMP: + case XFRMA_ENCAP: + case XFRMA_TMPL: + return xfrm_nla_cpy(dst, src, nla_len(src)); + case XFRMA_SA: + return xfrm_nla_cpy(dst, src, XMSGSIZE(compat_xfrm_usersa_info)); + case XFRMA_POLICY: + return xfrm_nla_cpy(dst, src, XMSGSIZE(compat_xfrm_userpolicy_info)); + case XFRMA_SEC_CTX: + return xfrm_nla_cpy(dst, src, nla_len(src)); + case XFRMA_LTIME_VAL: + return nla_put_64bit(dst, src->nla_type, nla_len(src), + nla_data(src), XFRMA_PAD); + case XFRMA_REPLAY_VAL: + case XFRMA_REPLAY_THRESH: + case XFRMA_ETIMER_THRESH: + case XFRMA_SRCADDR: + case XFRMA_COADDR: + return xfrm_nla_cpy(dst, src, nla_len(src)); + case XFRMA_LASTUSED: + return nla_put_64bit(dst, src->nla_type, nla_len(src), + nla_data(src), XFRMA_PAD); + case XFRMA_POLICY_TYPE: + case XFRMA_MIGRATE: + case XFRMA_ALG_AEAD: + case XFRMA_KMADDRESS: + case XFRMA_ALG_AUTH_TRUNC: + case XFRMA_MARK: + case XFRMA_TFCPAD: + case XFRMA_REPLAY_ESN_VAL: + case XFRMA_SA_EXTRA_FLAGS: + case XFRMA_PROTO: + case XFRMA_ADDRESS_FILTER: + case XFRMA_OFFLOAD_DEV: + case XFRMA_SET_MARK: + case XFRMA_SET_MARK_MASK: + case XFRMA_IF_ID: + return xfrm_nla_cpy(dst, src, nla_len(src)); + default: + BUILD_BUG_ON(XFRMA_MAX != XFRMA_IF_ID); + WARN_ONCE(1, "unsupported nla_type %d", src->nla_type); + return -EOPNOTSUPP; + } +} + +/* Take kernel-built (64bit layout) and create 32bit layout for userspace */ +static int xfrm_xlate64(struct sk_buff *dst, const struct nlmsghdr *nlh_src) +{ + u16 type = nlh_src->nlmsg_type - XFRM_MSG_BASE; + const struct nlattr *nla, *attrs; + struct nlmsghdr *nlh_dst; + int len, remaining; + + nlh_dst = xfrm_nlmsg_put_compat(dst, nlh_src, type); + if (IS_ERR(nlh_dst)) + return PTR_ERR(nlh_dst); + + attrs = nlmsg_attrdata(nlh_src, xfrm_msg_min[type]); + len = nlmsg_attrlen(nlh_src, xfrm_msg_min[type]); + + nla_for_each_attr(nla, attrs, len, remaining) { + int err = xfrm_xlate64_attr(dst, nla); + + if (err) + return err; + } + + nlmsg_end(dst, nlh_dst); + + return 0; +} + +static int xfrm_alloc_compat(struct sk_buff *skb, const struct nlmsghdr *nlh_src) +{ + u16 type = nlh_src->nlmsg_type - XFRM_MSG_BASE; + struct sk_buff *new = NULL; + int err; + + if (WARN_ON_ONCE(type >= ARRAY_SIZE(xfrm_msg_min))) + return -EOPNOTSUPP; + + if (skb_shinfo(skb)->frag_list == NULL) { + new = alloc_skb(skb->len + skb_tailroom(skb), GFP_ATOMIC); + if (!new) + return -ENOMEM; + skb_shinfo(skb)->frag_list = new; + } + + err = xfrm_xlate64(skb_shinfo(skb)->frag_list, nlh_src); + if (err) { + if (new) { + kfree_skb(new); + skb_shinfo(skb)->frag_list = NULL; + } + return err; + } + + return 0; +} + +/* Calculates len of translated 64-bit message. */ +static size_t xfrm_user_rcv_calculate_len64(const struct nlmsghdr *src, + struct nlattr *attrs[XFRMA_MAX+1]) +{ + size_t len = nlmsg_len(src); + + switch (src->nlmsg_type) { + case XFRM_MSG_NEWSA: + case XFRM_MSG_NEWPOLICY: + case XFRM_MSG_ALLOCSPI: + case XFRM_MSG_ACQUIRE: + case XFRM_MSG_UPDPOLICY: + case XFRM_MSG_UPDSA: + len += 4; + break; + case XFRM_MSG_EXPIRE: + case XFRM_MSG_POLEXPIRE: + len += 8; + break; + default: + break; + } + + if (attrs[XFRMA_SA]) + len += 4; + if (attrs[XFRMA_POLICY]) + len += 4; + + /* XXX: some attrs may need to be realigned + * if !CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + */ + + return len; +} + +static int xfrm_attr_cpy32(void *dst, size_t *pos, const struct nlattr *src, + size_t size, int copy_len, int payload) +{ + struct nlmsghdr *nlmsg = dst; + struct nlattr *nla; + + if (WARN_ON_ONCE(copy_len > payload)) + copy_len = payload; + + if (size - *pos < nla_attr_size(payload)) + return -ENOBUFS; + + nla = dst + *pos; + + memcpy(nla, src, nla_attr_size(copy_len)); + nla->nla_len = nla_attr_size(payload); + *pos += nla_attr_size(payload); + nlmsg->nlmsg_len += nla->nla_len; + + memset(dst + *pos, 0, payload - copy_len); + *pos += payload - copy_len; + + return 0; +} + +static int xfrm_xlate32_attr(void *dst, const struct nlattr *nla, + size_t *pos, size_t size, + struct netlink_ext_ack *extack) +{ + int type = nla_type(nla); + u16 pol_len32, pol_len64; + int err; + + if (type > XFRMA_MAX) { + BUILD_BUG_ON(XFRMA_MAX != XFRMA_IF_ID); + NL_SET_ERR_MSG(extack, "Bad attribute"); + return -EOPNOTSUPP; + } + if (nla_len(nla) < compat_policy[type].len) { + NL_SET_ERR_MSG(extack, "Attribute bad length"); + return -EOPNOTSUPP; + } + + pol_len32 = compat_policy[type].len; + pol_len64 = xfrma_policy[type].len; + + /* XFRMA_SA and XFRMA_POLICY - need to know how-to translate */ + if (pol_len32 != pol_len64) { + if (nla_len(nla) != compat_policy[type].len) { + NL_SET_ERR_MSG(extack, "Attribute bad length"); + return -EOPNOTSUPP; + } + err = xfrm_attr_cpy32(dst, pos, nla, size, pol_len32, pol_len64); + if (err) + return err; + } + + return xfrm_attr_cpy32(dst, pos, nla, size, nla_len(nla), nla_len(nla)); +} + +static int xfrm_xlate32(struct nlmsghdr *dst, const struct nlmsghdr *src, + struct nlattr *attrs[XFRMA_MAX+1], + size_t size, u8 type, struct netlink_ext_ack *extack) +{ + size_t pos; + int i; + + memcpy(dst, src, NLMSG_HDRLEN); + dst->nlmsg_len = NLMSG_HDRLEN + xfrm_msg_min[type]; + memset(nlmsg_data(dst), 0, xfrm_msg_min[type]); + + switch (src->nlmsg_type) { + /* Compat message has the same layout as native */ + case XFRM_MSG_DELSA: + case XFRM_MSG_GETSA: + case XFRM_MSG_DELPOLICY: + case XFRM_MSG_GETPOLICY: + case XFRM_MSG_FLUSHSA: + case XFRM_MSG_FLUSHPOLICY: + case XFRM_MSG_NEWAE: + case XFRM_MSG_GETAE: + case XFRM_MSG_REPORT: + case XFRM_MSG_MIGRATE: + case XFRM_MSG_NEWSADINFO: + case XFRM_MSG_GETSADINFO: + case XFRM_MSG_NEWSPDINFO: + case XFRM_MSG_GETSPDINFO: + case XFRM_MSG_MAPPING: + memcpy(nlmsg_data(dst), nlmsg_data(src), compat_msg_min[type]); + break; + /* 4 byte alignment for trailing u64 on native, but not on compat */ + case XFRM_MSG_NEWSA: + case XFRM_MSG_NEWPOLICY: + case XFRM_MSG_UPDSA: + case XFRM_MSG_UPDPOLICY: + memcpy(nlmsg_data(dst), nlmsg_data(src), compat_msg_min[type]); + break; + case XFRM_MSG_EXPIRE: { + const struct compat_xfrm_user_expire *src_ue = nlmsg_data(src); + struct xfrm_user_expire *dst_ue = nlmsg_data(dst); + + /* compat_xfrm_user_expire has 4-byte smaller state */ + memcpy(dst_ue, src_ue, sizeof(src_ue->state)); + dst_ue->hard = src_ue->hard; + break; + } + case XFRM_MSG_ACQUIRE: { + const struct compat_xfrm_user_acquire *src_ua = nlmsg_data(src); + struct xfrm_user_acquire *dst_ua = nlmsg_data(dst); + + memcpy(dst_ua, src_ua, offsetof(struct compat_xfrm_user_acquire, aalgos)); + dst_ua->aalgos = src_ua->aalgos; + dst_ua->ealgos = src_ua->ealgos; + dst_ua->calgos = src_ua->calgos; + dst_ua->seq = src_ua->seq; + break; + } + case XFRM_MSG_POLEXPIRE: { + const struct compat_xfrm_user_polexpire *src_upe = nlmsg_data(src); + struct xfrm_user_polexpire *dst_upe = nlmsg_data(dst); + + /* compat_xfrm_user_polexpire has 4-byte smaller state */ + memcpy(dst_upe, src_upe, sizeof(src_upe->pol)); + dst_upe->hard = src_upe->hard; + break; + } + case XFRM_MSG_ALLOCSPI: { + const struct compat_xfrm_userspi_info *src_usi = nlmsg_data(src); + struct xfrm_userspi_info *dst_usi = nlmsg_data(dst); + + /* compat_xfrm_user_polexpire has 4-byte smaller state */ + memcpy(dst_usi, src_usi, sizeof(src_usi->info)); + dst_usi->min = src_usi->min; + dst_usi->max = src_usi->max; + break; + } + default: + NL_SET_ERR_MSG(extack, "Unsupported message type"); + return -EOPNOTSUPP; + } + pos = dst->nlmsg_len; + + for (i = 1; i < XFRMA_MAX + 1; i++) { + int err; + + if (i == XFRMA_PAD) + continue; + + if (!attrs[i]) + continue; + + err = xfrm_xlate32_attr(dst, attrs[i], &pos, size, extack); + if (err) + return err; + } + + return 0; +} + +static struct nlmsghdr *xfrm_user_rcv_msg_compat(const struct nlmsghdr *h32, + int maxtype, const struct nla_policy *policy, + struct netlink_ext_ack *extack) +{ + /* netlink_rcv_skb() checks if a message has full (struct nlmsghdr) */ + u16 type = h32->nlmsg_type - XFRM_MSG_BASE; + struct nlattr *attrs[XFRMA_MAX+1]; + struct nlmsghdr *h64; + size_t len; + int err; + + BUILD_BUG_ON(ARRAY_SIZE(xfrm_msg_min) != ARRAY_SIZE(compat_msg_min)); + + if (type >= ARRAY_SIZE(xfrm_msg_min)) + return ERR_PTR(-EINVAL); + + /* Don't call parse: the message might have only nlmsg header */ + if ((h32->nlmsg_type == XFRM_MSG_GETSA || + h32->nlmsg_type == XFRM_MSG_GETPOLICY) && + (h32->nlmsg_flags & NLM_F_DUMP)) + return NULL; + + err = nlmsg_parse_deprecated(h32, compat_msg_min[type], attrs, + maxtype ? : XFRMA_MAX, policy ? : compat_policy, extack); + if (err < 0) + return ERR_PTR(err); + + len = xfrm_user_rcv_calculate_len64(h32, attrs); + /* The message doesn't need translation */ + if (len == nlmsg_len(h32)) + return NULL; + + len += NLMSG_HDRLEN; + h64 = kvmalloc(len, GFP_KERNEL | __GFP_ZERO); + if (!h64) + return ERR_PTR(-ENOMEM); + + err = xfrm_xlate32(h64, h32, attrs, len, type, extack); + if (err < 0) { + kvfree(h64); + return ERR_PTR(err); + } + + return h64; +} + +static int xfrm_user_policy_compat(u8 **pdata32, int optlen) +{ + struct compat_xfrm_userpolicy_info *p = (void *)*pdata32; + u8 *src_templates, *dst_templates; + u8 *data64; + + if (optlen < sizeof(*p)) + return -EINVAL; + + data64 = kmalloc_track_caller(optlen + 4, GFP_USER | __GFP_NOWARN); + if (!data64) + return -ENOMEM; + + memcpy(data64, *pdata32, sizeof(*p)); + memset(data64 + sizeof(*p), 0, 4); + + src_templates = *pdata32 + sizeof(*p); + dst_templates = data64 + sizeof(*p) + 4; + memcpy(dst_templates, src_templates, optlen - sizeof(*p)); + + kfree(*pdata32); + *pdata32 = data64; + return 0; +} + +static struct xfrm_translator xfrm_translator = { + .owner = THIS_MODULE, + .alloc_compat = xfrm_alloc_compat, + .rcv_msg_compat = xfrm_user_rcv_msg_compat, + .xlate_user_policy_sockptr = xfrm_user_policy_compat, +}; + +static int __init xfrm_compat_init(void) +{ + return xfrm_register_translator(&xfrm_translator); +} + +static void __exit xfrm_compat_exit(void) +{ + xfrm_unregister_translator(&xfrm_translator); +} + +module_init(xfrm_compat_init); +module_exit(xfrm_compat_exit); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Dmitry Safonov"); +MODULE_DESCRIPTION("XFRM 32-bit compatibility layer"); diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 69520ad3d83b..f9961884500b 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -2264,6 +2264,66 @@ static bool km_is_alive(const struct km_event *c) return is_alive; } +#if IS_ENABLED(CONFIG_XFRM_USER_COMPAT) +static DEFINE_SPINLOCK(xfrm_translator_lock); +static struct xfrm_translator __rcu *xfrm_translator; + +struct xfrm_translator *xfrm_get_translator(void) +{ + struct xfrm_translator *xtr; + + rcu_read_lock(); + xtr = rcu_dereference(xfrm_translator); + if (unlikely(!xtr)) + goto out; + if (!try_module_get(xtr->owner)) + xtr = NULL; +out: + rcu_read_unlock(); + return xtr; +} +EXPORT_SYMBOL_GPL(xfrm_get_translator); + +void xfrm_put_translator(struct xfrm_translator *xtr) +{ + module_put(xtr->owner); +} +EXPORT_SYMBOL_GPL(xfrm_put_translator); + +int xfrm_register_translator(struct xfrm_translator *xtr) +{ + int err = 0; + + spin_lock_bh(&xfrm_translator_lock); + if (unlikely(xfrm_translator != NULL)) + err = -EEXIST; + else + rcu_assign_pointer(xfrm_translator, xtr); + spin_unlock_bh(&xfrm_translator_lock); + + return err; +} +EXPORT_SYMBOL_GPL(xfrm_register_translator); + +int xfrm_unregister_translator(struct xfrm_translator *xtr) +{ + int err = 0; + + spin_lock_bh(&xfrm_translator_lock); + if (likely(xfrm_translator != NULL)) { + if (rcu_access_pointer(xfrm_translator) != xtr) + err = -EINVAL; + else + RCU_INIT_POINTER(xfrm_translator, NULL); + } + spin_unlock_bh(&xfrm_translator_lock); + synchronize_rcu(); + + return err; +} +EXPORT_SYMBOL_GPL(xfrm_unregister_translator); +#endif + int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) { int err; @@ -2271,9 +2331,6 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) struct xfrm_mgr *km; struct xfrm_policy *pol = NULL; - if (in_compat_syscall()) - return -EOPNOTSUPP; - if (sockptr_is_null(optval) && !optlen) { xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL); xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL); @@ -2288,6 +2345,20 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (IS_ERR(data)) return PTR_ERR(data); + if (in_compat_syscall()) { + struct xfrm_translator *xtr = xfrm_get_translator(); + + if (!xtr) + return -EOPNOTSUPP; + + err = xtr->xlate_user_policy_sockptr(&data, optlen); + xfrm_put_translator(xtr); + if (err) { + kfree(data); + return err; + } + } + err = -EINVAL; rcu_read_lock(); list_for_each_entry_rcu(km, &xfrm_km_list, list) { diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index fbb7d9d06478..d0c32a8fcc4a 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -975,6 +975,7 @@ static int dump_one_state(struct xfrm_state *x, int count, void *ptr) struct xfrm_dump_info *sp = ptr; struct sk_buff *in_skb = sp->in_skb; struct sk_buff *skb = sp->out_skb; + struct xfrm_translator *xtr; struct xfrm_usersa_info *p; struct nlmsghdr *nlh; int err; @@ -992,6 +993,18 @@ static int dump_one_state(struct xfrm_state *x, int count, void *ptr) return err; } nlmsg_end(skb, nlh); + + xtr = xfrm_get_translator(); + if (xtr) { + err = xtr->alloc_compat(skb, nlh); + + xfrm_put_translator(xtr); + if (err) { + nlmsg_cancel(skb, nlh); + return err; + } + } + return 0; } @@ -1006,7 +1019,6 @@ static int xfrm_dump_sa_done(struct netlink_callback *cb) return 0; } -static const struct nla_policy xfrma_policy[XFRMA_MAX+1]; static int xfrm_dump_sa(struct sk_buff *skb, struct netlink_callback *cb) { struct net *net = sock_net(skb->sk); @@ -1083,12 +1095,24 @@ static inline int xfrm_nlmsg_multicast(struct net *net, struct sk_buff *skb, u32 pid, unsigned int group) { struct sock *nlsk = rcu_dereference(net->xfrm.nlsk); + struct xfrm_translator *xtr; if (!nlsk) { kfree_skb(skb); return -EPIPE; } + xtr = xfrm_get_translator(); + if (xtr) { + int err = xtr->alloc_compat(skb, nlmsg_hdr(skb)); + + xfrm_put_translator(xtr); + if (err) { + kfree_skb(skb); + return err; + } + } + return nlmsg_multicast(nlsk, skb, pid, group, GFP_ATOMIC); } @@ -1308,6 +1332,7 @@ static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh, struct net *net = sock_net(skb->sk); struct xfrm_state *x; struct xfrm_userspi_info *p; + struct xfrm_translator *xtr; struct sk_buff *resp_skb; xfrm_address_t *daddr; int family; @@ -1358,6 +1383,17 @@ static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh, goto out; } + xtr = xfrm_get_translator(); + if (xtr) { + err = xtr->alloc_compat(skb, nlmsg_hdr(skb)); + + xfrm_put_translator(xtr); + if (err) { + kfree_skb(resp_skb); + goto out; + } + } + err = nlmsg_unicast(net->xfrm.nlsk, resp_skb, NETLINK_CB(skb).portid); out: @@ -1764,6 +1800,7 @@ static int dump_one_policy(struct xfrm_policy *xp, int dir, int count, void *ptr struct xfrm_userpolicy_info *p; struct sk_buff *in_skb = sp->in_skb; struct sk_buff *skb = sp->out_skb; + struct xfrm_translator *xtr; struct nlmsghdr *nlh; int err; @@ -1788,6 +1825,18 @@ static int dump_one_policy(struct xfrm_policy *xp, int dir, int count, void *ptr return err; } nlmsg_end(skb, nlh); + + xtr = xfrm_get_translator(); + if (xtr) { + err = xtr->alloc_compat(skb, nlh); + + xfrm_put_translator(xtr); + if (err) { + nlmsg_cancel(skb, nlh); + return err; + } + } + return 0; } @@ -2533,7 +2582,7 @@ static int xfrm_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, #define XMSGSIZE(type) sizeof(struct type) -static const int xfrm_msg_min[XFRM_NR_MSGTYPES] = { +const int xfrm_msg_min[XFRM_NR_MSGTYPES] = { [XFRM_MSG_NEWSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_info), [XFRM_MSG_DELSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_id), [XFRM_MSG_GETSA - XFRM_MSG_BASE] = XMSGSIZE(xfrm_usersa_id), @@ -2556,10 +2605,11 @@ static const int xfrm_msg_min[XFRM_NR_MSGTYPES] = { [XFRM_MSG_NEWSPDINFO - XFRM_MSG_BASE] = sizeof(u32), [XFRM_MSG_GETSPDINFO - XFRM_MSG_BASE] = sizeof(u32), }; +EXPORT_SYMBOL_GPL(xfrm_msg_min); #undef XMSGSIZE -static const struct nla_policy xfrma_policy[XFRMA_MAX+1] = { +const struct nla_policy xfrma_policy[XFRMA_MAX+1] = { [XFRMA_SA] = { .len = sizeof(struct xfrm_usersa_info)}, [XFRMA_POLICY] = { .len = sizeof(struct xfrm_userpolicy_info)}, [XFRMA_LASTUSED] = { .type = NLA_U64}, @@ -2591,6 +2641,7 @@ static const struct nla_policy xfrma_policy[XFRMA_MAX+1] = { [XFRMA_SET_MARK_MASK] = { .type = NLA_U32 }, [XFRMA_IF_ID] = { .type = NLA_U32 }, }; +EXPORT_SYMBOL_GPL(xfrma_policy); static const struct nla_policy xfrma_spd_policy[XFRMA_SPD_MAX+1] = { [XFRMA_SPD_IPV4_HTHRESH] = { .len = sizeof(struct xfrmu_spdhthresh) }, @@ -2640,11 +2691,9 @@ static int xfrm_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, struct net *net = sock_net(skb->sk); struct nlattr *attrs[XFRMA_MAX+1]; const struct xfrm_link *link; + struct nlmsghdr *nlh64 = NULL; int type, err; - if (in_compat_syscall()) - return -EOPNOTSUPP; - type = nlh->nlmsg_type; if (type > XFRM_MSG_MAX) return -EINVAL; @@ -2656,32 +2705,55 @@ static int xfrm_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, if (!netlink_net_capable(skb, CAP_NET_ADMIN)) return -EPERM; + if (in_compat_syscall()) { + struct xfrm_translator *xtr = xfrm_get_translator(); + + if (!xtr) + return -EOPNOTSUPP; + + nlh64 = xtr->rcv_msg_compat(nlh, link->nla_max, + link->nla_pol, extack); + xfrm_put_translator(xtr); + if (IS_ERR(nlh64)) + return PTR_ERR(nlh64); + if (nlh64) + nlh = nlh64; + } + if ((type == (XFRM_MSG_GETSA - XFRM_MSG_BASE) || type == (XFRM_MSG_GETPOLICY - XFRM_MSG_BASE)) && (nlh->nlmsg_flags & NLM_F_DUMP)) { - if (link->dump == NULL) - return -EINVAL; + struct netlink_dump_control c = { + .start = link->start, + .dump = link->dump, + .done = link->done, + }; - { - struct netlink_dump_control c = { - .start = link->start, - .dump = link->dump, - .done = link->done, - }; - return netlink_dump_start(net->xfrm.nlsk, skb, nlh, &c); + if (link->dump == NULL) { + err = -EINVAL; + goto err; } + + err = netlink_dump_start(net->xfrm.nlsk, skb, nlh, &c); + goto err; } err = nlmsg_parse_deprecated(nlh, xfrm_msg_min[type], attrs, link->nla_max ? : XFRMA_MAX, link->nla_pol ? : xfrma_policy, extack); if (err < 0) - return err; + goto err; - if (link->doit == NULL) - return -EINVAL; + if (link->doit == NULL) { + err = -EINVAL; + goto err; + } - return link->doit(skb, nlh, attrs); + err = link->doit(skb, nlh, attrs); + +err: + kvfree(nlh64); + return err; } static void xfrm_netlink_rcv(struct sk_buff *skb) diff --git a/samples/bpf/sockex3_kern.c b/samples/bpf/sockex3_kern.c index cab9cca0b8eb..8142d02b33e6 100644 --- a/samples/bpf/sockex3_kern.c +++ b/samples/bpf/sockex3_kern.c @@ -31,28 +31,30 @@ struct { #define PARSE_IP 3 #define PARSE_IPV6 4 -/* protocol dispatch routine. - * It tail-calls next BPF program depending on eth proto - * Note, we could have used: - * bpf_tail_call(skb, &jmp_table, proto); - * but it would need large prog_array +/* Protocol dispatch routine. It tail-calls next BPF program depending + * on eth proto. Note, we could have used ... + * + * bpf_tail_call(skb, &jmp_table, proto); + * + * ... but it would need large prog_array and cannot be optimised given + * the map key is not static. */ static inline void parse_eth_proto(struct __sk_buff *skb, u32 proto) { switch (proto) { case ETH_P_8021Q: case ETH_P_8021AD: - bpf_tail_call(skb, &jmp_table, PARSE_VLAN); + bpf_tail_call_static(skb, &jmp_table, PARSE_VLAN); break; case ETH_P_MPLS_UC: case ETH_P_MPLS_MC: - bpf_tail_call(skb, &jmp_table, PARSE_MPLS); + bpf_tail_call_static(skb, &jmp_table, PARSE_MPLS); break; case ETH_P_IP: - bpf_tail_call(skb, &jmp_table, PARSE_IP); + bpf_tail_call_static(skb, &jmp_table, PARSE_IP); break; case ETH_P_IPV6: - bpf_tail_call(skb, &jmp_table, PARSE_IPV6); + bpf_tail_call_static(skb, &jmp_table, PARSE_IPV6); break; } } diff --git a/scripts/bpf_helpers_doc.py b/scripts/bpf_helpers_doc.py index 08388173973f..7d86fdd190be 100755 --- a/scripts/bpf_helpers_doc.py +++ b/scripts/bpf_helpers_doc.py @@ -433,6 +433,7 @@ class PrinterHelpers(Printer): 'struct sk_msg_md', 'struct xdp_md', 'struct path', + 'struct btf_ptr', ] known_types = { '...', @@ -474,6 +475,7 @@ class PrinterHelpers(Printer): 'struct udp6_sock', 'struct task_struct', 'struct path', + 'struct btf_ptr', } mapped_types = { 'u8': '__u8', diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index a22812561064..4f556cfcbfbe 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -414,6 +414,9 @@ enum { /* Enable memory-mapping BPF map */ BPF_F_MMAPABLE = (1U << 10), + +/* Share perf_event among processes */ + BPF_F_PRESERVE_ELEMS = (1U << 11), }; /* Flags for BPF_PROG_QUERY. */ @@ -424,6 +427,11 @@ enum { */ #define BPF_F_QUERY_EFFECTIVE (1U << 0) +/* Flags for BPF_PROG_TEST_RUN */ + +/* If set, run the test on the cpu specified by bpf_attr.test.cpu */ +#define BPF_F_TEST_RUN_ON_CPU (1U << 0) + /* type for BPF_ENABLE_STATS */ enum bpf_stats_type { /* enabled run_time_ns and run_cnt */ @@ -566,6 +574,8 @@ union bpf_attr { */ __aligned_u64 ctx_in; __aligned_u64 ctx_out; + __u32 flags; + __u32 cpu; } test; struct { /* anonymous struct used by BPF_*_GET_*_ID */ @@ -632,8 +642,13 @@ union bpf_attr { }; __u32 attach_type; /* attach type */ __u32 flags; /* extra flags */ - __aligned_u64 iter_info; /* extra bpf_iter_link_info */ - __u32 iter_info_len; /* iter_info length */ + union { + __u32 target_btf_id; /* btf_id of target to attach to */ + struct { + __aligned_u64 iter_info; /* extra bpf_iter_link_info */ + __u32 iter_info_len; /* iter_info length */ + }; + }; } link_create; struct { /* struct used by BPF_LINK_UPDATE command */ @@ -2512,7 +2527,7 @@ union bpf_attr { * result is from *reuse*\ **->socks**\ [] using the hash of the * tuple. * - * long bpf_sk_release(struct bpf_sock *sock) + * long bpf_sk_release(void *sock) * Description * Release the reference held by *sock*. *sock* must be a * non-**NULL** pointer that was returned from @@ -2692,7 +2707,7 @@ union bpf_attr { * result is from *reuse*\ **->socks**\ [] using the hash of the * tuple. * - * long bpf_tcp_check_syncookie(struct bpf_sock *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) + * long bpf_tcp_check_syncookie(void *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) * Description * Check whether *iph* and *th* contain a valid SYN cookie ACK for * the listening socket in *sk*. @@ -2861,6 +2876,7 @@ union bpf_attr { * 0 on success. * * **-ENOENT** if the bpf-local-storage cannot be found. + * **-EINVAL** if sk is not a fullsock (e.g. a request_sock). * * long bpf_send_signal(u32 sig) * Description @@ -2877,7 +2893,7 @@ union bpf_attr { * * **-EAGAIN** if bpf program can try again. * - * s64 bpf_tcp_gen_syncookie(struct bpf_sock *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) + * s64 bpf_tcp_gen_syncookie(void *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) * Description * Try to issue a SYN cookie for the packet with corresponding * IP/TCP headers, *iph* and *th*, on the listening socket in *sk*. @@ -3106,7 +3122,7 @@ union bpf_attr { * Return * The id is returned or 0 in case the id could not be retrieved. * - * long bpf_sk_assign(struct sk_buff *skb, struct bpf_sock *sk, u64 flags) + * long bpf_sk_assign(struct sk_buff *skb, void *sk, u64 flags) * Description * Helper is overloaded depending on BPF program type. This * description applies to **BPF_PROG_TYPE_SCHED_CLS** and @@ -3234,11 +3250,11 @@ union bpf_attr { * * **-EOVERFLOW** if an overflow happened: The same object will be tried again. * - * u64 bpf_sk_cgroup_id(struct bpf_sock *sk) + * u64 bpf_sk_cgroup_id(void *sk) * Description * Return the cgroup v2 id of the socket *sk*. * - * *sk* must be a non-**NULL** pointer to a full socket, e.g. one + * *sk* must be a non-**NULL** pointer to a socket, e.g. one * returned from **bpf_sk_lookup_xxx**\ (), * **bpf_sk_fullsock**\ (), etc. The format of returned id is * same as in **bpf_skb_cgroup_id**\ (). @@ -3248,7 +3264,7 @@ union bpf_attr { * Return * The id is returned or 0 in case the id could not be retrieved. * - * u64 bpf_sk_ancestor_cgroup_id(struct bpf_sock *sk, int ancestor_level) + * u64 bpf_sk_ancestor_cgroup_id(void *sk, int ancestor_level) * Description * Return id of cgroup v2 that is ancestor of cgroup associated * with the *sk* at the *ancestor_level*. The root cgroup is at @@ -3586,6 +3602,72 @@ union bpf_attr { * the data in *dst*. This is a wrapper of **copy_from_user**\ (). * Return * 0 on success, or a negative error in case of failure. + * + * long bpf_snprintf_btf(char *str, u32 str_size, struct btf_ptr *ptr, u32 btf_ptr_size, u64 flags) + * Description + * Use BTF to store a string representation of *ptr*->ptr in *str*, + * using *ptr*->type_id. This value should specify the type + * that *ptr*->ptr points to. LLVM __builtin_btf_type_id(type, 1) + * can be used to look up vmlinux BTF type ids. Traversing the + * data structure using BTF, the type information and values are + * stored in the first *str_size* - 1 bytes of *str*. Safe copy of + * the pointer data is carried out to avoid kernel crashes during + * operation. Smaller types can use string space on the stack; + * larger programs can use map data to store the string + * representation. + * + * The string can be subsequently shared with userspace via + * bpf_perf_event_output() or ring buffer interfaces. + * bpf_trace_printk() is to be avoided as it places too small + * a limit on string size to be useful. + * + * *flags* is a combination of + * + * **BTF_F_COMPACT** + * no formatting around type information + * **BTF_F_NONAME** + * no struct/union member names/types + * **BTF_F_PTR_RAW** + * show raw (unobfuscated) pointer values; + * equivalent to printk specifier %px. + * **BTF_F_ZERO** + * show zero-valued struct/union members; they + * are not displayed by default + * + * Return + * The number of bytes that were written (or would have been + * written if output had to be truncated due to string size), + * or a negative error in cases of failure. + * + * long bpf_seq_printf_btf(struct seq_file *m, struct btf_ptr *ptr, u32 ptr_size, u64 flags) + * Description + * Use BTF to write to seq_write a string representation of + * *ptr*->ptr, using *ptr*->type_id as per bpf_snprintf_btf(). + * *flags* are identical to those used for bpf_snprintf_btf. + * Return + * 0 on success or a negative error in case of failure. + * + * u64 bpf_skb_cgroup_classid(struct sk_buff *skb) + * Description + * See **bpf_get_cgroup_classid**\ () for the main description. + * This helper differs from **bpf_get_cgroup_classid**\ () in that + * the cgroup v1 net_cls class is retrieved only from the *skb*'s + * associated socket instead of the current process. + * Return + * The id is returned or 0 in case the id could not be retrieved. + * + * long bpf_redirect_neigh(u32 ifindex, u64 flags) + * Description + * Redirect the packet to another net device of index *ifindex* + * and fill in L2 addresses from neighboring subsystem. This helper + * is somewhat similar to **bpf_redirect**\ (), except that it + * fills in e.g. MAC addresses based on the L3 information from + * the packet. This helper is supported for IPv4 and IPv6 protocols. + * The *flags* argument is reserved and must be 0. The helper is + * currently only supported for tc BPF program types. + * Return + * The helper returns **TC_ACT_REDIRECT** on success or + * **TC_ACT_SHOT** on error. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -3737,6 +3819,10 @@ union bpf_attr { FN(inode_storage_delete), \ FN(d_path), \ FN(copy_from_user), \ + FN(snprintf_btf), \ + FN(seq_printf_btf), \ + FN(skb_cgroup_classid), \ + FN(redirect_neigh), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper @@ -4845,4 +4931,34 @@ struct bpf_sk_lookup { __u32 local_port; /* Host byte order */ }; +/* + * struct btf_ptr is used for typed pointer representation; the + * type id is used to render the pointer data as the appropriate type + * via the bpf_snprintf_btf() helper described above. A flags field - + * potentially to specify additional details about the BTF pointer + * (rather than its mode of display) - is included for future use. + * Display flags - BTF_F_* - are passed to bpf_snprintf_btf separately. + */ +struct btf_ptr { + void *ptr; + __u32 type_id; + __u32 flags; /* BTF ptr flags; unused at present. */ +}; + +/* + * Flags to control bpf_snprintf_btf() behaviour. + * - BTF_F_COMPACT: no formatting around type information + * - BTF_F_NONAME: no struct/union member names/types + * - BTF_F_PTR_RAW: show raw (unobfuscated) pointer values; + * equivalent to %px. + * - BTF_F_ZERO: show zero-valued struct/union members; they + * are not displayed by default + */ +enum { + BTF_F_COMPACT = (1ULL << 0), + BTF_F_NONAME = (1ULL << 1), + BTF_F_PTR_RAW = (1ULL << 2), + BTF_F_ZERO = (1ULL << 3), +}; + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile index f43249696d9f..5f9abed3e226 100644 --- a/tools/lib/bpf/Makefile +++ b/tools/lib/bpf/Makefile @@ -98,19 +98,18 @@ PC_FILE = libbpf.pc ifdef EXTRA_CFLAGS CFLAGS := $(EXTRA_CFLAGS) else - CFLAGS := -g -Wall + CFLAGS := -g -O2 endif # Append required CFLAGS override CFLAGS += $(EXTRA_WARNINGS) -Wno-switch-enum override CFLAGS += -Werror -Wall -override CFLAGS += -fPIC override CFLAGS += $(INCLUDES) override CFLAGS += -fvisibility=hidden override CFLAGS += -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 # flags specific for shared library -SHLIB_FLAGS := -DSHARED +SHLIB_FLAGS := -DSHARED -fPIC ifeq ($(VERBOSE),1) Q = diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 2baa1308737c..d27e34133973 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -586,19 +586,31 @@ int bpf_link_create(int prog_fd, int target_fd, enum bpf_attach_type attach_type, const struct bpf_link_create_opts *opts) { + __u32 target_btf_id, iter_info_len; union bpf_attr attr; if (!OPTS_VALID(opts, bpf_link_create_opts)) return -EINVAL; + iter_info_len = OPTS_GET(opts, iter_info_len, 0); + target_btf_id = OPTS_GET(opts, target_btf_id, 0); + + if (iter_info_len && target_btf_id) + return -EINVAL; + memset(&attr, 0, sizeof(attr)); attr.link_create.prog_fd = prog_fd; attr.link_create.target_fd = target_fd; attr.link_create.attach_type = attach_type; attr.link_create.flags = OPTS_GET(opts, flags, 0); - attr.link_create.iter_info = - ptr_to_u64(OPTS_GET(opts, iter_info, (void *)0)); - attr.link_create.iter_info_len = OPTS_GET(opts, iter_info_len, 0); + + if (iter_info_len) { + attr.link_create.iter_info = + ptr_to_u64(OPTS_GET(opts, iter_info, (void *)0)); + attr.link_create.iter_info_len = iter_info_len; + } else if (target_btf_id) { + attr.link_create.target_btf_id = target_btf_id; + } return sys_bpf(BPF_LINK_CREATE, &attr, sizeof(attr)); } @@ -712,6 +724,37 @@ int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr) return ret; } +int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts) +{ + union bpf_attr attr; + int ret; + + if (!OPTS_VALID(opts, bpf_test_run_opts)) + return -EINVAL; + + memset(&attr, 0, sizeof(attr)); + attr.test.prog_fd = prog_fd; + attr.test.cpu = OPTS_GET(opts, cpu, 0); + attr.test.flags = OPTS_GET(opts, flags, 0); + attr.test.repeat = OPTS_GET(opts, repeat, 0); + attr.test.duration = OPTS_GET(opts, duration, 0); + attr.test.ctx_size_in = OPTS_GET(opts, ctx_size_in, 0); + attr.test.ctx_size_out = OPTS_GET(opts, ctx_size_out, 0); + attr.test.data_size_in = OPTS_GET(opts, data_size_in, 0); + attr.test.data_size_out = OPTS_GET(opts, data_size_out, 0); + attr.test.ctx_in = ptr_to_u64(OPTS_GET(opts, ctx_in, NULL)); + attr.test.ctx_out = ptr_to_u64(OPTS_GET(opts, ctx_out, NULL)); + attr.test.data_in = ptr_to_u64(OPTS_GET(opts, data_in, NULL)); + attr.test.data_out = ptr_to_u64(OPTS_GET(opts, data_out, NULL)); + + ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr)); + OPTS_SET(opts, data_size_out, attr.test.data_size_out); + OPTS_SET(opts, ctx_size_out, attr.test.ctx_size_out); + OPTS_SET(opts, duration, attr.test.duration); + OPTS_SET(opts, retval, attr.test.retval); + return ret; +} + static int bpf_obj_get_next_id(__u32 start_id, __u32 *next_id, int cmd) { union bpf_attr attr; @@ -815,7 +858,7 @@ int bpf_raw_tracepoint_open(const char *name, int prog_fd) return sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, sizeof(attr)); } -int bpf_load_btf(void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size, +int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size, bool do_log) { union bpf_attr attr = {}; diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 8c1ac4b42f90..875dde20d56e 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -174,8 +174,9 @@ struct bpf_link_create_opts { __u32 flags; union bpf_iter_link_info *iter_info; __u32 iter_info_len; + __u32 target_btf_id; }; -#define bpf_link_create_opts__last_field iter_info_len +#define bpf_link_create_opts__last_field target_btf_id LIBBPF_API int bpf_link_create(int prog_fd, int target_fd, enum bpf_attach_type attach_type, @@ -234,7 +235,7 @@ LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt); LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd); -LIBBPF_API int bpf_load_btf(void *btf, __u32 btf_size, char *log_buf, +LIBBPF_API int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size, bool do_log); LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, __u32 *prog_id, __u32 *fd_type, @@ -251,6 +252,32 @@ struct bpf_prog_bind_opts { LIBBPF_API int bpf_prog_bind_map(int prog_fd, int map_fd, const struct bpf_prog_bind_opts *opts); + +struct bpf_test_run_opts { + size_t sz; /* size of this struct for forward/backward compatibility */ + const void *data_in; /* optional */ + void *data_out; /* optional */ + __u32 data_size_in; + __u32 data_size_out; /* in: max length of data_out + * out: length of data_out + */ + const void *ctx_in; /* optional */ + void *ctx_out; /* optional */ + __u32 ctx_size_in; + __u32 ctx_size_out; /* in: max length of ctx_out + * out: length of cxt_out + */ + __u32 retval; /* out: return code of the BPF program */ + int repeat; + __u32 duration; /* out: average per repetition in ns */ + __u32 flags; + __u32 cpu; +}; +#define bpf_test_run_opts__last_field cpu + +LIBBPF_API int bpf_prog_test_run_opts(int prog_fd, + struct bpf_test_run_opts *opts); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h index 1106777df00b..2bdb7d6dbad2 100644 --- a/tools/lib/bpf/bpf_helpers.h +++ b/tools/lib/bpf/bpf_helpers.h @@ -54,6 +54,52 @@ #endif /* + * Helper macro to throw a compilation error if __bpf_unreachable() gets + * built into the resulting code. This works given BPF back end does not + * implement __builtin_trap(). This is useful to assert that certain paths + * of the program code are never used and hence eliminated by the compiler. + * + * For example, consider a switch statement that covers known cases used by + * the program. __bpf_unreachable() can then reside in the default case. If + * the program gets extended such that a case is not covered in the switch + * statement, then it will throw a build error due to the default case not + * being compiled out. + */ +#ifndef __bpf_unreachable +# define __bpf_unreachable() __builtin_trap() +#endif + +/* + * Helper function to perform a tail call with a constant/immediate map slot. + */ +static __always_inline void +bpf_tail_call_static(void *ctx, const void *map, const __u32 slot) +{ + if (!__builtin_constant_p(slot)) + __bpf_unreachable(); + + /* + * Provide a hard guarantee that LLVM won't optimize setting r2 (map + * pointer) and r3 (constant map index) from _different paths_ ending + * up at the _same_ call insn as otherwise we won't be able to use the + * jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel + * given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key + * tracking for prog array pokes") for details on verifier tracking. + * + * Note on clobber list: we need to stay in-line with BPF calling + * convention, so even if we don't end up using r0, r4, r5, we need + * to mark them as clobber so that LLVM doesn't end up using them + * before / after the call. + */ + asm volatile("r1 = %[ctx]\n\t" + "r2 = %[map]\n\t" + "r3 = %[slot]\n\t" + "call 12" + :: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot) + : "r0", "r1", "r2", "r3", "r4", "r5"); +} + +/* * Helper structure used by eBPF C program * to describe BPF map attributes to libbpf loader */ diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index a3d259e614b0..231b07203e3d 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) /* Copyright (c) 2018 Facebook */ +#include <byteswap.h> #include <endian.h> #include <stdio.h> #include <stdlib.h> @@ -27,17 +28,72 @@ static struct btf_type btf_void; struct btf { - union { - struct btf_header *hdr; - void *data; - }; - struct btf_type **types; - const char *strings; - void *nohdr_data; + /* raw BTF data in native endianness */ + void *raw_data; + /* raw BTF data in non-native endianness */ + void *raw_data_swapped; + __u32 raw_size; + /* whether target endianness differs from the native one */ + bool swapped_endian; + + /* + * When BTF is loaded from an ELF or raw memory it is stored + * in a contiguous memory block. The hdr, type_data, and, strs_data + * point inside that memory region to their respective parts of BTF + * representation: + * + * +--------------------------------+ + * | Header | Types | Strings | + * +--------------------------------+ + * ^ ^ ^ + * | | | + * hdr | | + * types_data-+ | + * strs_data------------+ + * + * If BTF data is later modified, e.g., due to types added or + * removed, BTF deduplication performed, etc, this contiguous + * representation is broken up into three independently allocated + * memory regions to be able to modify them independently. + * raw_data is nulled out at that point, but can be later allocated + * and cached again if user calls btf__get_raw_data(), at which point + * raw_data will contain a contiguous copy of header, types, and + * strings: + * + * +----------+ +---------+ +-----------+ + * | Header | | Types | | Strings | + * +----------+ +---------+ +-----------+ + * ^ ^ ^ + * | | | + * hdr | | + * types_data----+ | + * strs_data------------------+ + * + * +----------+---------+-----------+ + * | Header | Types | Strings | + * raw_data----->+----------+---------+-----------+ + */ + struct btf_header *hdr; + + void *types_data; + size_t types_data_cap; /* used size stored in hdr->type_len */ + + /* type ID to `struct btf_type *` lookup index */ + __u32 *type_offs; + size_t type_offs_cap; __u32 nr_types; - __u32 types_size; - __u32 data_size; + + void *strs_data; + size_t strs_data_cap; /* used size stored in hdr->str_len */ + + /* lookup index for each unique string in strings section */ + struct hashmap *strs_hash; + /* whether strings are already deduplicated */ + bool strs_deduped; + /* BTF object FD, if loaded into kernel */ int fd; + + /* Pointer size (in bytes) for a target architecture of this BTF */ int ptr_sz; }; @@ -46,60 +102,114 @@ static inline __u64 ptr_to_u64(const void *ptr) return (__u64) (unsigned long) ptr; } -static int btf_add_type(struct btf *btf, struct btf_type *t) +/* Ensure given dynamically allocated memory region pointed to by *data* with + * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough + * memory to accomodate *add_cnt* new elements, assuming *cur_cnt* elements + * are already used. At most *max_cnt* elements can be ever allocated. + * If necessary, memory is reallocated and all existing data is copied over, + * new pointer to the memory region is stored at *data, new memory region + * capacity (in number of elements) is stored in *cap. + * On success, memory pointer to the beginning of unused memory is returned. + * On error, NULL is returned. + */ +void *btf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, + size_t cur_cnt, size_t max_cnt, size_t add_cnt) { - if (btf->types_size - btf->nr_types < 2) { - struct btf_type **new_types; - __u32 expand_by, new_size; + size_t new_cnt; + void *new_data; - if (btf->types_size == BTF_MAX_NR_TYPES) - return -E2BIG; + if (cur_cnt + add_cnt <= *cap_cnt) + return *data + cur_cnt * elem_sz; - expand_by = max(btf->types_size >> 2, 16U); - new_size = min(BTF_MAX_NR_TYPES, btf->types_size + expand_by); + /* requested more than the set limit */ + if (cur_cnt + add_cnt > max_cnt) + return NULL; - new_types = libbpf_reallocarray(btf->types, new_size, sizeof(*new_types)); - if (!new_types) - return -ENOMEM; + new_cnt = *cap_cnt; + new_cnt += new_cnt / 4; /* expand by 25% */ + if (new_cnt < 16) /* but at least 16 elements */ + new_cnt = 16; + if (new_cnt > max_cnt) /* but not exceeding a set limit */ + new_cnt = max_cnt; + if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */ + new_cnt = cur_cnt + add_cnt; + + new_data = libbpf_reallocarray(*data, new_cnt, elem_sz); + if (!new_data) + return NULL; - if (btf->nr_types == 0) - new_types[0] = &btf_void; + /* zero out newly allocated portion of memory */ + memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); - btf->types = new_types; - btf->types_size = new_size; - } + *data = new_data; + *cap_cnt = new_cnt; + return new_data + cur_cnt * elem_sz; +} + +/* Ensure given dynamically allocated memory region has enough allocated space + * to accommodate *need_cnt* elements of size *elem_sz* bytes each + */ +int btf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt) +{ + void *p; + + if (need_cnt <= *cap_cnt) + return 0; + + p = btf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); + if (!p) + return -ENOMEM; + + return 0; +} - btf->types[++(btf->nr_types)] = t; +static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off) +{ + __u32 *p; + p = btf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), + btf->nr_types + 1, BTF_MAX_NR_TYPES, 1); + if (!p) + return -ENOMEM; + + *p = type_off; return 0; } +static void btf_bswap_hdr(struct btf_header *h) +{ + h->magic = bswap_16(h->magic); + h->hdr_len = bswap_32(h->hdr_len); + h->type_off = bswap_32(h->type_off); + h->type_len = bswap_32(h->type_len); + h->str_off = bswap_32(h->str_off); + h->str_len = bswap_32(h->str_len); +} + static int btf_parse_hdr(struct btf *btf) { - const struct btf_header *hdr = btf->hdr; + struct btf_header *hdr = btf->hdr; __u32 meta_left; - if (btf->data_size < sizeof(struct btf_header)) { + if (btf->raw_size < sizeof(struct btf_header)) { pr_debug("BTF header not found\n"); return -EINVAL; } - if (hdr->magic != BTF_MAGIC) { + if (hdr->magic == bswap_16(BTF_MAGIC)) { + btf->swapped_endian = true; + if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) { + pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", + bswap_32(hdr->hdr_len)); + return -ENOTSUP; + } + btf_bswap_hdr(hdr); + } else if (hdr->magic != BTF_MAGIC) { pr_debug("Invalid BTF magic:%x\n", hdr->magic); return -EINVAL; } - if (hdr->version != BTF_VERSION) { - pr_debug("Unsupported BTF version:%u\n", hdr->version); - return -ENOTSUP; - } - - if (hdr->flags) { - pr_debug("Unsupported BTF flags:%x\n", hdr->flags); - return -ENOTSUP; - } - - meta_left = btf->data_size - sizeof(*hdr); + meta_left = btf->raw_size - sizeof(*hdr); if (!meta_left) { pr_debug("BTF has no data\n"); return -EINVAL; @@ -125,15 +235,13 @@ static int btf_parse_hdr(struct btf *btf) return -EINVAL; } - btf->nohdr_data = btf->hdr + 1; - return 0; } static int btf_parse_str_sec(struct btf *btf) { const struct btf_header *hdr = btf->hdr; - const char *start = btf->nohdr_data + hdr->str_off; + const char *start = btf->strs_data; const char *end = start + btf->hdr->str_len; if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || @@ -142,14 +250,12 @@ static int btf_parse_str_sec(struct btf *btf) return -EINVAL; } - btf->strings = start; - return 0; } -static int btf_type_size(struct btf_type *t) +static int btf_type_size(const struct btf_type *t) { - int base_size = sizeof(struct btf_type); + const int base_size = sizeof(struct btf_type); __u16 vlen = btf_vlen(t); switch (btf_kind(t)) { @@ -182,25 +288,120 @@ static int btf_type_size(struct btf_type *t) } } +static void btf_bswap_type_base(struct btf_type *t) +{ + t->name_off = bswap_32(t->name_off); + t->info = bswap_32(t->info); + t->type = bswap_32(t->type); +} + +static int btf_bswap_type_rest(struct btf_type *t) +{ + struct btf_var_secinfo *v; + struct btf_member *m; + struct btf_array *a; + struct btf_param *p; + struct btf_enum *e; + __u16 vlen = btf_vlen(t); + int i; + + switch (btf_kind(t)) { + case BTF_KIND_FWD: + case BTF_KIND_CONST: + case BTF_KIND_VOLATILE: + case BTF_KIND_RESTRICT: + case BTF_KIND_PTR: + case BTF_KIND_TYPEDEF: + case BTF_KIND_FUNC: + return 0; + case BTF_KIND_INT: + *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1)); + return 0; + case BTF_KIND_ENUM: + for (i = 0, e = btf_enum(t); i < vlen; i++, e++) { + e->name_off = bswap_32(e->name_off); + e->val = bswap_32(e->val); + } + return 0; + case BTF_KIND_ARRAY: + a = btf_array(t); + a->type = bswap_32(a->type); + a->index_type = bswap_32(a->index_type); + a->nelems = bswap_32(a->nelems); + return 0; + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: + for (i = 0, m = btf_members(t); i < vlen; i++, m++) { + m->name_off = bswap_32(m->name_off); + m->type = bswap_32(m->type); + m->offset = bswap_32(m->offset); + } + return 0; + case BTF_KIND_FUNC_PROTO: + for (i = 0, p = btf_params(t); i < vlen; i++, p++) { + p->name_off = bswap_32(p->name_off); + p->type = bswap_32(p->type); + } + return 0; + case BTF_KIND_VAR: + btf_var(t)->linkage = bswap_32(btf_var(t)->linkage); + return 0; + case BTF_KIND_DATASEC: + for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) { + v->type = bswap_32(v->type); + v->offset = bswap_32(v->offset); + v->size = bswap_32(v->size); + } + return 0; + default: + pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t)); + return -EINVAL; + } +} + static int btf_parse_type_sec(struct btf *btf) { struct btf_header *hdr = btf->hdr; - void *nohdr_data = btf->nohdr_data; - void *next_type = nohdr_data + hdr->type_off; - void *end_type = nohdr_data + hdr->str_off; + void *next_type = btf->types_data; + void *end_type = next_type + hdr->type_len; + int err, i = 0, type_size; - while (next_type < end_type) { - struct btf_type *t = next_type; - int type_size; - int err; + /* VOID (type_id == 0) is specially handled by btf__get_type_by_id(), + * so ensure we can never properly use its offset from index by + * setting it to a large value + */ + err = btf_add_type_idx_entry(btf, UINT_MAX); + if (err) + return err; + + while (next_type + sizeof(struct btf_type) <= end_type) { + i++; + + if (btf->swapped_endian) + btf_bswap_type_base(next_type); - type_size = btf_type_size(t); + type_size = btf_type_size(next_type); if (type_size < 0) return type_size; - next_type += type_size; - err = btf_add_type(btf, t); + if (next_type + type_size > end_type) { + pr_warn("BTF type [%d] is malformed\n", i); + return -EINVAL; + } + + if (btf->swapped_endian && btf_bswap_type_rest(next_type)) + return -EINVAL; + + err = btf_add_type_idx_entry(btf, next_type - btf->types_data); if (err) return err; + + next_type += type_size; + btf->nr_types++; + } + + if (next_type != end_type) { + pr_warn("BTF types data is malformed\n"); + return -EINVAL; } return 0; @@ -211,12 +412,20 @@ __u32 btf__get_nr_types(const struct btf *btf) return btf->nr_types; } +/* internal helper returning non-const pointer to a type */ +static struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id) +{ + if (type_id == 0) + return &btf_void; + + return btf->types_data + btf->type_offs[type_id]; +} + const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id) { if (type_id > btf->nr_types) return NULL; - - return btf->types[type_id]; + return btf_type_by_id((struct btf *)btf, type_id); } static int determine_ptr_size(const struct btf *btf) @@ -283,6 +492,38 @@ int btf__set_pointer_size(struct btf *btf, size_t ptr_sz) return 0; } +static bool is_host_big_endian(void) +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + return false; +#elif __BYTE_ORDER == __BIG_ENDIAN + return true; +#else +# error "Unrecognized __BYTE_ORDER__" +#endif +} + +enum btf_endianness btf__endianness(const struct btf *btf) +{ + if (is_host_big_endian()) + return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN; + else + return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN; +} + +int btf__set_endianness(struct btf *btf, enum btf_endianness endian) +{ + if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN) + return -EINVAL; + + btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN); + if (!btf->swapped_endian) { + free(btf->raw_data_swapped); + btf->raw_data_swapped = NULL; + } + return 0; +} + static bool btf_type_is_void(const struct btf_type *t) { return t == &btf_void || btf_is_fwd(t); @@ -414,7 +655,7 @@ __s32 btf__find_by_name(const struct btf *btf, const char *type_name) return 0; for (i = 1; i <= btf->nr_types; i++) { - const struct btf_type *t = btf->types[i]; + const struct btf_type *t = btf__type_by_id(btf, i); const char *name = btf__name_by_offset(btf, t->name_off); if (name && !strcmp(type_name, name)) @@ -433,7 +674,7 @@ __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, return 0; for (i = 1; i <= btf->nr_types; i++) { - const struct btf_type *t = btf->types[i]; + const struct btf_type *t = btf__type_by_id(btf, i); const char *name; if (btf_kind(t) != kind) @@ -446,6 +687,11 @@ __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, return -ENOENT; } +static bool btf_is_modifiable(const struct btf *btf) +{ + return (void *)btf->hdr != btf->raw_data; +} + void btf__free(struct btf *btf) { if (IS_ERR_OR_NULL(btf)) @@ -454,11 +700,55 @@ void btf__free(struct btf *btf) if (btf->fd >= 0) close(btf->fd); - free(btf->data); - free(btf->types); + if (btf_is_modifiable(btf)) { + /* if BTF was modified after loading, it will have a split + * in-memory representation for header, types, and strings + * sections, so we need to free all of them individually. It + * might still have a cached contiguous raw data present, + * which will be unconditionally freed below. + */ + free(btf->hdr); + free(btf->types_data); + free(btf->strs_data); + } + free(btf->raw_data); + free(btf->raw_data_swapped); + free(btf->type_offs); free(btf); } +struct btf *btf__new_empty(void) +{ + struct btf *btf; + + btf = calloc(1, sizeof(*btf)); + if (!btf) + return ERR_PTR(-ENOMEM); + + btf->fd = -1; + btf->ptr_sz = sizeof(void *); + btf->swapped_endian = false; + + /* +1 for empty string at offset 0 */ + btf->raw_size = sizeof(struct btf_header) + 1; + btf->raw_data = calloc(1, btf->raw_size); + if (!btf->raw_data) { + free(btf); + return ERR_PTR(-ENOMEM); + } + + btf->hdr = btf->raw_data; + btf->hdr->hdr_len = sizeof(struct btf_header); + btf->hdr->magic = BTF_MAGIC; + btf->hdr->version = BTF_VERSION; + + btf->types_data = btf->raw_data + btf->hdr->hdr_len; + btf->strs_data = btf->raw_data + btf->hdr->hdr_len; + btf->hdr->str_len = 1; /* empty string at offset 0 */ + + return btf; +} + struct btf *btf__new(const void *data, __u32 size) { struct btf *btf; @@ -468,26 +758,28 @@ struct btf *btf__new(const void *data, __u32 size) if (!btf) return ERR_PTR(-ENOMEM); - btf->fd = -1; - - btf->data = malloc(size); - if (!btf->data) { + btf->raw_data = malloc(size); + if (!btf->raw_data) { err = -ENOMEM; goto done; } + memcpy(btf->raw_data, data, size); + btf->raw_size = size; - memcpy(btf->data, data, size); - btf->data_size = size; - + btf->hdr = btf->raw_data; err = btf_parse_hdr(btf); if (err) goto done; + btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off; + btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off; + err = btf_parse_str_sec(btf); + err = err ?: btf_parse_type_sec(btf); if (err) goto done; - err = btf_parse_type_sec(btf); + btf->fd = -1; done: if (err) { @@ -498,17 +790,6 @@ done: return btf; } -static bool btf_check_endianness(const GElf_Ehdr *ehdr) -{ -#if __BYTE_ORDER == __LITTLE_ENDIAN - return ehdr->e_ident[EI_DATA] == ELFDATA2LSB; -#elif __BYTE_ORDER == __BIG_ENDIAN - return ehdr->e_ident[EI_DATA] == ELFDATA2MSB; -#else -# error "Unrecognized __BYTE_ORDER__" -#endif -} - struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) { Elf_Data *btf_data = NULL, *btf_ext_data = NULL; @@ -541,10 +822,6 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) pr_warn("failed to get EHDR from %s\n", path); goto done; } - if (!btf_check_endianness(&ehdr)) { - pr_warn("non-native ELF endianness is not supported\n"); - goto done; - } if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) { pr_warn("failed to get e_shstrndx from %s\n", path); goto done; @@ -656,7 +933,7 @@ struct btf *btf__parse_raw(const char *path) err = -EIO; goto err_out; } - if (magic != BTF_MAGIC) { + if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) { /* definitely not a raw BTF */ err = -EPROTO; goto err_out; @@ -789,7 +1066,7 @@ int btf__finalize_data(struct bpf_object *obj, struct btf *btf) __u32 i; for (i = 1; i <= btf->nr_types; i++) { - struct btf_type *t = btf->types[i]; + struct btf_type *t = btf_type_by_id(btf, i); /* Loader needs to fix up some of the things compiler * couldn't get its hands on while emitting BTF. This @@ -806,10 +1083,13 @@ int btf__finalize_data(struct bpf_object *obj, struct btf *btf) return err; } +static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian); + int btf__load(struct btf *btf) { - __u32 log_buf_size = 0; + __u32 log_buf_size = 0, raw_size; char *log_buf = NULL; + void *raw_data; int err = 0; if (btf->fd >= 0) @@ -824,8 +1104,16 @@ retry_load: *log_buf = 0; } - btf->fd = bpf_load_btf(btf->data, btf->data_size, - log_buf, log_buf_size, false); + raw_data = btf_get_raw_data(btf, &raw_size, false); + if (!raw_data) { + err = -ENOMEM; + goto done; + } + /* cache native raw data representation */ + btf->raw_size = raw_size; + btf->raw_data = raw_data; + + btf->fd = bpf_load_btf(raw_data, raw_size, log_buf, log_buf_size, false); if (btf->fd < 0) { if (!log_buf || errno == ENOSPC) { log_buf_size = max((__u32)BPF_LOG_BUF_SIZE, @@ -856,20 +1144,88 @@ void btf__set_fd(struct btf *btf, int fd) btf->fd = fd; } -const void *btf__get_raw_data(const struct btf *btf, __u32 *size) +static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian) +{ + struct btf_header *hdr = btf->hdr; + struct btf_type *t; + void *data, *p; + __u32 data_sz; + int i; + + data = swap_endian ? btf->raw_data_swapped : btf->raw_data; + if (data) { + *size = btf->raw_size; + return data; + } + + data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len; + data = calloc(1, data_sz); + if (!data) + return NULL; + p = data; + + memcpy(p, hdr, hdr->hdr_len); + if (swap_endian) + btf_bswap_hdr(p); + p += hdr->hdr_len; + + memcpy(p, btf->types_data, hdr->type_len); + if (swap_endian) { + for (i = 1; i <= btf->nr_types; i++) { + t = p + btf->type_offs[i]; + /* btf_bswap_type_rest() relies on native t->info, so + * we swap base type info after we swapped all the + * additional information + */ + if (btf_bswap_type_rest(t)) + goto err_out; + btf_bswap_type_base(t); + } + } + p += hdr->type_len; + + memcpy(p, btf->strs_data, hdr->str_len); + p += hdr->str_len; + + *size = data_sz; + return data; +err_out: + free(data); + return NULL; +} + +const void *btf__get_raw_data(const struct btf *btf_ro, __u32 *size) { - *size = btf->data_size; - return btf->data; + struct btf *btf = (struct btf *)btf_ro; + __u32 data_sz; + void *data; + + data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian); + if (!data) + return NULL; + + btf->raw_size = data_sz; + if (btf->swapped_endian) + btf->raw_data_swapped = data; + else + btf->raw_data = data; + *size = data_sz; + return data; } -const char *btf__name_by_offset(const struct btf *btf, __u32 offset) +const char *btf__str_by_offset(const struct btf *btf, __u32 offset) { if (offset < btf->hdr->str_len) - return &btf->strings[offset]; + return btf->strs_data + offset; else return NULL; } +const char *btf__name_by_offset(const struct btf *btf, __u32 offset) +{ + return btf__str_by_offset(btf, offset); +} + int btf__get_from_id(__u32 id, struct btf **btf) { struct bpf_btf_info btf_info = { 0 }; @@ -1005,6 +1361,970 @@ int btf__get_map_kv_tids(const struct btf *btf, const char *map_name, return 0; } +static size_t strs_hash_fn(const void *key, void *ctx) +{ + struct btf *btf = ctx; + const char *str = btf->strs_data + (long)key; + + return str_hash(str); +} + +static bool strs_hash_equal_fn(const void *key1, const void *key2, void *ctx) +{ + struct btf *btf = ctx; + const char *str1 = btf->strs_data + (long)key1; + const char *str2 = btf->strs_data + (long)key2; + + return strcmp(str1, str2) == 0; +} + +static void btf_invalidate_raw_data(struct btf *btf) +{ + if (btf->raw_data) { + free(btf->raw_data); + btf->raw_data = NULL; + } + if (btf->raw_data_swapped) { + free(btf->raw_data_swapped); + btf->raw_data_swapped = NULL; + } +} + +/* Ensure BTF is ready to be modified (by splitting into a three memory + * regions for header, types, and strings). Also invalidate cached + * raw_data, if any. + */ +static int btf_ensure_modifiable(struct btf *btf) +{ + void *hdr, *types, *strs, *strs_end, *s; + struct hashmap *hash = NULL; + long off; + int err; + + if (btf_is_modifiable(btf)) { + /* any BTF modification invalidates raw_data */ + btf_invalidate_raw_data(btf); + return 0; + } + + /* split raw data into three memory regions */ + hdr = malloc(btf->hdr->hdr_len); + types = malloc(btf->hdr->type_len); + strs = malloc(btf->hdr->str_len); + if (!hdr || !types || !strs) + goto err_out; + + memcpy(hdr, btf->hdr, btf->hdr->hdr_len); + memcpy(types, btf->types_data, btf->hdr->type_len); + memcpy(strs, btf->strs_data, btf->hdr->str_len); + + /* build lookup index for all strings */ + hash = hashmap__new(strs_hash_fn, strs_hash_equal_fn, btf); + if (IS_ERR(hash)) { + err = PTR_ERR(hash); + hash = NULL; + goto err_out; + } + + strs_end = strs + btf->hdr->str_len; + for (off = 0, s = strs; s < strs_end; off += strlen(s) + 1, s = strs + off) { + /* hashmap__add() returns EEXIST if string with the same + * content already is in the hash map + */ + err = hashmap__add(hash, (void *)off, (void *)off); + if (err == -EEXIST) + continue; /* duplicate */ + if (err) + goto err_out; + } + + /* only when everything was successful, update internal state */ + btf->hdr = hdr; + btf->types_data = types; + btf->types_data_cap = btf->hdr->type_len; + btf->strs_data = strs; + btf->strs_data_cap = btf->hdr->str_len; + btf->strs_hash = hash; + /* if BTF was created from scratch, all strings are guaranteed to be + * unique and deduplicated + */ + btf->strs_deduped = btf->hdr->str_len <= 1; + + /* invalidate raw_data representation */ + btf_invalidate_raw_data(btf); + + return 0; + +err_out: + hashmap__free(hash); + free(hdr); + free(types); + free(strs); + return -ENOMEM; +} + +static void *btf_add_str_mem(struct btf *btf, size_t add_sz) +{ + return btf_add_mem(&btf->strs_data, &btf->strs_data_cap, 1, + btf->hdr->str_len, BTF_MAX_STR_OFFSET, add_sz); +} + +/* Find an offset in BTF string section that corresponds to a given string *s*. + * Returns: + * - >0 offset into string section, if string is found; + * - -ENOENT, if string is not in the string section; + * - <0, on any other error. + */ +int btf__find_str(struct btf *btf, const char *s) +{ + long old_off, new_off, len; + void *p; + + /* BTF needs to be in a modifiable state to build string lookup index */ + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + /* see btf__add_str() for why we do this */ + len = strlen(s) + 1; + p = btf_add_str_mem(btf, len); + if (!p) + return -ENOMEM; + + new_off = btf->hdr->str_len; + memcpy(p, s, len); + + if (hashmap__find(btf->strs_hash, (void *)new_off, (void **)&old_off)) + return old_off; + + return -ENOENT; +} + +/* Add a string s to the BTF string section. + * Returns: + * - > 0 offset into string section, on success; + * - < 0, on error. + */ +int btf__add_str(struct btf *btf, const char *s) +{ + long old_off, new_off, len; + void *p; + int err; + + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + /* Hashmap keys are always offsets within btf->strs_data, so to even + * look up some string from the "outside", we need to first append it + * at the end, so that it can be addressed with an offset. Luckily, + * until btf->hdr->str_len is incremented, that string is just a piece + * of garbage for the rest of BTF code, so no harm, no foul. On the + * other hand, if the string is unique, it's already appended and + * ready to be used, only a simple btf->hdr->str_len increment away. + */ + len = strlen(s) + 1; + p = btf_add_str_mem(btf, len); + if (!p) + return -ENOMEM; + + new_off = btf->hdr->str_len; + memcpy(p, s, len); + + /* Now attempt to add the string, but only if the string with the same + * contents doesn't exist already (HASHMAP_ADD strategy). If such + * string exists, we'll get its offset in old_off (that's old_key). + */ + err = hashmap__insert(btf->strs_hash, (void *)new_off, (void *)new_off, + HASHMAP_ADD, (const void **)&old_off, NULL); + if (err == -EEXIST) + return old_off; /* duplicated string, return existing offset */ + if (err) + return err; + + btf->hdr->str_len += len; /* new unique string, adjust data length */ + return new_off; +} + +static void *btf_add_type_mem(struct btf *btf, size_t add_sz) +{ + return btf_add_mem(&btf->types_data, &btf->types_data_cap, 1, + btf->hdr->type_len, UINT_MAX, add_sz); +} + +static __u32 btf_type_info(int kind, int vlen, int kflag) +{ + return (kflag << 31) | (kind << 24) | vlen; +} + +static void btf_type_inc_vlen(struct btf_type *t) +{ + t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t)); +} + +/* + * Append new BTF_KIND_INT type with: + * - *name* - non-empty, non-NULL type name; + * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes; + * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL. + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding) +{ + struct btf_type *t; + int sz, err, name_off; + + /* non-empty name */ + if (!name || !name[0]) + return -EINVAL; + /* byte_sz must be power of 2 */ + if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) + return -EINVAL; + if (encoding & ~(BTF_INT_SIGNED | BTF_INT_CHAR | BTF_INT_BOOL)) + return -EINVAL; + + /* deconstruct BTF, if necessary, and invalidate raw_data */ + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_type) + sizeof(int); + t = btf_add_type_mem(btf, sz); + if (!t) + return -ENOMEM; + + /* if something goes wrong later, we might end up with an extra string, + * but that shouldn't be a problem, because BTF can't be constructed + * completely anyway and will most probably be just discarded + */ + name_off = btf__add_str(btf, name); + if (name_off < 0) + return name_off; + + t->name_off = name_off; + t->info = btf_type_info(BTF_KIND_INT, 0, 0); + t->size = byte_sz; + /* set INT info, we don't allow setting legacy bit offset/size */ + *(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8); + + err = btf_add_type_idx_entry(btf, btf->hdr->type_len); + if (err) + return err; + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + btf->nr_types++; + return btf->nr_types; +} + +/* it's completely legal to append BTF types with type IDs pointing forward to + * types that haven't been appended yet, so we only make sure that id looks + * sane, we can't guarantee that ID will always be valid + */ +static int validate_type_id(int id) +{ + if (id < 0 || id > BTF_MAX_NR_TYPES) + return -EINVAL; + return 0; +} + +/* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */ +static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id) +{ + struct btf_type *t; + int sz, name_off = 0, err; + + if (validate_type_id(ref_type_id)) + return -EINVAL; + + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_type); + t = btf_add_type_mem(btf, sz); + if (!t) + return -ENOMEM; + + if (name && name[0]) { + name_off = btf__add_str(btf, name); + if (name_off < 0) + return name_off; + } + + t->name_off = name_off; + t->info = btf_type_info(kind, 0, 0); + t->type = ref_type_id; + + err = btf_add_type_idx_entry(btf, btf->hdr->type_len); + if (err) + return err; + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + btf->nr_types++; + return btf->nr_types; +} + +/* + * Append new BTF_KIND_PTR type with: + * - *ref_type_id* - referenced type ID, it might not exist yet; + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_ptr(struct btf *btf, int ref_type_id) +{ + return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id); +} + +/* + * Append new BTF_KIND_ARRAY type with: + * - *index_type_id* - type ID of the type describing array index; + * - *elem_type_id* - type ID of the type describing array element; + * - *nr_elems* - the size of the array; + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems) +{ + struct btf_type *t; + struct btf_array *a; + int sz, err; + + if (validate_type_id(index_type_id) || validate_type_id(elem_type_id)) + return -EINVAL; + + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_type) + sizeof(struct btf_array); + t = btf_add_type_mem(btf, sz); + if (!t) + return -ENOMEM; + + t->name_off = 0; + t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0); + t->size = 0; + + a = btf_array(t); + a->type = elem_type_id; + a->index_type = index_type_id; + a->nelems = nr_elems; + + err = btf_add_type_idx_entry(btf, btf->hdr->type_len); + if (err) + return err; + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + btf->nr_types++; + return btf->nr_types; +} + +/* generic STRUCT/UNION append function */ +static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz) +{ + struct btf_type *t; + int sz, err, name_off = 0; + + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_type); + t = btf_add_type_mem(btf, sz); + if (!t) + return -ENOMEM; + + if (name && name[0]) { + name_off = btf__add_str(btf, name); + if (name_off < 0) + return name_off; + } + + /* start out with vlen=0 and no kflag; this will be adjusted when + * adding each member + */ + t->name_off = name_off; + t->info = btf_type_info(kind, 0, 0); + t->size = bytes_sz; + + err = btf_add_type_idx_entry(btf, btf->hdr->type_len); + if (err) + return err; + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + btf->nr_types++; + return btf->nr_types; +} + +/* + * Append new BTF_KIND_STRUCT type with: + * - *name* - name of the struct, can be NULL or empty for anonymous structs; + * - *byte_sz* - size of the struct, in bytes; + * + * Struct initially has no fields in it. Fields can be added by + * btf__add_field() right after btf__add_struct() succeeds. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_struct(struct btf *btf, const char *name, __u32 byte_sz) +{ + return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz); +} + +/* + * Append new BTF_KIND_UNION type with: + * - *name* - name of the union, can be NULL or empty for anonymous union; + * - *byte_sz* - size of the union, in bytes; + * + * Union initially has no fields in it. Fields can be added by + * btf__add_field() right after btf__add_union() succeeds. All fields + * should have *bit_offset* of 0. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz) +{ + return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz); +} + +/* + * Append new field for the current STRUCT/UNION type with: + * - *name* - name of the field, can be NULL or empty for anonymous field; + * - *type_id* - type ID for the type describing field type; + * - *bit_offset* - bit offset of the start of the field within struct/union; + * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields; + * Returns: + * - 0, on success; + * - <0, on error. + */ +int btf__add_field(struct btf *btf, const char *name, int type_id, + __u32 bit_offset, __u32 bit_size) +{ + struct btf_type *t; + struct btf_member *m; + bool is_bitfield; + int sz, name_off = 0; + + /* last type should be union/struct */ + if (btf->nr_types == 0) + return -EINVAL; + t = btf_type_by_id(btf, btf->nr_types); + if (!btf_is_composite(t)) + return -EINVAL; + + if (validate_type_id(type_id)) + return -EINVAL; + /* best-effort bit field offset/size enforcement */ + is_bitfield = bit_size || (bit_offset % 8 != 0); + if (is_bitfield && (bit_size == 0 || bit_size > 255 || bit_offset > 0xffffff)) + return -EINVAL; + + /* only offset 0 is allowed for unions */ + if (btf_is_union(t) && bit_offset) + return -EINVAL; + + /* decompose and invalidate raw data */ + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_member); + m = btf_add_type_mem(btf, sz); + if (!m) + return -ENOMEM; + + if (name && name[0]) { + name_off = btf__add_str(btf, name); + if (name_off < 0) + return name_off; + } + + m->name_off = name_off; + m->type = type_id; + m->offset = bit_offset | (bit_size << 24); + + /* btf_add_type_mem can invalidate t pointer */ + t = btf_type_by_id(btf, btf->nr_types); + /* update parent type's vlen and kflag */ + t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t)); + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + return 0; +} + +/* + * Append new BTF_KIND_ENUM type with: + * - *name* - name of the enum, can be NULL or empty for anonymous enums; + * - *byte_sz* - size of the enum, in bytes. + * + * Enum initially has no enum values in it (and corresponds to enum forward + * declaration). Enumerator values can be added by btf__add_enum_value() + * immediately after btf__add_enum() succeeds. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz) +{ + struct btf_type *t; + int sz, err, name_off = 0; + + /* byte_sz must be power of 2 */ + if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8) + return -EINVAL; + + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_type); + t = btf_add_type_mem(btf, sz); + if (!t) + return -ENOMEM; + + if (name && name[0]) { + name_off = btf__add_str(btf, name); + if (name_off < 0) + return name_off; + } + + /* start out with vlen=0; it will be adjusted when adding enum values */ + t->name_off = name_off; + t->info = btf_type_info(BTF_KIND_ENUM, 0, 0); + t->size = byte_sz; + + err = btf_add_type_idx_entry(btf, btf->hdr->type_len); + if (err) + return err; + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + btf->nr_types++; + return btf->nr_types; +} + +/* + * Append new enum value for the current ENUM type with: + * - *name* - name of the enumerator value, can't be NULL or empty; + * - *value* - integer value corresponding to enum value *name*; + * Returns: + * - 0, on success; + * - <0, on error. + */ +int btf__add_enum_value(struct btf *btf, const char *name, __s64 value) +{ + struct btf_type *t; + struct btf_enum *v; + int sz, name_off; + + /* last type should be BTF_KIND_ENUM */ + if (btf->nr_types == 0) + return -EINVAL; + t = btf_type_by_id(btf, btf->nr_types); + if (!btf_is_enum(t)) + return -EINVAL; + + /* non-empty name */ + if (!name || !name[0]) + return -EINVAL; + if (value < INT_MIN || value > UINT_MAX) + return -E2BIG; + + /* decompose and invalidate raw data */ + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_enum); + v = btf_add_type_mem(btf, sz); + if (!v) + return -ENOMEM; + + name_off = btf__add_str(btf, name); + if (name_off < 0) + return name_off; + + v->name_off = name_off; + v->val = value; + + /* update parent type's vlen */ + t = btf_type_by_id(btf, btf->nr_types); + btf_type_inc_vlen(t); + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + return 0; +} + +/* + * Append new BTF_KIND_FWD type with: + * - *name*, non-empty/non-NULL name; + * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT, + * BTF_FWD_UNION, or BTF_FWD_ENUM; + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind) +{ + if (!name || !name[0]) + return -EINVAL; + + switch (fwd_kind) { + case BTF_FWD_STRUCT: + case BTF_FWD_UNION: { + struct btf_type *t; + int id; + + id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0); + if (id <= 0) + return id; + t = btf_type_by_id(btf, id); + t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION); + return id; + } + case BTF_FWD_ENUM: + /* enum forward in BTF currently is just an enum with no enum + * values; we also assume a standard 4-byte size for it + */ + return btf__add_enum(btf, name, sizeof(int)); + default: + return -EINVAL; + } +} + +/* + * Append new BTF_KING_TYPEDEF type with: + * - *name*, non-empty/non-NULL name; + * - *ref_type_id* - referenced type ID, it might not exist yet; + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id) +{ + if (!name || !name[0]) + return -EINVAL; + + return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id); +} + +/* + * Append new BTF_KIND_VOLATILE type with: + * - *ref_type_id* - referenced type ID, it might not exist yet; + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_volatile(struct btf *btf, int ref_type_id) +{ + return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id); +} + +/* + * Append new BTF_KIND_CONST type with: + * - *ref_type_id* - referenced type ID, it might not exist yet; + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_const(struct btf *btf, int ref_type_id) +{ + return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id); +} + +/* + * Append new BTF_KIND_RESTRICT type with: + * - *ref_type_id* - referenced type ID, it might not exist yet; + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_restrict(struct btf *btf, int ref_type_id) +{ + return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id); +} + +/* + * Append new BTF_KIND_FUNC type with: + * - *name*, non-empty/non-NULL name; + * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet; + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_func(struct btf *btf, const char *name, + enum btf_func_linkage linkage, int proto_type_id) +{ + int id; + + if (!name || !name[0]) + return -EINVAL; + if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL && + linkage != BTF_FUNC_EXTERN) + return -EINVAL; + + id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id); + if (id > 0) { + struct btf_type *t = btf_type_by_id(btf, id); + + t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0); + } + return id; +} + +/* + * Append new BTF_KIND_FUNC_PROTO with: + * - *ret_type_id* - type ID for return result of a function. + * + * Function prototype initially has no arguments, but they can be added by + * btf__add_func_param() one by one, immediately after + * btf__add_func_proto() succeeded. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_func_proto(struct btf *btf, int ret_type_id) +{ + struct btf_type *t; + int sz, err; + + if (validate_type_id(ret_type_id)) + return -EINVAL; + + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_type); + t = btf_add_type_mem(btf, sz); + if (!t) + return -ENOMEM; + + /* start out with vlen=0; this will be adjusted when adding enum + * values, if necessary + */ + t->name_off = 0; + t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0); + t->type = ret_type_id; + + err = btf_add_type_idx_entry(btf, btf->hdr->type_len); + if (err) + return err; + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + btf->nr_types++; + return btf->nr_types; +} + +/* + * Append new function parameter for current FUNC_PROTO type with: + * - *name* - parameter name, can be NULL or empty; + * - *type_id* - type ID describing the type of the parameter. + * Returns: + * - 0, on success; + * - <0, on error. + */ +int btf__add_func_param(struct btf *btf, const char *name, int type_id) +{ + struct btf_type *t; + struct btf_param *p; + int sz, name_off = 0; + + if (validate_type_id(type_id)) + return -EINVAL; + + /* last type should be BTF_KIND_FUNC_PROTO */ + if (btf->nr_types == 0) + return -EINVAL; + t = btf_type_by_id(btf, btf->nr_types); + if (!btf_is_func_proto(t)) + return -EINVAL; + + /* decompose and invalidate raw data */ + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_param); + p = btf_add_type_mem(btf, sz); + if (!p) + return -ENOMEM; + + if (name && name[0]) { + name_off = btf__add_str(btf, name); + if (name_off < 0) + return name_off; + } + + p->name_off = name_off; + p->type = type_id; + + /* update parent type's vlen */ + t = btf_type_by_id(btf, btf->nr_types); + btf_type_inc_vlen(t); + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + return 0; +} + +/* + * Append new BTF_KIND_VAR type with: + * - *name* - non-empty/non-NULL name; + * - *linkage* - variable linkage, one of BTF_VAR_STATIC, + * BTF_VAR_GLOBAL_ALLOCATED, or BTF_VAR_GLOBAL_EXTERN; + * - *type_id* - type ID of the type describing the type of the variable. + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id) +{ + struct btf_type *t; + struct btf_var *v; + int sz, err, name_off; + + /* non-empty name */ + if (!name || !name[0]) + return -EINVAL; + if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED && + linkage != BTF_VAR_GLOBAL_EXTERN) + return -EINVAL; + if (validate_type_id(type_id)) + return -EINVAL; + + /* deconstruct BTF, if necessary, and invalidate raw_data */ + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_type) + sizeof(struct btf_var); + t = btf_add_type_mem(btf, sz); + if (!t) + return -ENOMEM; + + name_off = btf__add_str(btf, name); + if (name_off < 0) + return name_off; + + t->name_off = name_off; + t->info = btf_type_info(BTF_KIND_VAR, 0, 0); + t->type = type_id; + + v = btf_var(t); + v->linkage = linkage; + + err = btf_add_type_idx_entry(btf, btf->hdr->type_len); + if (err) + return err; + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + btf->nr_types++; + return btf->nr_types; +} + +/* + * Append new BTF_KIND_DATASEC type with: + * - *name* - non-empty/non-NULL name; + * - *byte_sz* - data section size, in bytes. + * + * Data section is initially empty. Variables info can be added with + * btf__add_datasec_var_info() calls, after btf__add_datasec() succeeds. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz) +{ + struct btf_type *t; + int sz, err, name_off; + + /* non-empty name */ + if (!name || !name[0]) + return -EINVAL; + + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_type); + t = btf_add_type_mem(btf, sz); + if (!t) + return -ENOMEM; + + name_off = btf__add_str(btf, name); + if (name_off < 0) + return name_off; + + /* start with vlen=0, which will be update as var_secinfos are added */ + t->name_off = name_off; + t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0); + t->size = byte_sz; + + err = btf_add_type_idx_entry(btf, btf->hdr->type_len); + if (err) + return err; + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + btf->nr_types++; + return btf->nr_types; +} + +/* + * Append new data section variable information entry for current DATASEC type: + * - *var_type_id* - type ID, describing type of the variable; + * - *offset* - variable offset within data section, in bytes; + * - *byte_sz* - variable size, in bytes. + * + * Returns: + * - 0, on success; + * - <0, on error. + */ +int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz) +{ + struct btf_type *t; + struct btf_var_secinfo *v; + int sz; + + /* last type should be BTF_KIND_DATASEC */ + if (btf->nr_types == 0) + return -EINVAL; + t = btf_type_by_id(btf, btf->nr_types); + if (!btf_is_datasec(t)) + return -EINVAL; + + if (validate_type_id(var_type_id)) + return -EINVAL; + + /* decompose and invalidate raw data */ + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + + sz = sizeof(struct btf_var_secinfo); + v = btf_add_type_mem(btf, sz); + if (!v) + return -ENOMEM; + + v->type = var_type_id; + v->offset = offset; + v->size = byte_sz; + + /* update parent type's vlen */ + t = btf_type_by_id(btf, btf->nr_types); + btf_type_inc_vlen(t); + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + return 0; +} + struct btf_ext_sec_setup_param { __u32 off; __u32 len; @@ -1151,7 +2471,10 @@ static int btf_ext_parse_hdr(__u8 *data, __u32 data_size) return -EINVAL; } - if (hdr->magic != BTF_MAGIC) { + if (hdr->magic == bswap_16(BTF_MAGIC)) { + pr_warn("BTF.ext in non-native endianness is not supported\n"); + return -ENOTSUP; + } else if (hdr->magic != BTF_MAGIC) { pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic); return -EINVAL; } @@ -1471,6 +2794,9 @@ int btf__dedup(struct btf *btf, struct btf_ext *btf_ext, return -EINVAL; } + if (btf_ensure_modifiable(btf)) + return -ENOMEM; + err = btf_dedup_strings(d); if (err < 0) { pr_debug("btf_dedup_strings failed:%d\n", err); @@ -1655,7 +2981,7 @@ static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext, /* special BTF "void" type is made canonical immediately */ d->map[0] = 0; for (i = 1; i <= btf->nr_types; i++) { - struct btf_type *t = d->btf->types[i]; + struct btf_type *t = btf_type_by_id(d->btf, i); /* VAR and DATASEC are never deduped and are self-canonical */ if (btf_is_var(t) || btf_is_datasec(t)) @@ -1694,7 +3020,7 @@ static int btf_for_each_str_off(struct btf_dedup *d, str_off_fn_t fn, void *ctx) struct btf_type *t; for (i = 1; i <= d->btf->nr_types; i++) { - t = d->btf->types[i]; + t = btf_type_by_id(d->btf, i); r = fn(&t->name_off, ctx); if (r) return r; @@ -1848,8 +3174,7 @@ static int btf_str_remap_offset(__u32 *str_off_ptr, void *ctx) */ static int btf_dedup_strings(struct btf_dedup *d) { - const struct btf_header *hdr = d->btf->hdr; - char *start = (char *)d->btf->nohdr_data + hdr->str_off; + char *start = d->btf->strs_data; char *end = start + d->btf->hdr->str_len; char *p = start, *tmp_strs = NULL; struct btf_str_ptrs strs = { @@ -1861,6 +3186,9 @@ static int btf_dedup_strings(struct btf_dedup *d) int i, j, err = 0, grp_idx; bool grp_used; + if (d->btf->strs_deduped) + return 0; + /* build index of all strings */ while (p < end) { if (strs.cnt + 1 > strs.cap) { @@ -1953,6 +3281,7 @@ static int btf_dedup_strings(struct btf_dedup *d) goto done; d->btf->hdr->str_len = end - start; + d->btf->strs_deduped = true; done: free(tmp_strs); @@ -2229,7 +3558,7 @@ static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2) */ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) { - struct btf_type *t = d->btf->types[type_id]; + struct btf_type *t = btf_type_by_id(d->btf, type_id); struct hashmap_entry *hash_entry; struct btf_type *cand; /* if we don't find equivalent type, then we are canonical */ @@ -2256,7 +3585,7 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) h = btf_hash_int(t); for_each_dedup_cand(d, hash_entry, h) { cand_id = (__u32)(long)hash_entry->value; - cand = d->btf->types[cand_id]; + cand = btf_type_by_id(d->btf, cand_id); if (btf_equal_int(t, cand)) { new_id = cand_id; break; @@ -2268,7 +3597,7 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) h = btf_hash_enum(t); for_each_dedup_cand(d, hash_entry, h) { cand_id = (__u32)(long)hash_entry->value; - cand = d->btf->types[cand_id]; + cand = btf_type_by_id(d->btf, cand_id); if (btf_equal_enum(t, cand)) { new_id = cand_id; break; @@ -2291,7 +3620,7 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) h = btf_hash_common(t); for_each_dedup_cand(d, hash_entry, h) { cand_id = (__u32)(long)hash_entry->value; - cand = d->btf->types[cand_id]; + cand = btf_type_by_id(d->btf, cand_id); if (btf_equal_common(t, cand)) { new_id = cand_id; break; @@ -2350,13 +3679,13 @@ static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id) { __u32 orig_type_id = type_id; - if (!btf_is_fwd(d->btf->types[type_id])) + if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) return type_id; while (is_type_mapped(d, type_id) && d->map[type_id] != type_id) type_id = d->map[type_id]; - if (!btf_is_fwd(d->btf->types[type_id])) + if (!btf_is_fwd(btf__type_by_id(d->btf, type_id))) return type_id; return orig_type_id; @@ -2484,8 +3813,8 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, if (btf_dedup_hypot_map_add(d, canon_id, cand_id)) return -ENOMEM; - cand_type = d->btf->types[cand_id]; - canon_type = d->btf->types[canon_id]; + cand_type = btf_type_by_id(d->btf, cand_id); + canon_type = btf_type_by_id(d->btf, canon_id); cand_kind = btf_kind(cand_type); canon_kind = btf_kind(canon_type); @@ -2636,8 +3965,8 @@ static void btf_dedup_merge_hypot_map(struct btf_dedup *d) targ_type_id = d->hypot_map[cand_type_id]; t_id = resolve_type_id(d, targ_type_id); c_id = resolve_type_id(d, cand_type_id); - t_kind = btf_kind(d->btf->types[t_id]); - c_kind = btf_kind(d->btf->types[c_id]); + t_kind = btf_kind(btf__type_by_id(d->btf, t_id)); + c_kind = btf_kind(btf__type_by_id(d->btf, c_id)); /* * Resolve FWD into STRUCT/UNION. * It's ok to resolve FWD into STRUCT/UNION that's not yet @@ -2705,7 +4034,7 @@ static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) if (d->map[type_id] <= BTF_MAX_NR_TYPES) return 0; - t = d->btf->types[type_id]; + t = btf_type_by_id(d->btf, type_id); kind = btf_kind(t); if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION) @@ -2726,7 +4055,7 @@ static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because * FWD and compatible STRUCT/UNION are considered equivalent. */ - cand_type = d->btf->types[cand_id]; + cand_type = btf_type_by_id(d->btf, cand_id); if (!btf_shallow_equal_struct(t, cand_type)) continue; @@ -2798,7 +4127,7 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) if (d->map[type_id] <= BTF_MAX_NR_TYPES) return resolve_type_id(d, type_id); - t = d->btf->types[type_id]; + t = btf_type_by_id(d->btf, type_id); d->map[type_id] = BTF_IN_PROGRESS_ID; switch (btf_kind(t)) { @@ -2816,7 +4145,7 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) h = btf_hash_common(t); for_each_dedup_cand(d, hash_entry, h) { cand_id = (__u32)(long)hash_entry->value; - cand = d->btf->types[cand_id]; + cand = btf_type_by_id(d->btf, cand_id); if (btf_equal_common(t, cand)) { new_id = cand_id; break; @@ -2840,7 +4169,7 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) h = btf_hash_array(t); for_each_dedup_cand(d, hash_entry, h) { cand_id = (__u32)(long)hash_entry->value; - cand = d->btf->types[cand_id]; + cand = btf_type_by_id(d->btf, cand_id); if (btf_equal_array(t, cand)) { new_id = cand_id; break; @@ -2872,7 +4201,7 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) h = btf_hash_fnproto(t); for_each_dedup_cand(d, hash_entry, h) { cand_id = (__u32)(long)hash_entry->value; - cand = d->btf->types[cand_id]; + cand = btf_type_by_id(d->btf, cand_id); if (btf_equal_fnproto(t, cand)) { new_id = cand_id; break; @@ -2920,9 +4249,9 @@ static int btf_dedup_ref_types(struct btf_dedup *d) */ static int btf_dedup_compact_types(struct btf_dedup *d) { - struct btf_type **new_types; + __u32 *new_offs; __u32 next_type_id = 1; - char *types_start, *p; + void *p; int i, len; /* we are going to reuse hypot_map to store compaction remapping */ @@ -2930,41 +4259,34 @@ static int btf_dedup_compact_types(struct btf_dedup *d) for (i = 1; i <= d->btf->nr_types; i++) d->hypot_map[i] = BTF_UNPROCESSED_ID; - types_start = d->btf->nohdr_data + d->btf->hdr->type_off; - p = types_start; + p = d->btf->types_data; for (i = 1; i <= d->btf->nr_types; i++) { if (d->map[i] != i) continue; - len = btf_type_size(d->btf->types[i]); + len = btf_type_size(btf__type_by_id(d->btf, i)); if (len < 0) return len; - memmove(p, d->btf->types[i], len); + memmove(p, btf__type_by_id(d->btf, i), len); d->hypot_map[i] = next_type_id; - d->btf->types[next_type_id] = (struct btf_type *)p; + d->btf->type_offs[next_type_id] = p - d->btf->types_data; p += len; next_type_id++; } /* shrink struct btf's internal types index and update btf_header */ d->btf->nr_types = next_type_id - 1; - d->btf->types_size = d->btf->nr_types; - d->btf->hdr->type_len = p - types_start; - new_types = libbpf_reallocarray(d->btf->types, (1 + d->btf->nr_types), - sizeof(struct btf_type *)); - if (!new_types) + d->btf->type_offs_cap = d->btf->nr_types + 1; + d->btf->hdr->type_len = p - d->btf->types_data; + new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap, + sizeof(*new_offs)); + if (!new_offs) return -ENOMEM; - d->btf->types = new_types; - - /* make sure string section follows type information without gaps */ - d->btf->hdr->str_off = p - (char *)d->btf->nohdr_data; - memmove(p, d->btf->strings, d->btf->hdr->str_len); - d->btf->strings = p; - p += d->btf->hdr->str_len; - - d->btf->data_size = p - (char *)d->btf->data; + d->btf->type_offs = new_offs; + d->btf->hdr->str_off = d->btf->hdr->type_len; + d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len; return 0; } @@ -2997,7 +4319,7 @@ static int btf_dedup_remap_type_id(struct btf_dedup *d, __u32 type_id) */ static int btf_dedup_remap_type(struct btf_dedup *d, __u32 type_id) { - struct btf_type *t = d->btf->types[type_id]; + struct btf_type *t = btf_type_by_id(d->btf, type_id); int i, r; switch (btf_kind(t)) { diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 2a55320d87d0..57247240a20a 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -5,6 +5,7 @@ #define __LIBBPF_BTF_H #include <stdarg.h> +#include <stdbool.h> #include <linux/btf.h> #include <linux/types.h> @@ -24,8 +25,14 @@ struct btf_type; struct bpf_object; +enum btf_endianness { + BTF_LITTLE_ENDIAN = 0, + BTF_BIG_ENDIAN = 1, +}; + LIBBPF_API void btf__free(struct btf *btf); LIBBPF_API struct btf *btf__new(const void *data, __u32 size); +LIBBPF_API struct btf *btf__new_empty(void); LIBBPF_API struct btf *btf__parse(const char *path, struct btf_ext **btf_ext); LIBBPF_API struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext); LIBBPF_API struct btf *btf__parse_raw(const char *path); @@ -40,6 +47,8 @@ LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 id); LIBBPF_API size_t btf__pointer_size(const struct btf *btf); LIBBPF_API int btf__set_pointer_size(struct btf *btf, size_t ptr_sz); +LIBBPF_API enum btf_endianness btf__endianness(const struct btf *btf); +LIBBPF_API int btf__set_endianness(struct btf *btf, enum btf_endianness endian); LIBBPF_API __s64 btf__resolve_size(const struct btf *btf, __u32 type_id); LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id); LIBBPF_API int btf__align_of(const struct btf *btf, __u32 id); @@ -47,6 +56,7 @@ LIBBPF_API int btf__fd(const struct btf *btf); LIBBPF_API void btf__set_fd(struct btf *btf, int fd); LIBBPF_API const void *btf__get_raw_data(const struct btf *btf, __u32 *size); LIBBPF_API const char *btf__name_by_offset(const struct btf *btf, __u32 offset); +LIBBPF_API const char *btf__str_by_offset(const struct btf *btf, __u32 offset); LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf); LIBBPF_API int btf__get_map_kv_tids(const struct btf *btf, const char *map_name, __u32 expected_key_size, @@ -72,6 +82,47 @@ LIBBPF_API __u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext); LIBBPF_API struct btf *libbpf_find_kernel_btf(void); +LIBBPF_API int btf__find_str(struct btf *btf, const char *s); +LIBBPF_API int btf__add_str(struct btf *btf, const char *s); + +LIBBPF_API int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding); +LIBBPF_API int btf__add_ptr(struct btf *btf, int ref_type_id); +LIBBPF_API int btf__add_array(struct btf *btf, + int index_type_id, int elem_type_id, __u32 nr_elems); +/* struct/union construction APIs */ +LIBBPF_API int btf__add_struct(struct btf *btf, const char *name, __u32 sz); +LIBBPF_API int btf__add_union(struct btf *btf, const char *name, __u32 sz); +LIBBPF_API int btf__add_field(struct btf *btf, const char *name, int field_type_id, + __u32 bit_offset, __u32 bit_size); + +/* enum construction APIs */ +LIBBPF_API int btf__add_enum(struct btf *btf, const char *name, __u32 bytes_sz); +LIBBPF_API int btf__add_enum_value(struct btf *btf, const char *name, __s64 value); + +enum btf_fwd_kind { + BTF_FWD_STRUCT = 0, + BTF_FWD_UNION = 1, + BTF_FWD_ENUM = 2, +}; + +LIBBPF_API int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind); +LIBBPF_API int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id); +LIBBPF_API int btf__add_volatile(struct btf *btf, int ref_type_id); +LIBBPF_API int btf__add_const(struct btf *btf, int ref_type_id); +LIBBPF_API int btf__add_restrict(struct btf *btf, int ref_type_id); + +/* func and func_proto construction APIs */ +LIBBPF_API int btf__add_func(struct btf *btf, const char *name, + enum btf_func_linkage linkage, int proto_type_id); +LIBBPF_API int btf__add_func_proto(struct btf *btf, int ret_type_id); +LIBBPF_API int btf__add_func_param(struct btf *btf, const char *name, int type_id); + +/* var & datasec construction APIs */ +LIBBPF_API int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id); +LIBBPF_API int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz); +LIBBPF_API int btf__add_datasec_var_info(struct btf *btf, int var_type_id, + __u32 offset, __u32 byte_sz); + struct btf_dedup_opts { unsigned int dedup_table_size; bool dont_resolve_fwds; diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 6c079b3c8679..2f9d685bd522 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -60,11 +60,14 @@ struct btf_dump { struct btf_dump_opts opts; int ptr_sz; bool strip_mods; + int last_id; /* per-type auxiliary state */ struct btf_dump_type_aux_state *type_states; + size_t type_states_cap; /* per-type optional cached unique name, must be freed, if present */ const char **cached_names; + size_t cached_names_cap; /* topo-sorted list of dependent type definitions */ __u32 *emit_queue; @@ -90,14 +93,7 @@ struct btf_dump { static size_t str_hash_fn(const void *key, void *ctx) { - const char *s = key; - size_t h = 0; - - while (*s) { - h = h * 31 + *s; - s++; - } - return h; + return str_hash(key); } static bool str_equal_fn(const void *a, const void *b, void *ctx) @@ -120,6 +116,7 @@ static void btf_dump_printf(const struct btf_dump *d, const char *fmt, ...) } static int btf_dump_mark_referenced(struct btf_dump *d); +static int btf_dump_resize(struct btf_dump *d); struct btf_dump *btf_dump__new(const struct btf *btf, const struct btf_ext *btf_ext, @@ -151,25 +148,8 @@ struct btf_dump *btf_dump__new(const struct btf *btf, d->ident_names = NULL; goto err; } - d->type_states = calloc(1 + btf__get_nr_types(d->btf), - sizeof(d->type_states[0])); - if (!d->type_states) { - err = -ENOMEM; - goto err; - } - d->cached_names = calloc(1 + btf__get_nr_types(d->btf), - sizeof(d->cached_names[0])); - if (!d->cached_names) { - err = -ENOMEM; - goto err; - } - /* VOID is special */ - d->type_states[0].order_state = ORDERED; - d->type_states[0].emit_state = EMITTED; - - /* eagerly determine referenced types for anon enums */ - err = btf_dump_mark_referenced(d); + err = btf_dump_resize(d); if (err) goto err; @@ -179,9 +159,38 @@ err: return ERR_PTR(err); } +static int btf_dump_resize(struct btf_dump *d) +{ + int err, last_id = btf__get_nr_types(d->btf); + + if (last_id <= d->last_id) + return 0; + + if (btf_ensure_mem((void **)&d->type_states, &d->type_states_cap, + sizeof(*d->type_states), last_id + 1)) + return -ENOMEM; + if (btf_ensure_mem((void **)&d->cached_names, &d->cached_names_cap, + sizeof(*d->cached_names), last_id + 1)) + return -ENOMEM; + + if (d->last_id == 0) { + /* VOID is special */ + d->type_states[0].order_state = ORDERED; + d->type_states[0].emit_state = EMITTED; + } + + /* eagerly determine referenced types for anon enums */ + err = btf_dump_mark_referenced(d); + if (err) + return err; + + d->last_id = last_id; + return 0; +} + void btf_dump__free(struct btf_dump *d) { - int i, cnt; + int i; if (IS_ERR_OR_NULL(d)) return; @@ -189,7 +198,7 @@ void btf_dump__free(struct btf_dump *d) free(d->type_states); if (d->cached_names) { /* any set cached name is owned by us and should be freed */ - for (i = 0, cnt = btf__get_nr_types(d->btf); i <= cnt; i++) { + for (i = 0; i <= d->last_id; i++) { if (d->cached_names[i]) free((void *)d->cached_names[i]); } @@ -229,6 +238,10 @@ int btf_dump__dump_type(struct btf_dump *d, __u32 id) if (id > btf__get_nr_types(d->btf)) return -EINVAL; + err = btf_dump_resize(d); + if (err) + return err; + d->emit_queue_cnt = 0; err = btf_dump_order_type(d, id, false); if (err < 0) @@ -258,7 +271,7 @@ static int btf_dump_mark_referenced(struct btf_dump *d) const struct btf_type *t; __u16 vlen; - for (i = 1; i <= n; i++) { + for (i = d->last_id + 1; i <= n; i++) { t = btf__type_by_id(d->btf, i); vlen = btf_vlen(t); @@ -313,6 +326,7 @@ static int btf_dump_mark_referenced(struct btf_dump *d) } return 0; } + static int btf_dump_add_emit_queue_id(struct btf_dump *d, __u32 id) { __u32 *new_queue; @@ -1056,11 +1070,15 @@ int btf_dump__emit_type_decl(struct btf_dump *d, __u32 id, const struct btf_dump_emit_type_decl_opts *opts) { const char *fname; - int lvl; + int lvl, err; if (!OPTS_VALID(opts, btf_dump_emit_type_decl_opts)) return -EINVAL; + err = btf_dump_resize(d); + if (err) + return -EINVAL; + fname = OPTS_GET(opts, field_name, ""); lvl = OPTS_GET(opts, indent_level, 0); d->strip_mods = OPTS_GET(opts, strip_mods, false); diff --git a/tools/lib/bpf/hashmap.h b/tools/lib/bpf/hashmap.h index e0af36b0e5d8..d9b385fe808c 100644 --- a/tools/lib/bpf/hashmap.h +++ b/tools/lib/bpf/hashmap.h @@ -25,6 +25,18 @@ static inline size_t hash_bits(size_t h, int bits) #endif } +/* generic C-string hashing function */ +static inline size_t str_hash(const char *s) +{ + size_t h = 0; + + while (*s) { + h = h * 31 + *s; + s++; + } + return h; +} + typedef size_t (*hashmap_hash_fn)(const void *key, void *ctx); typedef bool (*hashmap_equal_fn)(const void *key1, const void *key2, void *ctx); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 32dc444224d8..a4f55f8a460d 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -9390,9 +9390,11 @@ static struct bpf_link *attach_iter(const struct bpf_sec_def *sec, } static struct bpf_link * -bpf_program__attach_fd(struct bpf_program *prog, int target_fd, +bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id, const char *target_name) { + DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts, + .target_btf_id = btf_id); enum bpf_attach_type attach_type; char errmsg[STRERR_BUFSIZE]; struct bpf_link *link; @@ -9410,7 +9412,7 @@ bpf_program__attach_fd(struct bpf_program *prog, int target_fd, link->detach = &bpf_link__detach_fd; attach_type = bpf_program__get_expected_attach_type(prog); - link_fd = bpf_link_create(prog_fd, target_fd, attach_type, NULL); + link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts); if (link_fd < 0) { link_fd = -errno; free(link); @@ -9426,19 +9428,51 @@ bpf_program__attach_fd(struct bpf_program *prog, int target_fd, struct bpf_link * bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd) { - return bpf_program__attach_fd(prog, cgroup_fd, "cgroup"); + return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup"); } struct bpf_link * bpf_program__attach_netns(struct bpf_program *prog, int netns_fd) { - return bpf_program__attach_fd(prog, netns_fd, "netns"); + return bpf_program__attach_fd(prog, netns_fd, 0, "netns"); } struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex) { /* target_fd/target_ifindex use the same field in LINK_CREATE */ - return bpf_program__attach_fd(prog, ifindex, "xdp"); + return bpf_program__attach_fd(prog, ifindex, 0, "xdp"); +} + +struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog, + int target_fd, + const char *attach_func_name) +{ + int btf_id; + + if (!!target_fd != !!attach_func_name) { + pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n", + prog->name); + return ERR_PTR(-EINVAL); + } + + if (prog->type != BPF_PROG_TYPE_EXT) { + pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace", + prog->name); + return ERR_PTR(-EINVAL); + } + + if (target_fd) { + btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd); + if (btf_id < 0) + return ERR_PTR(btf_id); + + return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace"); + } else { + /* no target, so use raw_tracepoint_open for compatibility + * with old kernels + */ + return bpf_program__attach_trace(prog); + } } struct bpf_link * diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index a750f67a23f6..6909ee81113a 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -261,6 +261,9 @@ LIBBPF_API struct bpf_link * bpf_program__attach_netns(struct bpf_program *prog, int netns_fd); LIBBPF_API struct bpf_link * bpf_program__attach_xdp(struct bpf_program *prog, int ifindex); +LIBBPF_API struct bpf_link * +bpf_program__attach_freplace(struct bpf_program *prog, + int target_fd, const char *attach_func_name); struct bpf_map; diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 5f054dadf082..4ebfadf45b47 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -303,7 +303,34 @@ LIBBPF_0.1.0 { LIBBPF_0.2.0 { global: bpf_prog_bind_map; + bpf_prog_test_run_opts; + bpf_program__attach_freplace; bpf_program__section_name; + btf__add_array; + btf__add_const; + btf__add_enum; + btf__add_enum_value; + btf__add_datasec; + btf__add_datasec_var_info; + btf__add_field; + btf__add_func; + btf__add_func_param; + btf__add_func_proto; + btf__add_fwd; + btf__add_int; + btf__add_ptr; + btf__add_restrict; + btf__add_str; + btf__add_struct; + btf__add_typedef; + btf__add_union; + btf__add_var; + btf__add_volatile; + btf__endianness; + btf__find_str; + btf__new_empty; + btf__set_endianness; + btf__str_by_offset; perf_buffer__buffer_cnt; perf_buffer__buffer_fd; perf_buffer__epoll_fd; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4d1c366fca2c..d99bc847bf84 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -105,6 +105,10 @@ static inline void *libbpf_reallocarray(void *ptr, size_t nmemb, size_t size) return realloc(ptr, total); } +void *btf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, + size_t cur_cnt, size_t max_cnt, size_t add_cnt); +int btf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt); + static inline bool libbpf_validate_opts(const char *opts, size_t opts_sz, size_t user_sz, const char *type_name) @@ -136,6 +140,11 @@ static inline bool libbpf_validate_opts(const char *opts, ((opts) && opts->sz >= offsetofend(typeof(*(opts)), field)) #define OPTS_GET(opts, field, fallback_value) \ (OPTS_HAS(opts, field) ? (opts)->field : fallback_value) +#define OPTS_SET(opts, field, value) \ + do { \ + if (OPTS_HAS(opts, field)) \ + (opts)->field = value; \ + } while (0) int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz); int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore index e8fed558b8b8..3ab1200e172f 100644 --- a/tools/testing/selftests/bpf/.gitignore +++ b/tools/testing/selftests/bpf/.gitignore @@ -13,7 +13,6 @@ test_verifier_log feature test_sock test_sock_addr -test_sock_fields urandom_read test_sockmap test_lirc_mode2_user diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 59a5fa5fe837..bdbeafec371b 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -35,7 +35,7 @@ TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test test_verifier_log test_dev_cgroup test_tcpbpf_user \ test_sock test_sockmap get_cgroup_id_user test_socket_cookie \ test_cgroup_storage \ - test_netcnt test_tcpnotify_user test_sock_fields test_sysctl \ + test_netcnt test_tcpnotify_user test_sysctl \ test_progs-no_alu32 \ test_current_pid_tgid_new_ns diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 1a427685a8a8..332ed2f7b402 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -311,7 +311,6 @@ extern const struct bench bench_rename_kretprobe; extern const struct bench bench_rename_rawtp; extern const struct bench bench_rename_fentry; extern const struct bench bench_rename_fexit; -extern const struct bench bench_rename_fmodret; extern const struct bench bench_trig_base; extern const struct bench bench_trig_tp; extern const struct bench bench_trig_rawtp; @@ -333,7 +332,6 @@ static const struct bench *benchs[] = { &bench_rename_rawtp, &bench_rename_fentry, &bench_rename_fexit, - &bench_rename_fmodret, &bench_trig_base, &bench_trig_tp, &bench_trig_rawtp, @@ -464,4 +462,3 @@ int main(int argc, char **argv) return 0; } - diff --git a/tools/testing/selftests/bpf/benchs/bench_rename.c b/tools/testing/selftests/bpf/benchs/bench_rename.c index e74cff40f4fe..a967674098ad 100644 --- a/tools/testing/selftests/bpf/benchs/bench_rename.c +++ b/tools/testing/selftests/bpf/benchs/bench_rename.c @@ -106,12 +106,6 @@ static void setup_fexit() attach_bpf(ctx.skel->progs.prog5); } -static void setup_fmodret() -{ - setup_ctx(); - attach_bpf(ctx.skel->progs.prog6); -} - static void *consumer(void *input) { return NULL; @@ -182,14 +176,3 @@ const struct bench bench_rename_fexit = { .report_progress = hits_drops_report_progress, .report_final = hits_drops_report_final, }; - -const struct bench bench_rename_fmodret = { - .name = "rename-fmodret", - .validate = validate, - .setup = setup_fmodret, - .producer_thread = producer, - .consumer_thread = consumer, - .measure = measure, - .report_progress = hits_drops_report_progress, - .report_final = hits_drops_report_final, -}; diff --git a/tools/testing/selftests/bpf/bpf_tcp_helpers.h b/tools/testing/selftests/bpf/bpf_tcp_helpers.h index 5bf2fe9b1efa..2915664c335d 100644 --- a/tools/testing/selftests/bpf/bpf_tcp_helpers.h +++ b/tools/testing/selftests/bpf/bpf_tcp_helpers.h @@ -16,6 +16,7 @@ BPF_PROG(name, args) struct sock_common { unsigned char skc_state; + __u16 skc_num; } __attribute__((preserve_access_index)); enum sk_pacing { @@ -45,6 +46,10 @@ struct inet_connection_sock { __u64 icsk_ca_priv[104 / sizeof(__u64)]; } __attribute__((preserve_access_index)); +struct request_sock { + struct sock_common __req_common; +} __attribute__((preserve_access_index)); + struct tcp_sock { struct inet_connection_sock inet_conn; @@ -115,14 +120,6 @@ enum tcp_ca_event { CA_EVENT_ECN_IS_CE = 5, }; -enum tcp_ca_state { - TCP_CA_Open = 0, - TCP_CA_Disorder = 1, - TCP_CA_CWR = 2, - TCP_CA_Recovery = 3, - TCP_CA_Loss = 4 -}; - struct ack_sample { __u32 pkts_acked; __s32 rtt_us; diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c index fe1a83b9875c..448885b95eed 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c @@ -7,6 +7,7 @@ #include "bpf_iter_task.skel.h" #include "bpf_iter_task_stack.skel.h" #include "bpf_iter_task_file.skel.h" +#include "bpf_iter_task_btf.skel.h" #include "bpf_iter_tcp4.skel.h" #include "bpf_iter_tcp6.skel.h" #include "bpf_iter_udp4.skel.h" @@ -167,6 +168,83 @@ done: bpf_iter_task_file__destroy(skel); } +#define TASKBUFSZ 32768 + +static char taskbuf[TASKBUFSZ]; + +static int do_btf_read(struct bpf_iter_task_btf *skel) +{ + struct bpf_program *prog = skel->progs.dump_task_struct; + struct bpf_iter_task_btf__bss *bss = skel->bss; + int iter_fd = -1, len = 0, bufleft = TASKBUFSZ; + struct bpf_link *link; + char *buf = taskbuf; + int ret = 0; + + link = bpf_program__attach_iter(prog, NULL); + if (CHECK(IS_ERR(link), "attach_iter", "attach_iter failed\n")) + return ret; + + iter_fd = bpf_iter_create(bpf_link__fd(link)); + if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) + goto free_link; + + do { + len = read(iter_fd, buf, bufleft); + if (len > 0) { + buf += len; + bufleft -= len; + } + } while (len > 0); + + if (bss->skip) { + printf("%s:SKIP:no __builtin_btf_type_id\n", __func__); + ret = 1; + test__skip(); + goto free_link; + } + + if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno))) + goto free_link; + + CHECK(strstr(taskbuf, "(struct task_struct)") == NULL, + "check for btf representation of task_struct in iter data", + "struct task_struct not found"); +free_link: + if (iter_fd > 0) + close(iter_fd); + bpf_link__destroy(link); + return ret; +} + +static void test_task_btf(void) +{ + struct bpf_iter_task_btf__bss *bss; + struct bpf_iter_task_btf *skel; + int ret; + + skel = bpf_iter_task_btf__open_and_load(); + if (CHECK(!skel, "bpf_iter_task_btf__open_and_load", + "skeleton open_and_load failed\n")) + return; + + bss = skel->bss; + + ret = do_btf_read(skel); + if (ret) + goto cleanup; + + if (CHECK(bss->tasks == 0, "check if iterated over tasks", + "no task iteration, did BPF program run?\n")) + goto cleanup; + + CHECK(bss->seq_err != 0, "check for unexpected err", + "bpf_seq_printf_btf returned %ld", bss->seq_err); + +cleanup: + bpf_iter_task_btf__destroy(skel); +} + static void test_tcp4(void) { struct bpf_iter_tcp4 *skel; @@ -352,7 +430,7 @@ static void test_overflow(bool test_e2big_overflow, bool ret1) struct bpf_map_info map_info = {}; struct bpf_iter_test_kern4 *skel; struct bpf_link *link; - __u32 page_size; + __u32 iter_size; char *buf; skel = bpf_iter_test_kern4__open(); @@ -374,19 +452,19 @@ static void test_overflow(bool test_e2big_overflow, bool ret1) "map_creation failed: %s\n", strerror(errno))) goto free_map1; - /* bpf_seq_printf kernel buffer is one page, so one map + /* bpf_seq_printf kernel buffer is 8 pages, so one map * bpf_seq_write will mostly fill it, and the other map * will partially fill and then trigger overflow and need * bpf_seq_read restart. */ - page_size = sysconf(_SC_PAGE_SIZE); + iter_size = sysconf(_SC_PAGE_SIZE) << 3; if (test_e2big_overflow) { - skel->rodata->print_len = (page_size + 8) / 8; - expected_read_len = 2 * (page_size + 8); + skel->rodata->print_len = (iter_size + 8) / 8; + expected_read_len = 2 * (iter_size + 8); } else if (!ret1) { - skel->rodata->print_len = (page_size - 8) / 8; - expected_read_len = 2 * (page_size - 8); + skel->rodata->print_len = (iter_size - 8) / 8; + expected_read_len = 2 * (iter_size - 8); } else { skel->rodata->print_len = 1; expected_read_len = 2 * 8; @@ -957,6 +1035,8 @@ void test_bpf_iter(void) test_task_stack(); if (test__start_subtest("task_file")) test_task_file(); + if (test__start_subtest("task_btf")) + test_task_btf(); if (test__start_subtest("tcp4")) test_tcp4(); if (test__start_subtest("tcp6")) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index 39fb81d9daeb..c60091ee8a21 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -129,6 +129,109 @@ done: return err; } +static char *dump_buf; +static size_t dump_buf_sz; +static FILE *dump_buf_file; + +void test_btf_dump_incremental(void) +{ + struct btf *btf = NULL; + struct btf_dump *d = NULL; + struct btf_dump_opts opts; + int id, err, i; + + dump_buf_file = open_memstream(&dump_buf, &dump_buf_sz); + if (!ASSERT_OK_PTR(dump_buf_file, "dump_memstream")) + return; + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + goto err_out; + opts.ctx = dump_buf_file; + d = btf_dump__new(btf, NULL, &opts, btf_dump_printf); + if (!ASSERT_OK(libbpf_get_error(d), "btf_dump__new")) + goto err_out; + + /* First, generate BTF corresponding to the following C code: + * + * enum { VAL = 1 }; + * + * struct s { int x; }; + * + */ + id = btf__add_enum(btf, NULL, 4); + ASSERT_EQ(id, 1, "enum_id"); + err = btf__add_enum_value(btf, "VAL", 1); + ASSERT_OK(err, "enum_val_ok"); + + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 2, "int_id"); + + id = btf__add_struct(btf, "s", 4); + ASSERT_EQ(id, 3, "struct_id"); + err = btf__add_field(btf, "x", 2, 0, 0); + ASSERT_OK(err, "field_ok"); + + for (i = 1; i <= btf__get_nr_types(btf); i++) { + err = btf_dump__dump_type(d, i); + ASSERT_OK(err, "dump_type_ok"); + } + + fflush(dump_buf_file); + dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */ + ASSERT_STREQ(dump_buf, +"enum {\n" +" VAL = 1,\n" +"};\n" +"\n" +"struct s {\n" +" int x;\n" +"};\n\n", "c_dump1"); + + /* Now, after dumping original BTF, append another struct that embeds + * anonymous enum. It also has a name conflict with the first struct: + * + * struct s___2 { + * enum { VAL___2 = 1 } x; + * struct s s; + * }; + * + * This will test that btf_dump'er maintains internal state properly. + * Note that VAL___2 enum value. It's because we've already emitted + * that enum as a global anonymous enum, so btf_dump will ensure that + * enum values don't conflict; + * + */ + fseek(dump_buf_file, 0, SEEK_SET); + + id = btf__add_struct(btf, "s", 4); + ASSERT_EQ(id, 4, "struct_id"); + err = btf__add_field(btf, "x", 1, 0, 0); + ASSERT_OK(err, "field_ok"); + err = btf__add_field(btf, "s", 3, 32, 0); + ASSERT_OK(err, "field_ok"); + + for (i = 1; i <= btf__get_nr_types(btf); i++) { + err = btf_dump__dump_type(d, i); + ASSERT_OK(err, "dump_type_ok"); + } + + fflush(dump_buf_file); + dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */ + ASSERT_STREQ(dump_buf, +"struct s___2 {\n" +" enum {\n" +" VAL___2 = 1,\n" +" } x;\n" +" struct s s;\n" +"};\n\n" , "c_dump1"); + +err_out: + fclose(dump_buf_file); + free(dump_buf); + btf_dump__free(d); + btf__free(btf); +} + void test_btf_dump() { int i; @@ -140,4 +243,6 @@ void test_btf_dump() { test_btf_dump_case(i, &btf_dump_test_cases[i]); } + if (test__start_subtest("btf_dump: incremental")) + test_btf_dump_incremental(); } diff --git a/tools/testing/selftests/bpf/prog_tests/btf_endian.c b/tools/testing/selftests/bpf/prog_tests/btf_endian.c new file mode 100644 index 000000000000..8c52d72c876e --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_endian.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020 Facebook */ +#define _GNU_SOURCE +#include <string.h> +#include <byteswap.h> +#include <test_progs.h> +#include <bpf/btf.h> + +static int duration = 0; + +void test_btf_endian() { +#if __BYTE_ORDER == __LITTLE_ENDIAN + enum btf_endianness endian = BTF_LITTLE_ENDIAN; +#elif __BYTE_ORDER == __BIG_ENDIAN + enum btf_endianness endian = BTF_BIG_ENDIAN; +#else +#error "Unrecognized __BYTE_ORDER" +#endif + enum btf_endianness swap_endian = 1 - endian; + struct btf *btf = NULL, *swap_btf = NULL; + const void *raw_data, *swap_raw_data; + const struct btf_type *t; + const struct btf_header *hdr; + __u32 raw_sz, swap_raw_sz; + int var_id; + + /* Load BTF in native endianness */ + btf = btf__parse_elf("btf_dump_test_case_syntax.o", NULL); + if (!ASSERT_OK_PTR(btf, "parse_native_btf")) + goto err_out; + + ASSERT_EQ(btf__endianness(btf), endian, "endian"); + btf__set_endianness(btf, swap_endian); + ASSERT_EQ(btf__endianness(btf), swap_endian, "endian"); + + /* Get raw BTF data in non-native endianness... */ + raw_data = btf__get_raw_data(btf, &raw_sz); + if (!ASSERT_OK_PTR(raw_data, "raw_data_inverted")) + goto err_out; + + /* ...and open it as a new BTF instance */ + swap_btf = btf__new(raw_data, raw_sz); + if (!ASSERT_OK_PTR(swap_btf, "parse_swap_btf")) + goto err_out; + + ASSERT_EQ(btf__endianness(swap_btf), swap_endian, "endian"); + ASSERT_EQ(btf__get_nr_types(swap_btf), btf__get_nr_types(btf), "nr_types"); + + swap_raw_data = btf__get_raw_data(swap_btf, &swap_raw_sz); + if (!ASSERT_OK_PTR(swap_raw_data, "swap_raw_data")) + goto err_out; + + /* both raw data should be identical (with non-native endianness) */ + ASSERT_OK(memcmp(raw_data, swap_raw_data, raw_sz), "mem_identical"); + + /* make sure that at least BTF header data is really swapped */ + hdr = swap_raw_data; + ASSERT_EQ(bswap_16(hdr->magic), BTF_MAGIC, "btf_magic_swapped"); + ASSERT_EQ(raw_sz, swap_raw_sz, "raw_sizes"); + + /* swap it back to native endianness */ + btf__set_endianness(swap_btf, endian); + swap_raw_data = btf__get_raw_data(swap_btf, &swap_raw_sz); + if (!ASSERT_OK_PTR(swap_raw_data, "swap_raw_data")) + goto err_out; + + /* now header should have native BTF_MAGIC */ + hdr = swap_raw_data; + ASSERT_EQ(hdr->magic, BTF_MAGIC, "btf_magic_native"); + ASSERT_EQ(raw_sz, swap_raw_sz, "raw_sizes"); + + /* now modify original BTF */ + var_id = btf__add_var(btf, "some_var", BTF_VAR_GLOBAL_ALLOCATED, 1); + CHECK(var_id <= 0, "var_id", "failed %d\n", var_id); + + btf__free(swap_btf); + swap_btf = NULL; + + btf__set_endianness(btf, swap_endian); + raw_data = btf__get_raw_data(btf, &raw_sz); + if (!ASSERT_OK_PTR(raw_data, "raw_data_inverted")) + goto err_out; + + /* and re-open swapped raw data again */ + swap_btf = btf__new(raw_data, raw_sz); + if (!ASSERT_OK_PTR(swap_btf, "parse_swap_btf")) + goto err_out; + + ASSERT_EQ(btf__endianness(swap_btf), swap_endian, "endian"); + ASSERT_EQ(btf__get_nr_types(swap_btf), btf__get_nr_types(btf), "nr_types"); + + /* the type should appear as if it was stored in native endianness */ + t = btf__type_by_id(swap_btf, var_id); + ASSERT_STREQ(btf__str_by_offset(swap_btf, t->name_off), "some_var", "var_name"); + ASSERT_EQ(btf_var(t)->linkage, BTF_VAR_GLOBAL_ALLOCATED, "var_linkage"); + ASSERT_EQ(t->type, 1, "var_type"); + +err_out: + btf__free(btf); + btf__free(swap_btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c b/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c new file mode 100644 index 000000000000..86ccf37e26b3 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020 Facebook */ + +#define _GNU_SOURCE +#include <netinet/in.h> +#include <arpa/inet.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <sched.h> +#include <linux/compiler.h> +#include <bpf/libbpf.h> + +#include "network_helpers.h" +#include "test_progs.h" +#include "test_btf_skc_cls_ingress.skel.h" + +static struct test_btf_skc_cls_ingress *skel; +struct sockaddr_in6 srv_sa6; +static __u32 duration; + +#define PROG_PIN_FILE "/sys/fs/bpf/btf_skc_cls_ingress" + +static int write_sysctl(const char *sysctl, const char *value) +{ + int fd, err, len; + + fd = open(sysctl, O_WRONLY); + if (CHECK(fd == -1, "open sysctl", "open(%s): %s (%d)\n", + sysctl, strerror(errno), errno)) + return -1; + + len = strlen(value); + err = write(fd, value, len); + close(fd); + if (CHECK(err != len, "write sysctl", + "write(%s, %s, %d): err:%d %s (%d)\n", + sysctl, value, len, err, strerror(errno), errno)) + return -1; + + return 0; +} + +static int prepare_netns(void) +{ + if (CHECK(unshare(CLONE_NEWNET), "create netns", + "unshare(CLONE_NEWNET): %s (%d)", + strerror(errno), errno)) + return -1; + + if (CHECK(system("ip link set dev lo up"), + "ip link set dev lo up", "failed\n")) + return -1; + + if (CHECK(system("tc qdisc add dev lo clsact"), + "tc qdisc add dev lo clsact", "failed\n")) + return -1; + + if (CHECK(system("tc filter add dev lo ingress bpf direct-action object-pinned " PROG_PIN_FILE), + "install tc cls-prog at ingress", "failed\n")) + return -1; + + /* Ensure 20 bytes options (i.e. in total 40 bytes tcp header) for the + * bpf_tcp_gen_syncookie() helper. + */ + if (write_sysctl("/proc/sys/net/ipv4/tcp_window_scaling", "1") || + write_sysctl("/proc/sys/net/ipv4/tcp_timestamps", "1") || + write_sysctl("/proc/sys/net/ipv4/tcp_sack", "1")) + return -1; + + return 0; +} + +static void reset_test(void) +{ + memset(&skel->bss->srv_sa6, 0, sizeof(skel->bss->srv_sa6)); + skel->bss->listen_tp_sport = 0; + skel->bss->req_sk_sport = 0; + skel->bss->recv_cookie = 0; + skel->bss->gen_cookie = 0; + skel->bss->linum = 0; +} + +static void print_err_line(void) +{ + if (skel->bss->linum) + printf("bpf prog error at line %u\n", skel->bss->linum); +} + +static void test_conn(void) +{ + int listen_fd = -1, cli_fd = -1, err; + socklen_t addrlen = sizeof(srv_sa6); + int srv_port; + + if (write_sysctl("/proc/sys/net/ipv4/tcp_syncookies", "1")) + return; + + listen_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0); + if (CHECK_FAIL(listen_fd == -1)) + return; + + err = getsockname(listen_fd, (struct sockaddr *)&srv_sa6, &addrlen); + if (CHECK(err, "getsockname(listen_fd)", "err:%d errno:%d\n", err, + errno)) + goto done; + memcpy(&skel->bss->srv_sa6, &srv_sa6, sizeof(srv_sa6)); + srv_port = ntohs(srv_sa6.sin6_port); + + cli_fd = connect_to_fd(listen_fd, 0); + if (CHECK_FAIL(cli_fd == -1)) + goto done; + + if (CHECK(skel->bss->listen_tp_sport != srv_port || + skel->bss->req_sk_sport != srv_port, + "Unexpected sk src port", + "listen_tp_sport:%u req_sk_sport:%u expected:%u\n", + skel->bss->listen_tp_sport, skel->bss->req_sk_sport, + srv_port)) + goto done; + + if (CHECK(skel->bss->gen_cookie || skel->bss->recv_cookie, + "Unexpected syncookie states", + "gen_cookie:%u recv_cookie:%u\n", + skel->bss->gen_cookie, skel->bss->recv_cookie)) + goto done; + + CHECK(skel->bss->linum, "bpf prog detected error", "at line %u\n", + skel->bss->linum); + +done: + if (listen_fd != -1) + close(listen_fd); + if (cli_fd != -1) + close(cli_fd); +} + +static void test_syncookie(void) +{ + int listen_fd = -1, cli_fd = -1, err; + socklen_t addrlen = sizeof(srv_sa6); + int srv_port; + + /* Enforce syncookie mode */ + if (write_sysctl("/proc/sys/net/ipv4/tcp_syncookies", "2")) + return; + + listen_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0); + if (CHECK_FAIL(listen_fd == -1)) + return; + + err = getsockname(listen_fd, (struct sockaddr *)&srv_sa6, &addrlen); + if (CHECK(err, "getsockname(listen_fd)", "err:%d errno:%d\n", err, + errno)) + goto done; + memcpy(&skel->bss->srv_sa6, &srv_sa6, sizeof(srv_sa6)); + srv_port = ntohs(srv_sa6.sin6_port); + + cli_fd = connect_to_fd(listen_fd, 0); + if (CHECK_FAIL(cli_fd == -1)) + goto done; + + if (CHECK(skel->bss->listen_tp_sport != srv_port, + "Unexpected tp src port", + "listen_tp_sport:%u expected:%u\n", + skel->bss->listen_tp_sport, srv_port)) + goto done; + + if (CHECK(skel->bss->req_sk_sport, + "Unexpected req_sk src port", + "req_sk_sport:%u expected:0\n", + skel->bss->req_sk_sport)) + goto done; + + if (CHECK(!skel->bss->gen_cookie || + skel->bss->gen_cookie != skel->bss->recv_cookie, + "Unexpected syncookie states", + "gen_cookie:%u recv_cookie:%u\n", + skel->bss->gen_cookie, skel->bss->recv_cookie)) + goto done; + + CHECK(skel->bss->linum, "bpf prog detected error", "at line %u\n", + skel->bss->linum); + +done: + if (listen_fd != -1) + close(listen_fd); + if (cli_fd != -1) + close(cli_fd); +} + +struct test { + const char *desc; + void (*run)(void); +}; + +#define DEF_TEST(name) { #name, test_##name } +static struct test tests[] = { + DEF_TEST(conn), + DEF_TEST(syncookie), +}; + +void test_btf_skc_cls_ingress(void) +{ + int i, err; + + skel = test_btf_skc_cls_ingress__open_and_load(); + if (CHECK(!skel, "test_btf_skc_cls_ingress__open_and_load", "failed\n")) + return; + + err = bpf_program__pin(skel->progs.cls_ingress, PROG_PIN_FILE); + if (CHECK(err, "bpf_program__pin", + "cannot pin bpf prog to %s. err:%d\n", PROG_PIN_FILE, err)) { + test_btf_skc_cls_ingress__destroy(skel); + return; + } + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (!test__start_subtest(tests[i].desc)) + continue; + + if (prepare_netns()) + break; + + tests[i].run(); + + print_err_line(); + reset_test(); + } + + bpf_program__unpin(skel->progs.cls_ingress, PROG_PIN_FILE); + test_btf_skc_cls_ingress__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_write.c b/tools/testing/selftests/bpf/prog_tests/btf_write.c new file mode 100644 index 000000000000..314e1e7c36df --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_write.c @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020 Facebook */ +#include <test_progs.h> +#include <bpf/btf.h> + +static int duration = 0; + +void test_btf_write() { + const struct btf_var_secinfo *vi; + const struct btf_type *t; + const struct btf_member *m; + const struct btf_enum *v; + const struct btf_param *p; + struct btf *btf; + int id, err, str_off; + + btf = btf__new_empty(); + if (CHECK(IS_ERR(btf), "new_empty", "failed: %ld\n", PTR_ERR(btf))) + return; + + str_off = btf__find_str(btf, "int"); + ASSERT_EQ(str_off, -ENOENT, "int_str_missing_off"); + + str_off = btf__add_str(btf, "int"); + ASSERT_EQ(str_off, 1, "int_str_off"); + + str_off = btf__find_str(btf, "int"); + ASSERT_EQ(str_off, 1, "int_str_found_off"); + + /* BTF_KIND_INT */ + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "int_id"); + + t = btf__type_by_id(btf, 1); + /* should re-use previously added "int" string */ + ASSERT_EQ(t->name_off, str_off, "int_name_off"); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "int", "int_name"); + ASSERT_EQ(btf_kind(t), BTF_KIND_INT, "int_kind"); + ASSERT_EQ(t->size, 4, "int_sz"); + ASSERT_EQ(btf_int_encoding(t), BTF_INT_SIGNED, "int_enc"); + ASSERT_EQ(btf_int_bits(t), 32, "int_bits"); + + /* invalid int size */ + id = btf__add_int(btf, "bad sz int", 7, 0); + ASSERT_ERR(id, "int_bad_sz"); + /* invalid encoding */ + id = btf__add_int(btf, "bad enc int", 4, 123); + ASSERT_ERR(id, "int_bad_enc"); + /* NULL name */ + id = btf__add_int(btf, NULL, 4, 0); + ASSERT_ERR(id, "int_bad_null_name"); + /* empty name */ + id = btf__add_int(btf, "", 4, 0); + ASSERT_ERR(id, "int_bad_empty_name"); + + /* PTR/CONST/VOLATILE/RESTRICT */ + id = btf__add_ptr(btf, 1); + ASSERT_EQ(id, 2, "ptr_id"); + t = btf__type_by_id(btf, 2); + ASSERT_EQ(btf_kind(t), BTF_KIND_PTR, "ptr_kind"); + ASSERT_EQ(t->type, 1, "ptr_type"); + + id = btf__add_const(btf, 5); /* points forward to restrict */ + ASSERT_EQ(id, 3, "const_id"); + t = btf__type_by_id(btf, 3); + ASSERT_EQ(btf_kind(t), BTF_KIND_CONST, "const_kind"); + ASSERT_EQ(t->type, 5, "const_type"); + + id = btf__add_volatile(btf, 3); + ASSERT_EQ(id, 4, "volatile_id"); + t = btf__type_by_id(btf, 4); + ASSERT_EQ(btf_kind(t), BTF_KIND_VOLATILE, "volatile_kind"); + ASSERT_EQ(t->type, 3, "volatile_type"); + + id = btf__add_restrict(btf, 4); + ASSERT_EQ(id, 5, "restrict_id"); + t = btf__type_by_id(btf, 5); + ASSERT_EQ(btf_kind(t), BTF_KIND_RESTRICT, "restrict_kind"); + ASSERT_EQ(t->type, 4, "restrict_type"); + + /* ARRAY */ + id = btf__add_array(btf, 1, 2, 10); /* int *[10] */ + ASSERT_EQ(id, 6, "array_id"); + t = btf__type_by_id(btf, 6); + ASSERT_EQ(btf_kind(t), BTF_KIND_ARRAY, "array_kind"); + ASSERT_EQ(btf_array(t)->index_type, 1, "array_index_type"); + ASSERT_EQ(btf_array(t)->type, 2, "array_elem_type"); + ASSERT_EQ(btf_array(t)->nelems, 10, "array_nelems"); + + /* STRUCT */ + err = btf__add_field(btf, "field", 1, 0, 0); + ASSERT_ERR(err, "no_struct_field"); + id = btf__add_struct(btf, "s1", 8); + ASSERT_EQ(id, 7, "struct_id"); + err = btf__add_field(btf, "f1", 1, 0, 0); + ASSERT_OK(err, "f1_res"); + err = btf__add_field(btf, "f2", 1, 32, 16); + ASSERT_OK(err, "f2_res"); + + t = btf__type_by_id(btf, 7); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "s1", "struct_name"); + ASSERT_EQ(btf_kind(t), BTF_KIND_STRUCT, "struct_kind"); + ASSERT_EQ(btf_vlen(t), 2, "struct_vlen"); + ASSERT_EQ(btf_kflag(t), true, "struct_kflag"); + ASSERT_EQ(t->size, 8, "struct_sz"); + m = btf_members(t) + 0; + ASSERT_STREQ(btf__str_by_offset(btf, m->name_off), "f1", "f1_name"); + ASSERT_EQ(m->type, 1, "f1_type"); + ASSERT_EQ(btf_member_bit_offset(t, 0), 0, "f1_bit_off"); + ASSERT_EQ(btf_member_bitfield_size(t, 0), 0, "f1_bit_sz"); + m = btf_members(t) + 1; + ASSERT_STREQ(btf__str_by_offset(btf, m->name_off), "f2", "f2_name"); + ASSERT_EQ(m->type, 1, "f2_type"); + ASSERT_EQ(btf_member_bit_offset(t, 1), 32, "f2_bit_off"); + ASSERT_EQ(btf_member_bitfield_size(t, 1), 16, "f2_bit_sz"); + + /* UNION */ + id = btf__add_union(btf, "u1", 8); + ASSERT_EQ(id, 8, "union_id"); + + /* invalid, non-zero offset */ + err = btf__add_field(btf, "field", 1, 1, 0); + ASSERT_ERR(err, "no_struct_field"); + + err = btf__add_field(btf, "f1", 1, 0, 16); + ASSERT_OK(err, "f1_res"); + + t = btf__type_by_id(btf, 8); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "u1", "union_name"); + ASSERT_EQ(btf_kind(t), BTF_KIND_UNION, "union_kind"); + ASSERT_EQ(btf_vlen(t), 1, "union_vlen"); + ASSERT_EQ(btf_kflag(t), true, "union_kflag"); + ASSERT_EQ(t->size, 8, "union_sz"); + m = btf_members(t) + 0; + ASSERT_STREQ(btf__str_by_offset(btf, m->name_off), "f1", "f1_name"); + ASSERT_EQ(m->type, 1, "f1_type"); + ASSERT_EQ(btf_member_bit_offset(t, 0), 0, "f1_bit_off"); + ASSERT_EQ(btf_member_bitfield_size(t, 0), 16, "f1_bit_sz"); + + /* ENUM */ + id = btf__add_enum(btf, "e1", 4); + ASSERT_EQ(id, 9, "enum_id"); + err = btf__add_enum_value(btf, "v1", 1); + ASSERT_OK(err, "v1_res"); + err = btf__add_enum_value(btf, "v2", 2); + ASSERT_OK(err, "v2_res"); + + t = btf__type_by_id(btf, 9); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "e1", "enum_name"); + ASSERT_EQ(btf_kind(t), BTF_KIND_ENUM, "enum_kind"); + ASSERT_EQ(btf_vlen(t), 2, "enum_vlen"); + ASSERT_EQ(t->size, 4, "enum_sz"); + v = btf_enum(t) + 0; + ASSERT_STREQ(btf__str_by_offset(btf, v->name_off), "v1", "v1_name"); + ASSERT_EQ(v->val, 1, "v1_val"); + v = btf_enum(t) + 1; + ASSERT_STREQ(btf__str_by_offset(btf, v->name_off), "v2", "v2_name"); + ASSERT_EQ(v->val, 2, "v2_val"); + + /* FWDs */ + id = btf__add_fwd(btf, "struct_fwd", BTF_FWD_STRUCT); + ASSERT_EQ(id, 10, "struct_fwd_id"); + t = btf__type_by_id(btf, 10); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "struct_fwd", "fwd_name"); + ASSERT_EQ(btf_kind(t), BTF_KIND_FWD, "fwd_kind"); + ASSERT_EQ(btf_kflag(t), 0, "fwd_kflag"); + + id = btf__add_fwd(btf, "union_fwd", BTF_FWD_UNION); + ASSERT_EQ(id, 11, "union_fwd_id"); + t = btf__type_by_id(btf, 11); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "union_fwd", "fwd_name"); + ASSERT_EQ(btf_kind(t), BTF_KIND_FWD, "fwd_kind"); + ASSERT_EQ(btf_kflag(t), 1, "fwd_kflag"); + + id = btf__add_fwd(btf, "enum_fwd", BTF_FWD_ENUM); + ASSERT_EQ(id, 12, "enum_fwd_id"); + t = btf__type_by_id(btf, 12); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "enum_fwd", "fwd_name"); + ASSERT_EQ(btf_kind(t), BTF_KIND_ENUM, "enum_fwd_kind"); + ASSERT_EQ(btf_vlen(t), 0, "enum_fwd_kind"); + ASSERT_EQ(t->size, 4, "enum_fwd_sz"); + + /* TYPEDEF */ + id = btf__add_typedef(btf, "typedef1", 1); + ASSERT_EQ(id, 13, "typedef_fwd_id"); + t = btf__type_by_id(btf, 13); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "typedef1", "typedef_name"); + ASSERT_EQ(btf_kind(t), BTF_KIND_TYPEDEF, "typedef_kind"); + ASSERT_EQ(t->type, 1, "typedef_type"); + + /* FUNC & FUNC_PROTO */ + id = btf__add_func(btf, "func1", BTF_FUNC_GLOBAL, 15); + ASSERT_EQ(id, 14, "func_id"); + t = btf__type_by_id(btf, 14); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "func1", "func_name"); + ASSERT_EQ(t->type, 15, "func_type"); + ASSERT_EQ(btf_kind(t), BTF_KIND_FUNC, "func_kind"); + ASSERT_EQ(btf_vlen(t), BTF_FUNC_GLOBAL, "func_vlen"); + + id = btf__add_func_proto(btf, 1); + ASSERT_EQ(id, 15, "func_proto_id"); + err = btf__add_func_param(btf, "p1", 1); + ASSERT_OK(err, "p1_res"); + err = btf__add_func_param(btf, "p2", 2); + ASSERT_OK(err, "p2_res"); + + t = btf__type_by_id(btf, 15); + ASSERT_EQ(btf_kind(t), BTF_KIND_FUNC_PROTO, "func_proto_kind"); + ASSERT_EQ(btf_vlen(t), 2, "func_proto_vlen"); + ASSERT_EQ(t->type, 1, "func_proto_ret_type"); + p = btf_params(t) + 0; + ASSERT_STREQ(btf__str_by_offset(btf, p->name_off), "p1", "p1_name"); + ASSERT_EQ(p->type, 1, "p1_type"); + p = btf_params(t) + 1; + ASSERT_STREQ(btf__str_by_offset(btf, p->name_off), "p2", "p2_name"); + ASSERT_EQ(p->type, 2, "p2_type"); + + /* VAR */ + id = btf__add_var(btf, "var1", BTF_VAR_GLOBAL_ALLOCATED, 1); + ASSERT_EQ(id, 16, "var_id"); + t = btf__type_by_id(btf, 16); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "var1", "var_name"); + ASSERT_EQ(btf_kind(t), BTF_KIND_VAR, "var_kind"); + ASSERT_EQ(t->type, 1, "var_type"); + ASSERT_EQ(btf_var(t)->linkage, BTF_VAR_GLOBAL_ALLOCATED, "var_type"); + + /* DATASECT */ + id = btf__add_datasec(btf, "datasec1", 12); + ASSERT_EQ(id, 17, "datasec_id"); + err = btf__add_datasec_var_info(btf, 1, 4, 8); + ASSERT_OK(err, "v1_res"); + + t = btf__type_by_id(btf, 17); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "datasec1", "datasec_name"); + ASSERT_EQ(t->size, 12, "datasec_sz"); + ASSERT_EQ(btf_kind(t), BTF_KIND_DATASEC, "datasec_kind"); + ASSERT_EQ(btf_vlen(t), 1, "datasec_vlen"); + vi = btf_var_secinfos(t) + 0; + ASSERT_EQ(vi->type, 1, "v1_type"); + ASSERT_EQ(vi->offset, 4, "v1_off"); + ASSERT_EQ(vi->size, 8, "v1_sz"); + + btf__free(btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index eda682727787..5c0448910426 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -2,36 +2,79 @@ /* Copyright (c) 2019 Facebook */ #include <test_progs.h> #include <network_helpers.h> +#include <bpf/btf.h> + +typedef int (*test_cb)(struct bpf_object *obj); + +static int check_data_map(struct bpf_object *obj, int prog_cnt, bool reset) +{ + struct bpf_map *data_map = NULL, *map; + __u64 *result = NULL; + const int zero = 0; + __u32 duration = 0; + int ret = -1, i; + + result = malloc((prog_cnt + 32 /* spare */) * sizeof(__u64)); + if (CHECK(!result, "alloc_memory", "failed to alloc memory")) + return -ENOMEM; + + bpf_object__for_each_map(map, obj) + if (bpf_map__is_internal(map)) { + data_map = map; + break; + } + if (CHECK(!data_map, "find_data_map", "data map not found\n")) + goto out; + + ret = bpf_map_lookup_elem(bpf_map__fd(data_map), &zero, result); + if (CHECK(ret, "get_result", + "failed to get output data: %d\n", ret)) + goto out; + + for (i = 0; i < prog_cnt; i++) { + if (CHECK(result[i] != 1, "result", + "fexit_bpf2bpf result[%d] failed err %llu\n", + i, result[i])) + goto out; + result[i] = 0; + } + if (reset) { + ret = bpf_map_update_elem(bpf_map__fd(data_map), &zero, result, 0); + if (CHECK(ret, "reset_result", "failed to reset result\n")) + goto out; + } + + ret = 0; +out: + free(result); + return ret; +} static void test_fexit_bpf2bpf_common(const char *obj_file, const char *target_obj_file, int prog_cnt, const char **prog_name, - bool run_prog) + bool run_prog, + test_cb cb) { - struct bpf_object *obj = NULL, *pkt_obj; - int err, pkt_fd, i; - struct bpf_link **link = NULL; + struct bpf_object *obj = NULL, *tgt_obj; struct bpf_program **prog = NULL; + struct bpf_link **link = NULL; __u32 duration = 0, retval; - struct bpf_map *data_map; - const int zero = 0; - __u64 *result = NULL; + int err, tgt_fd, i; err = bpf_prog_load(target_obj_file, BPF_PROG_TYPE_UNSPEC, - &pkt_obj, &pkt_fd); + &tgt_obj, &tgt_fd); if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n", target_obj_file, err, errno)) return; DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, - .attach_prog_fd = pkt_fd, + .attach_prog_fd = tgt_fd, ); link = calloc(sizeof(struct bpf_link *), prog_cnt); prog = calloc(sizeof(struct bpf_program *), prog_cnt); - result = malloc((prog_cnt + 32 /* spare */) * sizeof(__u64)); - if (CHECK(!link || !prog || !result, "alloc_memory", - "failed to alloc memory")) + if (CHECK(!link || !prog, "alloc_memory", "failed to alloc memory")) goto close_prog; obj = bpf_object__open_file(obj_file, &opts); @@ -53,39 +96,33 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, goto close_prog; } - if (!run_prog) - goto close_prog; + if (cb) { + err = cb(obj); + if (err) + goto close_prog; + } - data_map = bpf_object__find_map_by_name(obj, "fexit_bp.bss"); - if (CHECK(!data_map, "find_data_map", "data map not found\n")) + if (!run_prog) goto close_prog; - err = bpf_prog_test_run(pkt_fd, 1, &pkt_v6, sizeof(pkt_v6), + err = bpf_prog_test_run(tgt_fd, 1, &pkt_v6, sizeof(pkt_v6), NULL, NULL, &retval, &duration); CHECK(err || retval, "ipv6", "err %d errno %d retval %d duration %d\n", err, errno, retval, duration); - err = bpf_map_lookup_elem(bpf_map__fd(data_map), &zero, result); - if (CHECK(err, "get_result", - "failed to get output data: %d\n", err)) + if (check_data_map(obj, prog_cnt, false)) goto close_prog; - for (i = 0; i < prog_cnt; i++) - if (CHECK(result[i] != 1, "result", "fexit_bpf2bpf failed err %llu\n", - result[i])) - goto close_prog; - close_prog: for (i = 0; i < prog_cnt; i++) if (!IS_ERR_OR_NULL(link[i])) bpf_link__destroy(link[i]); if (!IS_ERR_OR_NULL(obj)) bpf_object__close(obj); - bpf_object__close(pkt_obj); + bpf_object__close(tgt_obj); free(link); free(prog); - free(result); } static void test_target_no_callees(void) @@ -96,7 +133,7 @@ static void test_target_no_callees(void) test_fexit_bpf2bpf_common("./fexit_bpf2bpf_simple.o", "./test_pkt_md_access.o", ARRAY_SIZE(prog_name), - prog_name, true); + prog_name, true, NULL); } static void test_target_yes_callees(void) @@ -110,7 +147,7 @@ static void test_target_yes_callees(void) test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o", "./test_pkt_access.o", ARRAY_SIZE(prog_name), - prog_name, true); + prog_name, true, NULL); } static void test_func_replace(void) @@ -128,7 +165,7 @@ static void test_func_replace(void) test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o", "./test_pkt_access.o", ARRAY_SIZE(prog_name), - prog_name, true); + prog_name, true, NULL); } static void test_func_replace_verify(void) @@ -139,9 +176,116 @@ static void test_func_replace_verify(void) test_fexit_bpf2bpf_common("./freplace_connect4.o", "./connect4_prog.o", ARRAY_SIZE(prog_name), - prog_name, false); + prog_name, false, NULL); } +static int test_second_attach(struct bpf_object *obj) +{ + const char *prog_name = "freplace/get_constant"; + const char *tgt_name = prog_name + 9; /* cut off freplace/ */ + const char *tgt_obj_file = "./test_pkt_access.o"; + struct bpf_program *prog = NULL; + struct bpf_object *tgt_obj; + __u32 duration = 0, retval; + struct bpf_link *link; + int err = 0, tgt_fd; + + prog = bpf_object__find_program_by_title(obj, prog_name); + if (CHECK(!prog, "find_prog", "prog %s not found\n", prog_name)) + return -ENOENT; + + err = bpf_prog_load(tgt_obj_file, BPF_PROG_TYPE_UNSPEC, + &tgt_obj, &tgt_fd); + if (CHECK(err, "second_prog_load", "file %s err %d errno %d\n", + tgt_obj_file, err, errno)) + return err; + + link = bpf_program__attach_freplace(prog, tgt_fd, tgt_name); + if (CHECK(IS_ERR(link), "second_link", "failed to attach second link prog_fd %d tgt_fd %d\n", bpf_program__fd(prog), tgt_fd)) + goto out; + + err = bpf_prog_test_run(tgt_fd, 1, &pkt_v6, sizeof(pkt_v6), + NULL, NULL, &retval, &duration); + if (CHECK(err || retval, "ipv6", + "err %d errno %d retval %d duration %d\n", + err, errno, retval, duration)) + goto out; + + err = check_data_map(obj, 1, true); + if (err) + goto out; + +out: + bpf_link__destroy(link); + bpf_object__close(tgt_obj); + return err; +} + +static void test_func_replace_multi(void) +{ + const char *prog_name[] = { + "freplace/get_constant", + }; + test_fexit_bpf2bpf_common("./freplace_get_constant.o", + "./test_pkt_access.o", + ARRAY_SIZE(prog_name), + prog_name, true, test_second_attach); +} + +static void test_fmod_ret_freplace(void) +{ + struct bpf_object *freplace_obj = NULL, *pkt_obj, *fmod_obj = NULL; + const char *freplace_name = "./freplace_get_constant.o"; + const char *fmod_ret_name = "./fmod_ret_freplace.o"; + DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts); + const char *tgt_name = "./test_pkt_access.o"; + struct bpf_link *freplace_link = NULL; + struct bpf_program *prog; + __u32 duration = 0; + int err, pkt_fd; + + err = bpf_prog_load(tgt_name, BPF_PROG_TYPE_UNSPEC, + &pkt_obj, &pkt_fd); + /* the target prog should load fine */ + if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n", + tgt_name, err, errno)) + return; + opts.attach_prog_fd = pkt_fd; + + freplace_obj = bpf_object__open_file(freplace_name, &opts); + if (CHECK(IS_ERR_OR_NULL(freplace_obj), "freplace_obj_open", + "failed to open %s: %ld\n", freplace_name, + PTR_ERR(freplace_obj))) + goto out; + + err = bpf_object__load(freplace_obj); + if (CHECK(err, "freplace_obj_load", "err %d\n", err)) + goto out; + + prog = bpf_program__next(NULL, freplace_obj); + freplace_link = bpf_program__attach_trace(prog); + if (CHECK(IS_ERR(freplace_link), "freplace_attach_trace", "failed to link\n")) + goto out; + + opts.attach_prog_fd = bpf_program__fd(prog); + fmod_obj = bpf_object__open_file(fmod_ret_name, &opts); + if (CHECK(IS_ERR_OR_NULL(fmod_obj), "fmod_obj_open", + "failed to open %s: %ld\n", fmod_ret_name, + PTR_ERR(fmod_obj))) + goto out; + + err = bpf_object__load(fmod_obj); + if (CHECK(!err, "fmod_obj_load", "loading fmod_ret should fail\n")) + goto out; + +out: + bpf_link__destroy(freplace_link); + bpf_object__close(freplace_obj); + bpf_object__close(fmod_obj); + bpf_object__close(pkt_obj); +} + + static void test_func_sockmap_update(void) { const char *prog_name[] = { @@ -150,7 +294,7 @@ static void test_func_sockmap_update(void) test_fexit_bpf2bpf_common("./freplace_cls_redirect.o", "./test_cls_redirect.o", ARRAY_SIZE(prog_name), - prog_name, false); + prog_name, false, NULL); } static void test_obj_load_failure_common(const char *obj_file, @@ -222,4 +366,8 @@ void test_fexit_bpf2bpf(void) test_func_replace_return_code(); if (test__start_subtest("func_map_prog_compatibility")) test_func_map_prog_compatibility(); + if (test__start_subtest("func_replace_multi")) + test_func_replace_multi(); + if (test__start_subtest("fmod_ret_freplace")) + test_fmod_ret_freplace(); } diff --git a/tools/testing/selftests/bpf/prog_tests/pe_preserve_elems.c b/tools/testing/selftests/bpf/prog_tests/pe_preserve_elems.c new file mode 100644 index 000000000000..673d38395253 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/pe_preserve_elems.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2019 Facebook */ +#include <test_progs.h> +#include <linux/bpf.h> +#include "test_pe_preserve_elems.skel.h" + +static int duration; + +static void test_one_map(struct bpf_map *map, struct bpf_program *prog, + bool has_share_pe) +{ + int err, key = 0, pfd = -1, mfd = bpf_map__fd(map); + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts); + struct perf_event_attr attr = { + .size = sizeof(struct perf_event_attr), + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_CPU_CLOCK, + }; + + pfd = syscall(__NR_perf_event_open, &attr, 0 /* pid */, + -1 /* cpu 0 */, -1 /* group id */, 0 /* flags */); + if (CHECK(pfd < 0, "perf_event_open", "failed\n")) + return; + + err = bpf_map_update_elem(mfd, &key, &pfd, BPF_ANY); + close(pfd); + if (CHECK(err < 0, "bpf_map_update_elem", "failed\n")) + return; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (CHECK(err < 0, "bpf_prog_test_run_opts", "failed\n")) + return; + if (CHECK(opts.retval != 0, "bpf_perf_event_read_value", + "failed with %d\n", opts.retval)) + return; + + /* closing mfd, prog still holds a reference on map */ + close(mfd); + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (CHECK(err < 0, "bpf_prog_test_run_opts", "failed\n")) + return; + + if (has_share_pe) { + CHECK(opts.retval != 0, "bpf_perf_event_read_value", + "failed with %d\n", opts.retval); + } else { + CHECK(opts.retval != -ENOENT, "bpf_perf_event_read_value", + "should have failed with %d, but got %d\n", -ENOENT, + opts.retval); + } +} + +void test_pe_preserve_elems(void) +{ + struct test_pe_preserve_elems *skel; + + skel = test_pe_preserve_elems__open_and_load(); + if (CHECK(!skel, "skel_open", "failed to open skeleton\n")) + return; + + test_one_map(skel->maps.array_1, skel->progs.read_array_1, false); + test_one_map(skel->maps.array_2, skel->progs.read_array_2, true); + + test_pe_preserve_elems__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_test_run.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_test_run.c new file mode 100644 index 000000000000..c5fb191874ac --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/raw_tp_test_run.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2019 Facebook */ +#include <test_progs.h> +#include <linux/bpf.h> +#include "bpf/libbpf_internal.h" +#include "test_raw_tp_test_run.skel.h" + +static int duration; + +void test_raw_tp_test_run(void) +{ + struct bpf_prog_test_run_attr test_attr = {}; + int comm_fd = -1, err, nr_online, i, prog_fd; + __u64 args[2] = {0x1234ULL, 0x5678ULL}; + int expected_retval = 0x1234 + 0x5678; + struct test_raw_tp_test_run *skel; + char buf[] = "new_name"; + bool *online = NULL; + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", &online, + &nr_online); + if (CHECK(err, "parse_cpu_mask_file", "err %d\n", err)) + return; + + skel = test_raw_tp_test_run__open_and_load(); + if (CHECK(!skel, "skel_open", "failed to open skeleton\n")) + goto cleanup; + + err = test_raw_tp_test_run__attach(skel); + if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err)) + goto cleanup; + + comm_fd = open("/proc/self/comm", O_WRONLY|O_TRUNC); + if (CHECK(comm_fd < 0, "open /proc/self/comm", "err %d\n", errno)) + goto cleanup; + + err = write(comm_fd, buf, sizeof(buf)); + CHECK(err < 0, "task rename", "err %d", errno); + + CHECK(skel->bss->count == 0, "check_count", "didn't increase\n"); + CHECK(skel->data->on_cpu != 0xffffffff, "check_on_cpu", "got wrong value\n"); + + prog_fd = bpf_program__fd(skel->progs.rename); + test_attr.prog_fd = prog_fd; + test_attr.ctx_in = args; + test_attr.ctx_size_in = sizeof(__u64); + + err = bpf_prog_test_run_xattr(&test_attr); + CHECK(err == 0, "test_run", "should fail for too small ctx\n"); + + test_attr.ctx_size_in = sizeof(args); + err = bpf_prog_test_run_xattr(&test_attr); + CHECK(err < 0, "test_run", "err %d\n", errno); + CHECK(test_attr.retval != expected_retval, "check_retval", + "expect 0x%x, got 0x%x\n", expected_retval, test_attr.retval); + + for (i = 0; i < nr_online; i++) { + if (!online[i]) + continue; + + opts.cpu = i; + opts.retval = 0; + err = bpf_prog_test_run_opts(prog_fd, &opts); + CHECK(err < 0, "test_run_opts", "err %d\n", errno); + CHECK(skel->data->on_cpu != i, "check_on_cpu", + "expect %d got %d\n", i, skel->data->on_cpu); + CHECK(opts.retval != expected_retval, + "check_retval", "expect 0x%x, got 0x%x\n", + expected_retval, opts.retval); + } + + /* invalid cpu ID should fail with ENXIO */ + opts.cpu = 0xffffffff; + err = bpf_prog_test_run_opts(prog_fd, &opts); + CHECK(err != -1 || errno != ENXIO, + "test_run_opts_fail", + "should failed with ENXIO\n"); + + /* non-zero cpu w/o BPF_F_TEST_RUN_ON_CPU should fail with EINVAL */ + opts.cpu = 1; + opts.flags = 0; + err = bpf_prog_test_run_opts(prog_fd, &opts); + CHECK(err != -1 || errno != EINVAL, + "test_run_opts_fail", + "should failed with EINVAL\n"); + +cleanup: + close(comm_fd); + test_raw_tp_test_run__destroy(skel); + free(online); +} diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c index 8826c652adad..6ace5e9efec1 100644 --- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c +++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c @@ -28,6 +28,12 @@ struct symbol test_symbols[] = { { "func", BTF_KIND_FUNC, -1 }, }; +/* Align the .BTF_ids section to 4 bytes */ +asm ( +".pushsection " BTF_IDS_SECTION " ,\"a\"; \n" +".balign 4, 0; \n" +".popsection; \n"); + BTF_ID_LIST(test_list_local) BTF_ID_UNUSED BTF_ID(typedef, S) diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c b/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c new file mode 100644 index 000000000000..686b40f11a45 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <linux/btf.h> +#include "netif_receive_skb.skel.h" + +/* Demonstrate that bpf_snprintf_btf succeeds and that various data types + * are formatted correctly. + */ +void test_snprintf_btf(void) +{ + struct netif_receive_skb *skel; + struct netif_receive_skb__bss *bss; + int err, duration = 0; + + skel = netif_receive_skb__open(); + if (CHECK(!skel, "skel_open", "failed to open skeleton\n")) + return; + + err = netif_receive_skb__load(skel); + if (CHECK(err, "skel_load", "failed to load skeleton: %d\n", err)) + goto cleanup; + + bss = skel->bss; + + err = netif_receive_skb__attach(skel); + if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err)) + goto cleanup; + + /* generate receive event */ + err = system("ping -c 1 127.0.0.1 > /dev/null"); + if (CHECK(err, "system", "ping failed: %d\n", err)) + goto cleanup; + + if (bss->skip) { + printf("%s:SKIP:no __builtin_btf_type_id\n", __func__); + test__skip(); + goto cleanup; + } + + /* + * Make sure netif_receive_skb program was triggered + * and it set expected return values from bpf_trace_printk()s + * and all tests ran. + */ + if (CHECK(bss->ret <= 0, + "bpf_snprintf_btf: got return value", + "ret <= 0 %ld test %d\n", bss->ret, bss->ran_subtests)) + goto cleanup; + + if (CHECK(bss->ran_subtests == 0, "check if subtests ran", + "no subtests ran, did BPF program run?")) + goto cleanup; + + if (CHECK(bss->num_subtests != bss->ran_subtests, + "check all subtests ran", + "only ran %d of %d tests\n", bss->num_subtests, + bss->ran_subtests)) + goto cleanup; + +cleanup: + netif_receive_skb__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sock_fields.c b/tools/testing/selftests/bpf/prog_tests/sock_fields.c new file mode 100644 index 000000000000..af87118e748e --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/sock_fields.c @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 Facebook */ + +#include <netinet/in.h> +#include <arpa/inet.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> + +#include <bpf/bpf.h> +#include <bpf/libbpf.h> +#include <linux/compiler.h> + +#include "network_helpers.h" +#include "cgroup_helpers.h" +#include "test_progs.h" +#include "bpf_rlimit.h" +#include "test_sock_fields.skel.h" + +enum bpf_linum_array_idx { + EGRESS_LINUM_IDX, + INGRESS_LINUM_IDX, + __NR_BPF_LINUM_ARRAY_IDX, +}; + +struct bpf_spinlock_cnt { + struct bpf_spin_lock lock; + __u32 cnt; +}; + +#define PARENT_CGROUP "/test-bpf-sock-fields" +#define CHILD_CGROUP "/test-bpf-sock-fields/child" +#define DATA "Hello BPF!" +#define DATA_LEN sizeof(DATA) + +static struct sockaddr_in6 srv_sa6, cli_sa6; +static int sk_pkt_out_cnt10_fd; +static struct test_sock_fields *skel; +static int sk_pkt_out_cnt_fd; +static __u64 parent_cg_id; +static __u64 child_cg_id; +static int linum_map_fd; +static __u32 duration; + +static __u32 egress_linum_idx = EGRESS_LINUM_IDX; +static __u32 ingress_linum_idx = INGRESS_LINUM_IDX; + +static void print_sk(const struct bpf_sock *sk, const char *prefix) +{ + char src_ip4[24], dst_ip4[24]; + char src_ip6[64], dst_ip6[64]; + + inet_ntop(AF_INET, &sk->src_ip4, src_ip4, sizeof(src_ip4)); + inet_ntop(AF_INET6, &sk->src_ip6, src_ip6, sizeof(src_ip6)); + inet_ntop(AF_INET, &sk->dst_ip4, dst_ip4, sizeof(dst_ip4)); + inet_ntop(AF_INET6, &sk->dst_ip6, dst_ip6, sizeof(dst_ip6)); + + printf("%s: state:%u bound_dev_if:%u family:%u type:%u protocol:%u mark:%u priority:%u " + "src_ip4:%x(%s) src_ip6:%x:%x:%x:%x(%s) src_port:%u " + "dst_ip4:%x(%s) dst_ip6:%x:%x:%x:%x(%s) dst_port:%u\n", + prefix, + sk->state, sk->bound_dev_if, sk->family, sk->type, sk->protocol, + sk->mark, sk->priority, + sk->src_ip4, src_ip4, + sk->src_ip6[0], sk->src_ip6[1], sk->src_ip6[2], sk->src_ip6[3], + src_ip6, sk->src_port, + sk->dst_ip4, dst_ip4, + sk->dst_ip6[0], sk->dst_ip6[1], sk->dst_ip6[2], sk->dst_ip6[3], + dst_ip6, ntohs(sk->dst_port)); +} + +static void print_tp(const struct bpf_tcp_sock *tp, const char *prefix) +{ + printf("%s: snd_cwnd:%u srtt_us:%u rtt_min:%u snd_ssthresh:%u rcv_nxt:%u " + "snd_nxt:%u snd:una:%u mss_cache:%u ecn_flags:%u " + "rate_delivered:%u rate_interval_us:%u packets_out:%u " + "retrans_out:%u total_retrans:%u segs_in:%u data_segs_in:%u " + "segs_out:%u data_segs_out:%u lost_out:%u sacked_out:%u " + "bytes_received:%llu bytes_acked:%llu\n", + prefix, + tp->snd_cwnd, tp->srtt_us, tp->rtt_min, tp->snd_ssthresh, + tp->rcv_nxt, tp->snd_nxt, tp->snd_una, tp->mss_cache, + tp->ecn_flags, tp->rate_delivered, tp->rate_interval_us, + tp->packets_out, tp->retrans_out, tp->total_retrans, + tp->segs_in, tp->data_segs_in, tp->segs_out, + tp->data_segs_out, tp->lost_out, tp->sacked_out, + tp->bytes_received, tp->bytes_acked); +} + +static void check_result(void) +{ + struct bpf_tcp_sock srv_tp, cli_tp, listen_tp; + struct bpf_sock srv_sk, cli_sk, listen_sk; + __u32 ingress_linum, egress_linum; + int err; + + err = bpf_map_lookup_elem(linum_map_fd, &egress_linum_idx, + &egress_linum); + CHECK(err == -1, "bpf_map_lookup_elem(linum_map_fd)", + "err:%d errno:%d\n", err, errno); + + err = bpf_map_lookup_elem(linum_map_fd, &ingress_linum_idx, + &ingress_linum); + CHECK(err == -1, "bpf_map_lookup_elem(linum_map_fd)", + "err:%d errno:%d\n", err, errno); + + memcpy(&srv_sk, &skel->bss->srv_sk, sizeof(srv_sk)); + memcpy(&srv_tp, &skel->bss->srv_tp, sizeof(srv_tp)); + memcpy(&cli_sk, &skel->bss->cli_sk, sizeof(cli_sk)); + memcpy(&cli_tp, &skel->bss->cli_tp, sizeof(cli_tp)); + memcpy(&listen_sk, &skel->bss->listen_sk, sizeof(listen_sk)); + memcpy(&listen_tp, &skel->bss->listen_tp, sizeof(listen_tp)); + + print_sk(&listen_sk, "listen_sk"); + print_sk(&srv_sk, "srv_sk"); + print_sk(&cli_sk, "cli_sk"); + print_tp(&listen_tp, "listen_tp"); + print_tp(&srv_tp, "srv_tp"); + print_tp(&cli_tp, "cli_tp"); + + CHECK(listen_sk.state != 10 || + listen_sk.family != AF_INET6 || + listen_sk.protocol != IPPROTO_TCP || + memcmp(listen_sk.src_ip6, &in6addr_loopback, + sizeof(listen_sk.src_ip6)) || + listen_sk.dst_ip6[0] || listen_sk.dst_ip6[1] || + listen_sk.dst_ip6[2] || listen_sk.dst_ip6[3] || + listen_sk.src_port != ntohs(srv_sa6.sin6_port) || + listen_sk.dst_port, + "listen_sk", + "Unexpected. Check listen_sk output. ingress_linum:%u\n", + ingress_linum); + + CHECK(srv_sk.state == 10 || + !srv_sk.state || + srv_sk.family != AF_INET6 || + srv_sk.protocol != IPPROTO_TCP || + memcmp(srv_sk.src_ip6, &in6addr_loopback, + sizeof(srv_sk.src_ip6)) || + memcmp(srv_sk.dst_ip6, &in6addr_loopback, + sizeof(srv_sk.dst_ip6)) || + srv_sk.src_port != ntohs(srv_sa6.sin6_port) || + srv_sk.dst_port != cli_sa6.sin6_port, + "srv_sk", "Unexpected. Check srv_sk output. egress_linum:%u\n", + egress_linum); + + CHECK(!skel->bss->lsndtime, "srv_tp", "Unexpected lsndtime:0\n"); + + CHECK(cli_sk.state == 10 || + !cli_sk.state || + cli_sk.family != AF_INET6 || + cli_sk.protocol != IPPROTO_TCP || + memcmp(cli_sk.src_ip6, &in6addr_loopback, + sizeof(cli_sk.src_ip6)) || + memcmp(cli_sk.dst_ip6, &in6addr_loopback, + sizeof(cli_sk.dst_ip6)) || + cli_sk.src_port != ntohs(cli_sa6.sin6_port) || + cli_sk.dst_port != srv_sa6.sin6_port, + "cli_sk", "Unexpected. Check cli_sk output. egress_linum:%u\n", + egress_linum); + + CHECK(listen_tp.data_segs_out || + listen_tp.data_segs_in || + listen_tp.total_retrans || + listen_tp.bytes_acked, + "listen_tp", + "Unexpected. Check listen_tp output. ingress_linum:%u\n", + ingress_linum); + + CHECK(srv_tp.data_segs_out != 2 || + srv_tp.data_segs_in || + srv_tp.snd_cwnd != 10 || + srv_tp.total_retrans || + srv_tp.bytes_acked < 2 * DATA_LEN, + "srv_tp", "Unexpected. Check srv_tp output. egress_linum:%u\n", + egress_linum); + + CHECK(cli_tp.data_segs_out || + cli_tp.data_segs_in != 2 || + cli_tp.snd_cwnd != 10 || + cli_tp.total_retrans || + cli_tp.bytes_received < 2 * DATA_LEN, + "cli_tp", "Unexpected. Check cli_tp output. egress_linum:%u\n", + egress_linum); + + CHECK(skel->bss->parent_cg_id != parent_cg_id, + "parent_cg_id", "%zu != %zu\n", + (size_t)skel->bss->parent_cg_id, (size_t)parent_cg_id); + + CHECK(skel->bss->child_cg_id != child_cg_id, + "child_cg_id", "%zu != %zu\n", + (size_t)skel->bss->child_cg_id, (size_t)child_cg_id); +} + +static void check_sk_pkt_out_cnt(int accept_fd, int cli_fd) +{ + struct bpf_spinlock_cnt pkt_out_cnt = {}, pkt_out_cnt10 = {}; + int err; + + pkt_out_cnt.cnt = ~0; + pkt_out_cnt10.cnt = ~0; + err = bpf_map_lookup_elem(sk_pkt_out_cnt_fd, &accept_fd, &pkt_out_cnt); + if (!err) + err = bpf_map_lookup_elem(sk_pkt_out_cnt10_fd, &accept_fd, + &pkt_out_cnt10); + + /* The bpf prog only counts for fullsock and + * passive connection did not become fullsock until 3WHS + * had been finished, so the bpf prog only counted two data + * packet out. + */ + CHECK(err || pkt_out_cnt.cnt < 0xeB9F + 2 || + pkt_out_cnt10.cnt < 0xeB9F + 20, + "bpf_map_lookup_elem(sk_pkt_out_cnt, &accept_fd)", + "err:%d errno:%d pkt_out_cnt:%u pkt_out_cnt10:%u\n", + err, errno, pkt_out_cnt.cnt, pkt_out_cnt10.cnt); + + pkt_out_cnt.cnt = ~0; + pkt_out_cnt10.cnt = ~0; + err = bpf_map_lookup_elem(sk_pkt_out_cnt_fd, &cli_fd, &pkt_out_cnt); + if (!err) + err = bpf_map_lookup_elem(sk_pkt_out_cnt10_fd, &cli_fd, + &pkt_out_cnt10); + /* Active connection is fullsock from the beginning. + * 1 SYN and 1 ACK during 3WHS + * 2 Acks on data packet. + * + * The bpf_prog initialized it to 0xeB9F. + */ + CHECK(err || pkt_out_cnt.cnt < 0xeB9F + 4 || + pkt_out_cnt10.cnt < 0xeB9F + 40, + "bpf_map_lookup_elem(sk_pkt_out_cnt, &cli_fd)", + "err:%d errno:%d pkt_out_cnt:%u pkt_out_cnt10:%u\n", + err, errno, pkt_out_cnt.cnt, pkt_out_cnt10.cnt); +} + +static int init_sk_storage(int sk_fd, __u32 pkt_out_cnt) +{ + struct bpf_spinlock_cnt scnt = {}; + int err; + + scnt.cnt = pkt_out_cnt; + err = bpf_map_update_elem(sk_pkt_out_cnt_fd, &sk_fd, &scnt, + BPF_NOEXIST); + if (CHECK(err, "bpf_map_update_elem(sk_pkt_out_cnt_fd)", + "err:%d errno:%d\n", err, errno)) + return err; + + err = bpf_map_update_elem(sk_pkt_out_cnt10_fd, &sk_fd, &scnt, + BPF_NOEXIST); + if (CHECK(err, "bpf_map_update_elem(sk_pkt_out_cnt10_fd)", + "err:%d errno:%d\n", err, errno)) + return err; + + return 0; +} + +static void test(void) +{ + int listen_fd = -1, cli_fd = -1, accept_fd = -1, err, i; + socklen_t addrlen = sizeof(struct sockaddr_in6); + char buf[DATA_LEN]; + + /* Prepare listen_fd */ + listen_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0); + /* start_server() has logged the error details */ + if (CHECK_FAIL(listen_fd == -1)) + goto done; + + err = getsockname(listen_fd, (struct sockaddr *)&srv_sa6, &addrlen); + if (CHECK(err, "getsockname(listen_fd)", "err:%d errno:%d\n", err, + errno)) + goto done; + memcpy(&skel->bss->srv_sa6, &srv_sa6, sizeof(srv_sa6)); + + cli_fd = connect_to_fd(listen_fd, 0); + if (CHECK_FAIL(cli_fd == -1)) + goto done; + + err = getsockname(cli_fd, (struct sockaddr *)&cli_sa6, &addrlen); + if (CHECK(err, "getsockname(cli_fd)", "err:%d errno:%d\n", + err, errno)) + goto done; + + accept_fd = accept(listen_fd, NULL, NULL); + if (CHECK(accept_fd == -1, "accept(listen_fd)", + "accept_fd:%d errno:%d\n", + accept_fd, errno)) + goto done; + + if (init_sk_storage(accept_fd, 0xeB9F)) + goto done; + + for (i = 0; i < 2; i++) { + /* Send some data from accept_fd to cli_fd. + * MSG_EOR to stop kernel from coalescing two pkts. + */ + err = send(accept_fd, DATA, DATA_LEN, MSG_EOR); + if (CHECK(err != DATA_LEN, "send(accept_fd)", + "err:%d errno:%d\n", err, errno)) + goto done; + + err = recv(cli_fd, buf, DATA_LEN, 0); + if (CHECK(err != DATA_LEN, "recv(cli_fd)", "err:%d errno:%d\n", + err, errno)) + goto done; + } + + shutdown(cli_fd, SHUT_WR); + err = recv(accept_fd, buf, 1, 0); + if (CHECK(err, "recv(accept_fd) for fin", "err:%d errno:%d\n", + err, errno)) + goto done; + shutdown(accept_fd, SHUT_WR); + err = recv(cli_fd, buf, 1, 0); + if (CHECK(err, "recv(cli_fd) for fin", "err:%d errno:%d\n", + err, errno)) + goto done; + check_sk_pkt_out_cnt(accept_fd, cli_fd); + check_result(); + +done: + if (accept_fd != -1) + close(accept_fd); + if (cli_fd != -1) + close(cli_fd); + if (listen_fd != -1) + close(listen_fd); +} + +void test_sock_fields(void) +{ + struct bpf_link *egress_link = NULL, *ingress_link = NULL; + int parent_cg_fd = -1, child_cg_fd = -1; + + /* Create a cgroup, get fd, and join it */ + parent_cg_fd = test__join_cgroup(PARENT_CGROUP); + if (CHECK_FAIL(parent_cg_fd < 0)) + return; + parent_cg_id = get_cgroup_id(PARENT_CGROUP); + if (CHECK_FAIL(!parent_cg_id)) + goto done; + + child_cg_fd = test__join_cgroup(CHILD_CGROUP); + if (CHECK_FAIL(child_cg_fd < 0)) + goto done; + child_cg_id = get_cgroup_id(CHILD_CGROUP); + if (CHECK_FAIL(!child_cg_id)) + goto done; + + skel = test_sock_fields__open_and_load(); + if (CHECK(!skel, "test_sock_fields__open_and_load", "failed\n")) + goto done; + + egress_link = bpf_program__attach_cgroup(skel->progs.egress_read_sock_fields, + child_cg_fd); + if (CHECK(IS_ERR(egress_link), "attach_cgroup(egress)", "err:%ld\n", + PTR_ERR(egress_link))) + goto done; + + ingress_link = bpf_program__attach_cgroup(skel->progs.ingress_read_sock_fields, + child_cg_fd); + if (CHECK(IS_ERR(ingress_link), "attach_cgroup(ingress)", "err:%ld\n", + PTR_ERR(ingress_link))) + goto done; + + linum_map_fd = bpf_map__fd(skel->maps.linum_map); + sk_pkt_out_cnt_fd = bpf_map__fd(skel->maps.sk_pkt_out_cnt); + sk_pkt_out_cnt10_fd = bpf_map__fd(skel->maps.sk_pkt_out_cnt10); + + test(); + +done: + bpf_link__destroy(egress_link); + bpf_link__destroy(ingress_link); + test_sock_fields__destroy(skel); + if (child_cg_fd != -1) + close(child_cg_fd); + if (parent_cg_fd != -1) + close(parent_cg_fd); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index 4b7a527e7e82..4c4224e3e10a 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -8,8 +8,6 @@ #include "test_sockmap_invalid_update.skel.h" #include "bpf_iter_sockmap.skel.h" -#include "progs/bpf_iter_sockmap.h" - #define TCP_REPAIR 19 /* TCP sock is under repair right now */ #define TCP_REPAIR_ON 1 @@ -50,6 +48,37 @@ error: return -1; } +static void compare_cookies(struct bpf_map *src, struct bpf_map *dst) +{ + __u32 i, max_entries = bpf_map__max_entries(src); + int err, duration = 0, src_fd, dst_fd; + + src_fd = bpf_map__fd(src); + dst_fd = bpf_map__fd(dst); + + for (i = 0; i < max_entries; i++) { + __u64 src_cookie, dst_cookie; + + err = bpf_map_lookup_elem(src_fd, &i, &src_cookie); + if (err && errno == ENOENT) { + err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); + CHECK(!err, "map_lookup_elem(dst)", "element %u not deleted\n", i); + CHECK(err && errno != ENOENT, "map_lookup_elem(dst)", "%s\n", + strerror(errno)); + continue; + } + if (CHECK(err, "lookup_elem(src)", "%s\n", strerror(errno))) + continue; + + err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); + if (CHECK(err, "lookup_elem(dst)", "%s\n", strerror(errno))) + continue; + + CHECK(dst_cookie != src_cookie, "cookie mismatch", + "%llu != %llu (pos %u)\n", dst_cookie, src_cookie, i); + } +} + /* Create a map, populate it with one socket, and free the map. */ static void test_sockmap_create_update_free(enum bpf_map_type map_type) { @@ -109,9 +138,9 @@ out: static void test_sockmap_update(enum bpf_map_type map_type) { struct bpf_prog_test_run_attr tattr; - int err, prog, src, dst, duration = 0; + int err, prog, src, duration = 0; struct test_sockmap_update *skel; - __u64 src_cookie, dst_cookie; + struct bpf_map *dst_map; const __u32 zero = 0; char dummy[14] = {0}; __s64 sk; @@ -127,18 +156,14 @@ static void test_sockmap_update(enum bpf_map_type map_type) prog = bpf_program__fd(skel->progs.copy_sock_map); src = bpf_map__fd(skel->maps.src); if (map_type == BPF_MAP_TYPE_SOCKMAP) - dst = bpf_map__fd(skel->maps.dst_sock_map); + dst_map = skel->maps.dst_sock_map; else - dst = bpf_map__fd(skel->maps.dst_sock_hash); + dst_map = skel->maps.dst_sock_hash; err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST); if (CHECK(err, "update_elem(src)", "errno=%u\n", errno)) goto out; - err = bpf_map_lookup_elem(src, &zero, &src_cookie); - if (CHECK(err, "lookup_elem(src, cookie)", "errno=%u\n", errno)) - goto out; - tattr = (struct bpf_prog_test_run_attr){ .prog_fd = prog, .repeat = 1, @@ -151,12 +176,7 @@ static void test_sockmap_update(enum bpf_map_type map_type) "errno=%u retval=%u\n", errno, tattr.retval)) goto out; - err = bpf_map_lookup_elem(dst, &zero, &dst_cookie); - if (CHECK(err, "lookup_elem(dst, cookie)", "errno=%u\n", errno)) - goto out; - - CHECK(dst_cookie != src_cookie, "cookie mismatch", "%llu != %llu\n", - dst_cookie, src_cookie); + compare_cookies(skel->maps.src, dst_map); out: test_sockmap_update__destroy(skel); @@ -174,14 +194,14 @@ static void test_sockmap_invalid_update(void) test_sockmap_invalid_update__destroy(skel); } -static void test_sockmap_iter(enum bpf_map_type map_type) +static void test_sockmap_copy(enum bpf_map_type map_type) { DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); int err, len, src_fd, iter_fd, duration = 0; union bpf_iter_link_info linfo = {0}; - __s64 sock_fd[SOCKMAP_MAX_ENTRIES]; - __u32 i, num_sockets, max_elems; + __u32 i, num_sockets, num_elems; struct bpf_iter_sockmap *skel; + __s64 *sock_fd = NULL; struct bpf_link *link; struct bpf_map *src; char buf[64]; @@ -190,22 +210,23 @@ static void test_sockmap_iter(enum bpf_map_type map_type) if (CHECK(!skel, "bpf_iter_sockmap__open_and_load", "skeleton open_and_load failed\n")) return; - for (i = 0; i < ARRAY_SIZE(sock_fd); i++) - sock_fd[i] = -1; - - /* Make sure we have at least one "empty" entry to test iteration of - * an empty slot. - */ - num_sockets = ARRAY_SIZE(sock_fd) - 1; - if (map_type == BPF_MAP_TYPE_SOCKMAP) { src = skel->maps.sockmap; - max_elems = bpf_map__max_entries(src); + num_elems = bpf_map__max_entries(src); + num_sockets = num_elems - 1; } else { src = skel->maps.sockhash; - max_elems = num_sockets; + num_elems = bpf_map__max_entries(src) - 1; + num_sockets = num_elems; } + sock_fd = calloc(num_sockets, sizeof(*sock_fd)); + if (CHECK(!sock_fd, "calloc(sock_fd)", "failed to allocate\n")) + goto out; + + for (i = 0; i < num_sockets; i++) + sock_fd[i] = -1; + src_fd = bpf_map__fd(src); for (i = 0; i < num_sockets; i++) { @@ -221,7 +242,7 @@ static void test_sockmap_iter(enum bpf_map_type map_type) linfo.map.map_fd = src_fd; opts.link_info = &linfo; opts.link_info_len = sizeof(linfo); - link = bpf_program__attach_iter(skel->progs.count_elems, &opts); + link = bpf_program__attach_iter(skel->progs.copy, &opts); if (CHECK(IS_ERR(link), "attach_iter", "attach_iter failed\n")) goto out; @@ -236,23 +257,26 @@ static void test_sockmap_iter(enum bpf_map_type map_type) goto close_iter; /* test results */ - if (CHECK(skel->bss->elems != max_elems, "elems", "got %u expected %u\n", - skel->bss->elems, max_elems)) + if (CHECK(skel->bss->elems != num_elems, "elems", "got %u expected %u\n", + skel->bss->elems, num_elems)) goto close_iter; if (CHECK(skel->bss->socks != num_sockets, "socks", "got %u expected %u\n", skel->bss->socks, num_sockets)) goto close_iter; + compare_cookies(src, skel->maps.dst); + close_iter: close(iter_fd); free_link: bpf_link__destroy(link); out: - for (i = 0; i < num_sockets; i++) { + for (i = 0; sock_fd && i < num_sockets; i++) if (sock_fd[i] >= 0) close(sock_fd[i]); - } + if (sock_fd) + free(sock_fd); bpf_iter_sockmap__destroy(skel); } @@ -272,8 +296,8 @@ void test_sockmap_basic(void) test_sockmap_update(BPF_MAP_TYPE_SOCKHASH); if (test__start_subtest("sockmap update in unsafe context")) test_sockmap_invalid_update(); - if (test__start_subtest("sockmap iter")) - test_sockmap_iter(BPF_MAP_TYPE_SOCKMAP); - if (test__start_subtest("sockhash iter")) - test_sockmap_iter(BPF_MAP_TYPE_SOCKHASH); + if (test__start_subtest("sockmap copy")) + test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP); + if (test__start_subtest("sockhash copy")) + test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_overhead.c b/tools/testing/selftests/bpf/prog_tests/test_overhead.c index 2702df2b2343..9966685866fd 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_overhead.c +++ b/tools/testing/selftests/bpf/prog_tests/test_overhead.c @@ -61,10 +61,9 @@ void test_test_overhead(void) const char *raw_tp_name = "raw_tp/task_rename"; const char *fentry_name = "fentry/__set_task_comm"; const char *fexit_name = "fexit/__set_task_comm"; - const char *fmodret_name = "fmod_ret/__set_task_comm"; const char *kprobe_func = "__set_task_comm"; struct bpf_program *kprobe_prog, *kretprobe_prog, *raw_tp_prog; - struct bpf_program *fentry_prog, *fexit_prog, *fmodret_prog; + struct bpf_program *fentry_prog, *fexit_prog; struct bpf_object *obj; struct bpf_link *link; int err, duration = 0; @@ -97,11 +96,6 @@ void test_test_overhead(void) if (CHECK(!fexit_prog, "find_probe", "prog '%s' not found\n", fexit_name)) goto cleanup; - fmodret_prog = bpf_object__find_program_by_title(obj, fmodret_name); - if (CHECK(!fmodret_prog, "find_probe", - "prog '%s' not found\n", fmodret_name)) - goto cleanup; - err = bpf_object__load(obj); if (CHECK(err, "obj_load", "err %d\n", err)) goto cleanup; @@ -148,12 +142,6 @@ void test_test_overhead(void) test_run("fexit"); bpf_link__destroy(link); - /* attach fmod_ret */ - link = bpf_program__attach_trace(fmodret_prog); - if (CHECK(IS_ERR(link), "attach fmod_ret", "err %ld\n", PTR_ERR(link))) - goto cleanup; - test_run("fmod_ret"); - bpf_link__destroy(link); cleanup: prctl(PR_SET_NAME, comm, 0L, 0L, 0L); bpf_object__close(obj); diff --git a/tools/testing/selftests/bpf/prog_tests/trace_ext.c b/tools/testing/selftests/bpf/prog_tests/trace_ext.c new file mode 100644 index 000000000000..924441d4362d --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/trace_ext.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include <test_progs.h> +#include <network_helpers.h> +#include <sys/stat.h> +#include <linux/sched.h> +#include <sys/syscall.h> + +#include "test_pkt_md_access.skel.h" +#include "test_trace_ext.skel.h" +#include "test_trace_ext_tracing.skel.h" + +static __u32 duration; + +void test_trace_ext(void) +{ + struct test_pkt_md_access *skel_pkt = NULL; + struct test_trace_ext_tracing *skel_trace = NULL; + struct test_trace_ext_tracing__bss *bss_trace; + struct test_trace_ext *skel_ext = NULL; + struct test_trace_ext__bss *bss_ext; + int err, pkt_fd, ext_fd; + struct bpf_program *prog; + char buf[100]; + __u32 retval; + __u64 len; + + /* open/load/attach test_pkt_md_access */ + skel_pkt = test_pkt_md_access__open_and_load(); + if (CHECK(!skel_pkt, "setup", "classifier/test_pkt_md_access open failed\n")) + goto cleanup; + + err = test_pkt_md_access__attach(skel_pkt); + if (CHECK(err, "setup", "classifier/test_pkt_md_access attach failed: %d\n", err)) + goto cleanup; + + prog = skel_pkt->progs.test_pkt_md_access; + pkt_fd = bpf_program__fd(prog); + + /* open extension */ + skel_ext = test_trace_ext__open(); + if (CHECK(!skel_ext, "setup", "freplace/test_pkt_md_access open failed\n")) + goto cleanup; + + /* set extension's attach target - test_pkt_md_access */ + prog = skel_ext->progs.test_pkt_md_access_new; + bpf_program__set_attach_target(prog, pkt_fd, "test_pkt_md_access"); + + /* load/attach extension */ + err = test_trace_ext__load(skel_ext); + if (CHECK(err, "setup", "freplace/test_pkt_md_access load failed\n")) { + libbpf_strerror(err, buf, sizeof(buf)); + fprintf(stderr, "%s\n", buf); + goto cleanup; + } + + err = test_trace_ext__attach(skel_ext); + if (CHECK(err, "setup", "freplace/test_pkt_md_access attach failed: %d\n", err)) + goto cleanup; + + prog = skel_ext->progs.test_pkt_md_access_new; + ext_fd = bpf_program__fd(prog); + + /* open tracing */ + skel_trace = test_trace_ext_tracing__open(); + if (CHECK(!skel_trace, "setup", "tracing/test_pkt_md_access_new open failed\n")) + goto cleanup; + + /* set tracing's attach target - fentry */ + prog = skel_trace->progs.fentry; + bpf_program__set_attach_target(prog, ext_fd, "test_pkt_md_access_new"); + + /* set tracing's attach target - fexit */ + prog = skel_trace->progs.fexit; + bpf_program__set_attach_target(prog, ext_fd, "test_pkt_md_access_new"); + + /* load/attach tracing */ + err = test_trace_ext_tracing__load(skel_trace); + if (CHECK(err, "setup", "tracing/test_pkt_md_access_new load failed\n")) { + libbpf_strerror(err, buf, sizeof(buf)); + fprintf(stderr, "%s\n", buf); + goto cleanup; + } + + err = test_trace_ext_tracing__attach(skel_trace); + if (CHECK(err, "setup", "tracing/test_pkt_md_access_new attach failed: %d\n", err)) + goto cleanup; + + /* trigger the test */ + err = bpf_prog_test_run(pkt_fd, 1, &pkt_v4, sizeof(pkt_v4), + NULL, NULL, &retval, &duration); + CHECK(err || retval, "run", "err %d errno %d retval %d\n", err, errno, retval); + + bss_ext = skel_ext->bss; + bss_trace = skel_trace->bss; + + len = bss_ext->ext_called; + + CHECK(bss_ext->ext_called == 0, + "check", "failed to trigger freplace/test_pkt_md_access\n"); + CHECK(bss_trace->fentry_called != len, + "check", "failed to trigger fentry/test_pkt_md_access_new\n"); + CHECK(bss_trace->fexit_called != len, + "check", "failed to trigger fexit/test_pkt_md_access_new\n"); + +cleanup: + test_trace_ext_tracing__destroy(skel_trace); + test_trace_ext__destroy(skel_ext); + test_pkt_md_access__destroy(skel_pkt); +} diff --git a/tools/testing/selftests/bpf/progs/bpf_cubic.c b/tools/testing/selftests/bpf/progs/bpf_cubic.c index ef574087f1e1..6939bfd8690f 100644 --- a/tools/testing/selftests/bpf/progs/bpf_cubic.c +++ b/tools/testing/selftests/bpf/progs/bpf_cubic.c @@ -15,6 +15,8 @@ */ #include <linux/bpf.h> +#include <linux/stddef.h> +#include <linux/tcp.h> #include "bpf_tcp_helpers.h" char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/bpf_dctcp.c b/tools/testing/selftests/bpf/progs/bpf_dctcp.c index 3fb4260570b1..4dc1a967776a 100644 --- a/tools/testing/selftests/bpf/progs/bpf_dctcp.c +++ b/tools/testing/selftests/bpf/progs/bpf_dctcp.c @@ -9,6 +9,8 @@ #include <stddef.h> #include <linux/bpf.h> #include <linux/types.h> +#include <linux/stddef.h> +#include <linux/tcp.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include "bpf_tcp_helpers.h" diff --git a/tools/testing/selftests/bpf/progs/bpf_flow.c b/tools/testing/selftests/bpf/progs/bpf_flow.c index de6de9221518..5a65f6b51377 100644 --- a/tools/testing/selftests/bpf/progs/bpf_flow.c +++ b/tools/testing/selftests/bpf/progs/bpf_flow.c @@ -118,18 +118,18 @@ static __always_inline int parse_eth_proto(struct __sk_buff *skb, __be16 proto) switch (proto) { case bpf_htons(ETH_P_IP): - bpf_tail_call(skb, &jmp_table, IP); + bpf_tail_call_static(skb, &jmp_table, IP); break; case bpf_htons(ETH_P_IPV6): - bpf_tail_call(skb, &jmp_table, IPV6); + bpf_tail_call_static(skb, &jmp_table, IPV6); break; case bpf_htons(ETH_P_MPLS_MC): case bpf_htons(ETH_P_MPLS_UC): - bpf_tail_call(skb, &jmp_table, MPLS); + bpf_tail_call_static(skb, &jmp_table, MPLS); break; case bpf_htons(ETH_P_8021Q): case bpf_htons(ETH_P_8021AD): - bpf_tail_call(skb, &jmp_table, VLAN); + bpf_tail_call_static(skb, &jmp_table, VLAN); break; default: /* Protocol not supported */ @@ -246,10 +246,10 @@ static __always_inline int parse_ipv6_proto(struct __sk_buff *skb, __u8 nexthdr) switch (nexthdr) { case IPPROTO_HOPOPTS: case IPPROTO_DSTOPTS: - bpf_tail_call(skb, &jmp_table, IPV6OP); + bpf_tail_call_static(skb, &jmp_table, IPV6OP); break; case IPPROTO_FRAGMENT: - bpf_tail_call(skb, &jmp_table, IPV6FR); + bpf_tail_call_static(skb, &jmp_table, IPV6FR); break; default: return parse_ip_proto(skb, nexthdr); diff --git a/tools/testing/selftests/bpf/progs/bpf_iter.h b/tools/testing/selftests/bpf/progs/bpf_iter.h index df682af75510..6a1255465fd6 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter.h +++ b/tools/testing/selftests/bpf/progs/bpf_iter.h @@ -14,6 +14,11 @@ #define bpf_iter__bpf_map_elem bpf_iter__bpf_map_elem___not_used #define bpf_iter__bpf_sk_storage_map bpf_iter__bpf_sk_storage_map___not_used #define bpf_iter__sockmap bpf_iter__sockmap___not_used +#define btf_ptr btf_ptr___not_used +#define BTF_F_COMPACT BTF_F_COMPACT___not_used +#define BTF_F_NONAME BTF_F_NONAME___not_used +#define BTF_F_PTR_RAW BTF_F_PTR_RAW___not_used +#define BTF_F_ZERO BTF_F_ZERO___not_used #include "vmlinux.h" #undef bpf_iter_meta #undef bpf_iter__bpf_map @@ -28,6 +33,11 @@ #undef bpf_iter__bpf_map_elem #undef bpf_iter__bpf_sk_storage_map #undef bpf_iter__sockmap +#undef btf_ptr +#undef BTF_F_COMPACT +#undef BTF_F_NONAME +#undef BTF_F_PTR_RAW +#undef BTF_F_ZERO struct bpf_iter_meta { struct seq_file *seq; @@ -105,3 +115,16 @@ struct bpf_iter__sockmap { void *key; struct sock *sk; }; + +struct btf_ptr { + void *ptr; + __u32 type_id; + __u32 flags; +}; + +enum { + BTF_F_COMPACT = (1ULL << 0), + BTF_F_NONAME = (1ULL << 1), + BTF_F_PTR_RAW = (1ULL << 2), + BTF_F_ZERO = (1ULL << 3), +}; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c b/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c index 0e27f73dd803..f3af0e30cead 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c @@ -2,7 +2,6 @@ /* Copyright (c) 2020 Cloudflare */ #include "bpf_iter.h" #include "bpf_tracing_net.h" -#include "bpf_iter_sockmap.h" #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include <errno.h> @@ -11,33 +10,50 @@ char _license[] SEC("license") = "GPL"; struct { __uint(type, BPF_MAP_TYPE_SOCKMAP); - __uint(max_entries, SOCKMAP_MAX_ENTRIES); + __uint(max_entries, 64); __type(key, __u32); __type(value, __u64); } sockmap SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_SOCKHASH); - __uint(max_entries, SOCKMAP_MAX_ENTRIES); + __uint(max_entries, 64); __type(key, __u32); __type(value, __u64); } sockhash SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_SOCKHASH); + __uint(max_entries, 64); + __type(key, __u32); + __type(value, __u64); +} dst SEC(".maps"); + __u32 elems = 0; __u32 socks = 0; SEC("iter/sockmap") -int count_elems(struct bpf_iter__sockmap *ctx) +int copy(struct bpf_iter__sockmap *ctx) { struct sock *sk = ctx->sk; __u32 tmp, *key = ctx->key; int ret; - if (key) - elems++; + if (!key) + return 0; + + elems++; + + /* We need a temporary buffer on the stack, since the verifier doesn't + * let us use the pointer from the context as an argument to the helper. + */ + tmp = *key; - if (sk) + if (sk) { socks++; + return bpf_map_update_elem(&dst, &tmp, sk, 0) != 0; + } - return 0; + ret = bpf_map_delete_elem(&dst, &tmp); + return ret && ret != -ENOENT; } diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.h b/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.h deleted file mode 100644 index 35a675d13c0f..000000000000 --- a/tools/testing/selftests/bpf/progs/bpf_iter_sockmap.h +++ /dev/null @@ -1,3 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#define SOCKMAP_MAX_ENTRIES (64) diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_task_btf.c b/tools/testing/selftests/bpf/progs/bpf_iter_task_btf.c new file mode 100644 index 000000000000..a1ddc36f13ec --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_iter_task_btf.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020, Oracle and/or its affiliates. */ +#include "bpf_iter.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> + +#include <errno.h> + +char _license[] SEC("license") = "GPL"; + +long tasks = 0; +long seq_err = 0; +bool skip = false; + +SEC("iter/task") +int dump_task_struct(struct bpf_iter__task *ctx) +{ + struct seq_file *seq = ctx->meta->seq; + struct task_struct *task = ctx->task; + static struct btf_ptr ptr = { }; + long ret; + +#if __has_builtin(__builtin_btf_type_id) + ptr.type_id = bpf_core_type_id_kernel(struct task_struct); + ptr.ptr = task; + + if (ctx->meta->seq_num == 0) + BPF_SEQ_PRINTF(seq, "Raw BTF task\n"); + + ret = bpf_seq_printf_btf(seq, &ptr, sizeof(ptr), 0); + switch (ret) { + case 0: + tasks++; + break; + case -ERANGE: + /* NULL task or task->fs, don't count it as an error. */ + break; + case -E2BIG: + return 1; + default: + seq_err = ret; + break; + } +#else + skip = true; +#endif + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/btf_ptr.h b/tools/testing/selftests/bpf/progs/btf_ptr.h new file mode 100644 index 000000000000..c3c9797c67db --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf_ptr.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2020, Oracle and/or its affiliates. */ +/* "undefine" structs in vmlinux.h, because we "override" them below */ +#define btf_ptr btf_ptr___not_used +#define BTF_F_COMPACT BTF_F_COMPACT___not_used +#define BTF_F_NONAME BTF_F_NONAME___not_used +#define BTF_F_PTR_RAW BTF_F_PTR_RAW___not_used +#define BTF_F_ZERO BTF_F_ZERO___not_used +#include "vmlinux.h" +#undef btf_ptr +#undef BTF_F_COMPACT +#undef BTF_F_NONAME +#undef BTF_F_PTR_RAW +#undef BTF_F_ZERO + +struct btf_ptr { + void *ptr; + __u32 type_id; + __u32 flags; +}; + +enum { + BTF_F_COMPACT = (1ULL << 0), + BTF_F_NONAME = (1ULL << 1), + BTF_F_PTR_RAW = (1ULL << 2), + BTF_F_ZERO = (1ULL << 3), +}; diff --git a/tools/testing/selftests/bpf/progs/fmod_ret_freplace.c b/tools/testing/selftests/bpf/progs/fmod_ret_freplace.c new file mode 100644 index 000000000000..c8943ccee6c0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/fmod_ret_freplace.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +volatile __u64 test_fmod_ret = 0; +SEC("fmod_ret/security_new_get_constant") +int BPF_PROG(fmod_ret_test, long val, int ret) +{ + test_fmod_ret = 1; + return 120; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/freplace_get_constant.c b/tools/testing/selftests/bpf/progs/freplace_get_constant.c new file mode 100644 index 000000000000..705e4b64dfc2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_get_constant.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> + +volatile __u64 test_get_constant = 0; +SEC("freplace/get_constant") +int security_new_get_constant(long val) +{ + if (val != 123) + return 0; + test_get_constant = 1; + return test_get_constant; /* original get_constant() returns val - 122 */ +} +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/netif_receive_skb.c b/tools/testing/selftests/bpf/progs/netif_receive_skb.c new file mode 100644 index 000000000000..6b670039ea67 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/netif_receive_skb.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020, Oracle and/or its affiliates. */ + +#include "btf_ptr.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> + +#include <errno.h> + +long ret = 0; +int num_subtests = 0; +int ran_subtests = 0; +bool skip = false; + +#define STRSIZE 2048 +#define EXPECTED_STRSIZE 256 + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#endif + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, char[STRSIZE]); +} strdata SEC(".maps"); + +static int __strncmp(const void *m1, const void *m2, size_t len) +{ + const unsigned char *s1 = m1; + const unsigned char *s2 = m2; + int i, delta = 0; + + for (i = 0; i < len; i++) { + delta = s1[i] - s2[i]; + if (delta || s1[i] == 0 || s2[i] == 0) + break; + } + return delta; +} + +#if __has_builtin(__builtin_btf_type_id) +#define TEST_BTF(_str, _type, _flags, _expected, ...) \ + do { \ + static const char _expectedval[EXPECTED_STRSIZE] = \ + _expected; \ + static const char _ptrtype[64] = #_type; \ + __u64 _hflags = _flags | BTF_F_COMPACT; \ + static _type _ptrdata = __VA_ARGS__; \ + static struct btf_ptr _ptr = { }; \ + int _cmp; \ + \ + ++num_subtests; \ + if (ret < 0) \ + break; \ + ++ran_subtests; \ + _ptr.ptr = &_ptrdata; \ + _ptr.type_id = bpf_core_type_id_kernel(_type); \ + if (_ptr.type_id <= 0) { \ + ret = -EINVAL; \ + break; \ + } \ + ret = bpf_snprintf_btf(_str, STRSIZE, \ + &_ptr, sizeof(_ptr), _hflags); \ + if (ret) \ + break; \ + _cmp = __strncmp(_str, _expectedval, EXPECTED_STRSIZE); \ + if (_cmp != 0) { \ + bpf_printk("(%d) got %s", _cmp, _str); \ + bpf_printk("(%d) expected %s", _cmp, \ + _expectedval); \ + ret = -EBADMSG; \ + break; \ + } \ + } while (0) +#endif + +/* Use where expected data string matches its stringified declaration */ +#define TEST_BTF_C(_str, _type, _flags, ...) \ + TEST_BTF(_str, _type, _flags, "(" #_type ")" #__VA_ARGS__, \ + __VA_ARGS__) + +/* TRACE_EVENT(netif_receive_skb, + * TP_PROTO(struct sk_buff *skb), + */ +SEC("tp_btf/netif_receive_skb") +int BPF_PROG(trace_netif_receive_skb, struct sk_buff *skb) +{ + static __u64 flags[] = { 0, BTF_F_COMPACT, BTF_F_ZERO, BTF_F_PTR_RAW, + BTF_F_NONAME, BTF_F_COMPACT | BTF_F_ZERO | + BTF_F_PTR_RAW | BTF_F_NONAME }; + static struct btf_ptr p = { }; + __u32 key = 0; + int i, __ret; + char *str; + +#if __has_builtin(__builtin_btf_type_id) + str = bpf_map_lookup_elem(&strdata, &key); + if (!str) + return 0; + + /* Ensure we can write skb string representation */ + p.type_id = bpf_core_type_id_kernel(struct sk_buff); + p.ptr = skb; + for (i = 0; i < ARRAY_SIZE(flags); i++) { + ++num_subtests; + ret = bpf_snprintf_btf(str, STRSIZE, &p, sizeof(p), 0); + if (ret < 0) + bpf_printk("returned %d when writing skb", ret); + ++ran_subtests; + } + + /* Check invalid ptr value */ + p.ptr = 0; + __ret = bpf_snprintf_btf(str, STRSIZE, &p, sizeof(p), 0); + if (__ret >= 0) { + bpf_printk("printing NULL should generate error, got (%d)", + __ret); + ret = -ERANGE; + } + + /* Verify type display for various types. */ + + /* simple int */ + TEST_BTF_C(str, int, 0, 1234); + TEST_BTF(str, int, BTF_F_NONAME, "1234", 1234); + /* zero value should be printed at toplevel */ + TEST_BTF(str, int, 0, "(int)0", 0); + TEST_BTF(str, int, BTF_F_NONAME, "0", 0); + TEST_BTF(str, int, BTF_F_ZERO, "(int)0", 0); + TEST_BTF(str, int, BTF_F_NONAME | BTF_F_ZERO, "0", 0); + TEST_BTF_C(str, int, 0, -4567); + TEST_BTF(str, int, BTF_F_NONAME, "-4567", -4567); + + /* simple char */ + TEST_BTF_C(str, char, 0, 100); + TEST_BTF(str, char, BTF_F_NONAME, "100", 100); + /* zero value should be printed at toplevel */ + TEST_BTF(str, char, 0, "(char)0", 0); + TEST_BTF(str, char, BTF_F_NONAME, "0", 0); + TEST_BTF(str, char, BTF_F_ZERO, "(char)0", 0); + TEST_BTF(str, char, BTF_F_NONAME | BTF_F_ZERO, "0", 0); + + /* simple typedef */ + TEST_BTF_C(str, uint64_t, 0, 100); + TEST_BTF(str, u64, BTF_F_NONAME, "1", 1); + /* zero value should be printed at toplevel */ + TEST_BTF(str, u64, 0, "(u64)0", 0); + TEST_BTF(str, u64, BTF_F_NONAME, "0", 0); + TEST_BTF(str, u64, BTF_F_ZERO, "(u64)0", 0); + TEST_BTF(str, u64, BTF_F_NONAME|BTF_F_ZERO, "0", 0); + + /* typedef struct */ + TEST_BTF_C(str, atomic_t, 0, {.counter = (int)1,}); + TEST_BTF(str, atomic_t, BTF_F_NONAME, "{1,}", {.counter = 1,}); + /* typedef with 0 value should be printed at toplevel */ + TEST_BTF(str, atomic_t, 0, "(atomic_t){}", {.counter = 0,}); + TEST_BTF(str, atomic_t, BTF_F_NONAME, "{}", {.counter = 0,}); + TEST_BTF(str, atomic_t, BTF_F_ZERO, "(atomic_t){.counter = (int)0,}", + {.counter = 0,}); + TEST_BTF(str, atomic_t, BTF_F_NONAME|BTF_F_ZERO, + "{0,}", {.counter = 0,}); + + /* enum where enum value does (and does not) exist */ + TEST_BTF_C(str, enum bpf_cmd, 0, BPF_MAP_CREATE); + TEST_BTF(str, enum bpf_cmd, 0, "(enum bpf_cmd)BPF_MAP_CREATE", 0); + TEST_BTF(str, enum bpf_cmd, BTF_F_NONAME, "BPF_MAP_CREATE", + BPF_MAP_CREATE); + TEST_BTF(str, enum bpf_cmd, BTF_F_NONAME|BTF_F_ZERO, + "BPF_MAP_CREATE", 0); + + TEST_BTF(str, enum bpf_cmd, BTF_F_ZERO, "(enum bpf_cmd)BPF_MAP_CREATE", + BPF_MAP_CREATE); + TEST_BTF(str, enum bpf_cmd, BTF_F_NONAME|BTF_F_ZERO, + "BPF_MAP_CREATE", BPF_MAP_CREATE); + TEST_BTF_C(str, enum bpf_cmd, 0, 2000); + TEST_BTF(str, enum bpf_cmd, BTF_F_NONAME, "2000", 2000); + + /* simple struct */ + TEST_BTF_C(str, struct btf_enum, 0, + {.name_off = (__u32)3,.val = (__s32)-1,}); + TEST_BTF(str, struct btf_enum, BTF_F_NONAME, "{3,-1,}", + { .name_off = 3, .val = -1,}); + TEST_BTF(str, struct btf_enum, BTF_F_NONAME, "{-1,}", + { .name_off = 0, .val = -1,}); + TEST_BTF(str, struct btf_enum, BTF_F_NONAME|BTF_F_ZERO, "{0,-1,}", + { .name_off = 0, .val = -1,}); + /* empty struct should be printed */ + TEST_BTF(str, struct btf_enum, 0, "(struct btf_enum){}", + { .name_off = 0, .val = 0,}); + TEST_BTF(str, struct btf_enum, BTF_F_NONAME, "{}", + { .name_off = 0, .val = 0,}); + TEST_BTF(str, struct btf_enum, BTF_F_ZERO, + "(struct btf_enum){.name_off = (__u32)0,.val = (__s32)0,}", + { .name_off = 0, .val = 0,}); + + /* struct with pointers */ + TEST_BTF(str, struct list_head, BTF_F_PTR_RAW, + "(struct list_head){.next = (struct list_head *)0x0000000000000001,}", + { .next = (struct list_head *)1 }); + /* NULL pointer should not be displayed */ + TEST_BTF(str, struct list_head, BTF_F_PTR_RAW, + "(struct list_head){}", + { .next = (struct list_head *)0 }); + + /* struct with char array */ + TEST_BTF(str, struct bpf_prog_info, 0, + "(struct bpf_prog_info){.name = (char[])['f','o','o',],}", + { .name = "foo",}); + TEST_BTF(str, struct bpf_prog_info, BTF_F_NONAME, + "{['f','o','o',],}", + {.name = "foo",}); + /* leading null char means do not display string */ + TEST_BTF(str, struct bpf_prog_info, 0, + "(struct bpf_prog_info){}", + {.name = {'\0', 'f', 'o', 'o'}}); + /* handle non-printable characters */ + TEST_BTF(str, struct bpf_prog_info, 0, + "(struct bpf_prog_info){.name = (char[])[1,2,3,],}", + { .name = {1, 2, 3, 0}}); + + /* struct with non-char array */ + TEST_BTF(str, struct __sk_buff, 0, + "(struct __sk_buff){.cb = (__u32[])[1,2,3,4,5,],}", + { .cb = {1, 2, 3, 4, 5,},}); + TEST_BTF(str, struct __sk_buff, BTF_F_NONAME, + "{[1,2,3,4,5,],}", + { .cb = { 1, 2, 3, 4, 5},}); + /* For non-char, arrays, show non-zero values only */ + TEST_BTF(str, struct __sk_buff, 0, + "(struct __sk_buff){.cb = (__u32[])[1,],}", + { .cb = { 0, 0, 1, 0, 0},}); + + /* struct with bitfields */ + TEST_BTF_C(str, struct bpf_insn, 0, + {.code = (__u8)1,.dst_reg = (__u8)0x2,.src_reg = (__u8)0x3,.off = (__s16)4,.imm = (__s32)5,}); + TEST_BTF(str, struct bpf_insn, BTF_F_NONAME, "{1,0x2,0x3,4,5,}", + {.code = 1, .dst_reg = 0x2, .src_reg = 0x3, .off = 4, + .imm = 5,}); +#else + skip = true; +#endif + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tailcall1.c b/tools/testing/selftests/bpf/progs/tailcall1.c index 1f407e65ae52..7115bcefbe8a 100644 --- a/tools/testing/selftests/bpf/progs/tailcall1.c +++ b/tools/testing/selftests/bpf/progs/tailcall1.c @@ -26,20 +26,20 @@ int entry(struct __sk_buff *skb) /* Multiple locations to make sure we patch * all of them. */ - bpf_tail_call(skb, &jmp_table, 0); - bpf_tail_call(skb, &jmp_table, 0); - bpf_tail_call(skb, &jmp_table, 0); - bpf_tail_call(skb, &jmp_table, 0); - - bpf_tail_call(skb, &jmp_table, 1); - bpf_tail_call(skb, &jmp_table, 1); - bpf_tail_call(skb, &jmp_table, 1); - bpf_tail_call(skb, &jmp_table, 1); - - bpf_tail_call(skb, &jmp_table, 2); - bpf_tail_call(skb, &jmp_table, 2); - bpf_tail_call(skb, &jmp_table, 2); - bpf_tail_call(skb, &jmp_table, 2); + bpf_tail_call_static(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); + + bpf_tail_call_static(skb, &jmp_table, 1); + bpf_tail_call_static(skb, &jmp_table, 1); + bpf_tail_call_static(skb, &jmp_table, 1); + bpf_tail_call_static(skb, &jmp_table, 1); + + bpf_tail_call_static(skb, &jmp_table, 2); + bpf_tail_call_static(skb, &jmp_table, 2); + bpf_tail_call_static(skb, &jmp_table, 2); + bpf_tail_call_static(skb, &jmp_table, 2); return 3; } diff --git a/tools/testing/selftests/bpf/progs/tailcall2.c b/tools/testing/selftests/bpf/progs/tailcall2.c index a093e739cf0e..0431e4fe7efd 100644 --- a/tools/testing/selftests/bpf/progs/tailcall2.c +++ b/tools/testing/selftests/bpf/progs/tailcall2.c @@ -13,14 +13,14 @@ struct { SEC("classifier/0") int bpf_func_0(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 1); + bpf_tail_call_static(skb, &jmp_table, 1); return 0; } SEC("classifier/1") int bpf_func_1(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 2); + bpf_tail_call_static(skb, &jmp_table, 2); return 1; } @@ -33,25 +33,25 @@ int bpf_func_2(struct __sk_buff *skb) SEC("classifier/3") int bpf_func_3(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 4); + bpf_tail_call_static(skb, &jmp_table, 4); return 3; } SEC("classifier/4") int bpf_func_4(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 3); + bpf_tail_call_static(skb, &jmp_table, 3); return 4; } SEC("classifier") int entry(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); /* Check multi-prog update. */ - bpf_tail_call(skb, &jmp_table, 2); + bpf_tail_call_static(skb, &jmp_table, 2); /* Check tail call limit. */ - bpf_tail_call(skb, &jmp_table, 3); + bpf_tail_call_static(skb, &jmp_table, 3); return 3; } diff --git a/tools/testing/selftests/bpf/progs/tailcall3.c b/tools/testing/selftests/bpf/progs/tailcall3.c index cabda877cf0a..739dc2a51e74 100644 --- a/tools/testing/selftests/bpf/progs/tailcall3.c +++ b/tools/testing/selftests/bpf/progs/tailcall3.c @@ -16,14 +16,14 @@ SEC("classifier/0") int bpf_func_0(struct __sk_buff *skb) { count++; - bpf_tail_call(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); return 1; } SEC("classifier") int entry(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); return 0; } diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf1.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf1.c index b5d9c8e778ae..0103f3dd9f02 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf1.c +++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf1.c @@ -21,7 +21,7 @@ TAIL_FUNC(1) static __noinline int subprog_tail(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); return skb->len * 2; } @@ -29,7 +29,7 @@ int subprog_tail(struct __sk_buff *skb) SEC("classifier") int entry(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 1); + bpf_tail_call_static(skb, &jmp_table, 1); return subprog_tail(skb); } diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf2.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf2.c index a004ab28ce28..7b1c04183824 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf2.c +++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf2.c @@ -14,9 +14,9 @@ static __noinline int subprog_tail(struct __sk_buff *skb) { if (load_byte(skb, 0)) - bpf_tail_call(skb, &jmp_table, 1); + bpf_tail_call_static(skb, &jmp_table, 1); else - bpf_tail_call(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); return 1; } @@ -32,7 +32,7 @@ int bpf_func_0(struct __sk_buff *skb) SEC("classifier") int entry(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); return 0; } diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf3.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf3.c index 96dbef2b6b7c..0d5482bea6c9 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf3.c +++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf3.c @@ -16,9 +16,9 @@ int subprog_tail2(struct __sk_buff *skb) volatile char arr[64] = {}; if (load_word(skb, 0) || load_half(skb, 0)) - bpf_tail_call(skb, &jmp_table, 10); + bpf_tail_call_static(skb, &jmp_table, 10); else - bpf_tail_call(skb, &jmp_table, 1); + bpf_tail_call_static(skb, &jmp_table, 1); return skb->len; } @@ -28,7 +28,7 @@ int subprog_tail(struct __sk_buff *skb) { volatile char arr[64] = {}; - bpf_tail_call(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); return skb->len * 2; } diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c index 98b40a95bc67..9a1b166b7fbe 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c +++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c @@ -14,21 +14,21 @@ static volatile int count; __noinline int subprog_tail_2(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 2); + bpf_tail_call_static(skb, &jmp_table, 2); return skb->len * 3; } __noinline int subprog_tail_1(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 1); + bpf_tail_call_static(skb, &jmp_table, 1); return skb->len * 2; } __noinline int subprog_tail(struct __sk_buff *skb) { - bpf_tail_call(skb, &jmp_table, 0); + bpf_tail_call_static(skb, &jmp_table, 0); return skb->len; } diff --git a/tools/testing/selftests/bpf/progs/test_btf_skc_cls_ingress.c b/tools/testing/selftests/bpf/progs/test_btf_skc_cls_ingress.c new file mode 100644 index 000000000000..9a6b85dd52d2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_btf_skc_cls_ingress.c @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020 Facebook */ + +#include <string.h> +#include <errno.h> +#include <netinet/in.h> +#include <linux/stddef.h> +#include <linux/bpf.h> +#include <linux/ipv6.h> +#include <linux/tcp.h> +#include <linux/if_ether.h> +#include <linux/pkt_cls.h> + +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> +#include "bpf_tcp_helpers.h" + +struct sockaddr_in6 srv_sa6 = {}; +__u16 listen_tp_sport = 0; +__u16 req_sk_sport = 0; +__u32 recv_cookie = 0; +__u32 gen_cookie = 0; +__u32 linum = 0; + +#define LOG() ({ if (!linum) linum = __LINE__; }) + +static void test_syncookie_helper(struct ipv6hdr *ip6h, struct tcphdr *th, + struct tcp_sock *tp, + struct __sk_buff *skb) +{ + if (th->syn) { + __s64 mss_cookie; + void *data_end; + + data_end = (void *)(long)(skb->data_end); + + if (th->doff * 4 != 40) { + LOG(); + return; + } + + if ((void *)th + 40 > data_end) { + LOG(); + return; + } + + mss_cookie = bpf_tcp_gen_syncookie(tp, ip6h, sizeof(*ip6h), + th, 40); + if (mss_cookie < 0) { + if (mss_cookie != -ENOENT) + LOG(); + } else { + gen_cookie = (__u32)mss_cookie; + } + } else if (gen_cookie) { + /* It was in cookie mode */ + int ret = bpf_tcp_check_syncookie(tp, ip6h, sizeof(*ip6h), + th, sizeof(*th)); + + if (ret < 0) { + if (ret != -ENOENT) + LOG(); + } else { + recv_cookie = bpf_ntohl(th->ack_seq) - 1; + } + } +} + +static int handle_ip6_tcp(struct ipv6hdr *ip6h, struct __sk_buff *skb) +{ + struct bpf_sock_tuple *tuple; + struct bpf_sock *bpf_skc; + unsigned int tuple_len; + struct tcphdr *th; + void *data_end; + + data_end = (void *)(long)(skb->data_end); + + th = (struct tcphdr *)(ip6h + 1); + if (th + 1 > data_end) + return TC_ACT_OK; + + /* Is it the testing traffic? */ + if (th->dest != srv_sa6.sin6_port) + return TC_ACT_OK; + + tuple_len = sizeof(tuple->ipv6); + tuple = (struct bpf_sock_tuple *)&ip6h->saddr; + if ((void *)tuple + tuple_len > data_end) { + LOG(); + return TC_ACT_OK; + } + + bpf_skc = bpf_skc_lookup_tcp(skb, tuple, tuple_len, + BPF_F_CURRENT_NETNS, 0); + if (!bpf_skc) { + LOG(); + return TC_ACT_OK; + } + + if (bpf_skc->state == BPF_TCP_NEW_SYN_RECV) { + struct request_sock *req_sk; + + req_sk = (struct request_sock *)bpf_skc_to_tcp_request_sock(bpf_skc); + if (!req_sk) { + LOG(); + goto release; + } + + if (bpf_sk_assign(skb, req_sk, 0)) { + LOG(); + goto release; + } + + req_sk_sport = req_sk->__req_common.skc_num; + + bpf_sk_release(req_sk); + return TC_ACT_OK; + } else if (bpf_skc->state == BPF_TCP_LISTEN) { + struct tcp_sock *tp; + + tp = bpf_skc_to_tcp_sock(bpf_skc); + if (!tp) { + LOG(); + goto release; + } + + if (bpf_sk_assign(skb, tp, 0)) { + LOG(); + goto release; + } + + listen_tp_sport = tp->inet_conn.icsk_inet.sk.__sk_common.skc_num; + + test_syncookie_helper(ip6h, th, tp, skb); + bpf_sk_release(tp); + return TC_ACT_OK; + } + + if (bpf_sk_assign(skb, bpf_skc, 0)) + LOG(); + +release: + bpf_sk_release(bpf_skc); + return TC_ACT_OK; +} + +SEC("classifier/ingress") +int cls_ingress(struct __sk_buff *skb) +{ + struct ipv6hdr *ip6h; + struct ethhdr *eth; + void *data_end; + + data_end = (void *)(long)(skb->data_end); + + eth = (struct ethhdr *)(long)(skb->data); + if (eth + 1 > data_end) + return TC_ACT_OK; + + if (eth->h_proto != bpf_htons(ETH_P_IPV6)) + return TC_ACT_OK; + + ip6h = (struct ipv6hdr *)(eth + 1); + if (ip6h + 1 > data_end) + return TC_ACT_OK; + + if (ip6h->nexthdr == IPPROTO_TCP) + return handle_ip6_tcp(ip6h, skb); + + return TC_ACT_OK; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_overhead.c b/tools/testing/selftests/bpf/progs/test_overhead.c index 42403d088abc..abb7344b531f 100644 --- a/tools/testing/selftests/bpf/progs/test_overhead.c +++ b/tools/testing/selftests/bpf/progs/test_overhead.c @@ -39,10 +39,4 @@ int BPF_PROG(prog5, struct task_struct *tsk, const char *buf, bool exec) return 0; } -SEC("fmod_ret/__set_task_comm") -int BPF_PROG(prog6, struct task_struct *tsk, const char *buf, bool exec) -{ - return !tsk; -} - char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_pe_preserve_elems.c b/tools/testing/selftests/bpf/progs/test_pe_preserve_elems.c new file mode 100644 index 000000000000..fb22de7c365d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_pe_preserve_elems.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2020 Facebook +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); +} array_1 SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); + __uint(map_flags, BPF_F_PRESERVE_ELEMS); +} array_2 SEC(".maps"); + +SEC("raw_tp/sched_switch") +int BPF_PROG(read_array_1) +{ + struct bpf_perf_event_value val; + + return bpf_perf_event_read_value(&array_1, 0, &val, sizeof(val)); +} + +SEC("raw_tp/task_rename") +int BPF_PROG(read_array_2) +{ + struct bpf_perf_event_value val; + + return bpf_perf_event_read_value(&array_2, 0, &val, sizeof(val)); +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_raw_tp_test_run.c b/tools/testing/selftests/bpf/progs/test_raw_tp_test_run.c new file mode 100644 index 000000000000..4c63cc87b9d0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_raw_tp_test_run.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020 Facebook */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +__u32 count = 0; +__u32 on_cpu = 0xffffffff; + +SEC("raw_tp/task_rename") +int BPF_PROG(rename, struct task_struct *task, char *comm) +{ + + count++; + if ((__u64) task == 0x1234ULL && (__u64) comm == 0x5678ULL) { + on_cpu = bpf_get_smp_processor_id(); + return (long)task + (long)comm; + } + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_sk_lookup.c b/tools/testing/selftests/bpf/progs/test_sk_lookup.c index bbf8296f4d66..1032b292af5b 100644 --- a/tools/testing/selftests/bpf/progs/test_sk_lookup.c +++ b/tools/testing/selftests/bpf/progs/test_sk_lookup.c @@ -19,6 +19,17 @@ #define IP6(aaaa, bbbb, cccc, dddd) \ { bpf_htonl(aaaa), bpf_htonl(bbbb), bpf_htonl(cccc), bpf_htonl(dddd) } +/* Macros for least-significant byte and word accesses. */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define LSE_INDEX(index, size) (index) +#else +#define LSE_INDEX(index, size) ((size) - (index) - 1) +#endif +#define LSB(value, index) \ + (((__u8 *)&(value))[LSE_INDEX((index), sizeof(value))]) +#define LSW(value, index) \ + (((__u16 *)&(value))[LSE_INDEX((index), sizeof(value) / 2)]) + #define MAX_SOCKS 32 struct { @@ -369,171 +380,146 @@ int ctx_narrow_access(struct bpf_sk_lookup *ctx) { struct bpf_sock *sk; int err, family; - __u16 *half; - __u8 *byte; bool v4; v4 = (ctx->family == AF_INET); /* Narrow loads from family field */ - byte = (__u8 *)&ctx->family; - half = (__u16 *)&ctx->family; - if (byte[0] != (v4 ? AF_INET : AF_INET6) || - byte[1] != 0 || byte[2] != 0 || byte[3] != 0) + if (LSB(ctx->family, 0) != (v4 ? AF_INET : AF_INET6) || + LSB(ctx->family, 1) != 0 || LSB(ctx->family, 2) != 0 || LSB(ctx->family, 3) != 0) return SK_DROP; - if (half[0] != (v4 ? AF_INET : AF_INET6)) + if (LSW(ctx->family, 0) != (v4 ? AF_INET : AF_INET6)) return SK_DROP; - byte = (__u8 *)&ctx->protocol; - if (byte[0] != IPPROTO_TCP || - byte[1] != 0 || byte[2] != 0 || byte[3] != 0) + /* Narrow loads from protocol field */ + if (LSB(ctx->protocol, 0) != IPPROTO_TCP || + LSB(ctx->protocol, 1) != 0 || LSB(ctx->protocol, 2) != 0 || LSB(ctx->protocol, 3) != 0) return SK_DROP; - half = (__u16 *)&ctx->protocol; - if (half[0] != IPPROTO_TCP) + if (LSW(ctx->protocol, 0) != IPPROTO_TCP) return SK_DROP; /* Narrow loads from remote_port field. Expect non-0 value. */ - byte = (__u8 *)&ctx->remote_port; - if (byte[0] == 0 && byte[1] == 0 && byte[2] == 0 && byte[3] == 0) + if (LSB(ctx->remote_port, 0) == 0 && LSB(ctx->remote_port, 1) == 0 && + LSB(ctx->remote_port, 2) == 0 && LSB(ctx->remote_port, 3) == 0) return SK_DROP; - half = (__u16 *)&ctx->remote_port; - if (half[0] == 0) + if (LSW(ctx->remote_port, 0) == 0) return SK_DROP; /* Narrow loads from local_port field. Expect DST_PORT. */ - byte = (__u8 *)&ctx->local_port; - if (byte[0] != ((DST_PORT >> 0) & 0xff) || - byte[1] != ((DST_PORT >> 8) & 0xff) || - byte[2] != 0 || byte[3] != 0) + if (LSB(ctx->local_port, 0) != ((DST_PORT >> 0) & 0xff) || + LSB(ctx->local_port, 1) != ((DST_PORT >> 8) & 0xff) || + LSB(ctx->local_port, 2) != 0 || LSB(ctx->local_port, 3) != 0) return SK_DROP; - half = (__u16 *)&ctx->local_port; - if (half[0] != DST_PORT) + if (LSW(ctx->local_port, 0) != DST_PORT) return SK_DROP; /* Narrow loads from IPv4 fields */ if (v4) { /* Expect non-0.0.0.0 in remote_ip4 */ - byte = (__u8 *)&ctx->remote_ip4; - if (byte[0] == 0 && byte[1] == 0 && - byte[2] == 0 && byte[3] == 0) + if (LSB(ctx->remote_ip4, 0) == 0 && LSB(ctx->remote_ip4, 1) == 0 && + LSB(ctx->remote_ip4, 2) == 0 && LSB(ctx->remote_ip4, 3) == 0) return SK_DROP; - half = (__u16 *)&ctx->remote_ip4; - if (half[0] == 0 && half[1] == 0) + if (LSW(ctx->remote_ip4, 0) == 0 && LSW(ctx->remote_ip4, 1) == 0) return SK_DROP; /* Expect DST_IP4 in local_ip4 */ - byte = (__u8 *)&ctx->local_ip4; - if (byte[0] != ((DST_IP4 >> 0) & 0xff) || - byte[1] != ((DST_IP4 >> 8) & 0xff) || - byte[2] != ((DST_IP4 >> 16) & 0xff) || - byte[3] != ((DST_IP4 >> 24) & 0xff)) + if (LSB(ctx->local_ip4, 0) != ((DST_IP4 >> 0) & 0xff) || + LSB(ctx->local_ip4, 1) != ((DST_IP4 >> 8) & 0xff) || + LSB(ctx->local_ip4, 2) != ((DST_IP4 >> 16) & 0xff) || + LSB(ctx->local_ip4, 3) != ((DST_IP4 >> 24) & 0xff)) return SK_DROP; - half = (__u16 *)&ctx->local_ip4; - if (half[0] != ((DST_IP4 >> 0) & 0xffff) || - half[1] != ((DST_IP4 >> 16) & 0xffff)) + if (LSW(ctx->local_ip4, 0) != ((DST_IP4 >> 0) & 0xffff) || + LSW(ctx->local_ip4, 1) != ((DST_IP4 >> 16) & 0xffff)) return SK_DROP; } else { /* Expect 0.0.0.0 IPs when family != AF_INET */ - byte = (__u8 *)&ctx->remote_ip4; - if (byte[0] != 0 || byte[1] != 0 && - byte[2] != 0 || byte[3] != 0) + if (LSB(ctx->remote_ip4, 0) != 0 || LSB(ctx->remote_ip4, 1) != 0 || + LSB(ctx->remote_ip4, 2) != 0 || LSB(ctx->remote_ip4, 3) != 0) return SK_DROP; - half = (__u16 *)&ctx->remote_ip4; - if (half[0] != 0 || half[1] != 0) + if (LSW(ctx->remote_ip4, 0) != 0 || LSW(ctx->remote_ip4, 1) != 0) return SK_DROP; - byte = (__u8 *)&ctx->local_ip4; - if (byte[0] != 0 || byte[1] != 0 && - byte[2] != 0 || byte[3] != 0) + if (LSB(ctx->local_ip4, 0) != 0 || LSB(ctx->local_ip4, 1) != 0 || + LSB(ctx->local_ip4, 2) != 0 || LSB(ctx->local_ip4, 3) != 0) return SK_DROP; - half = (__u16 *)&ctx->local_ip4; - if (half[0] != 0 || half[1] != 0) + if (LSW(ctx->local_ip4, 0) != 0 || LSW(ctx->local_ip4, 1) != 0) return SK_DROP; } /* Narrow loads from IPv6 fields */ if (!v4) { - /* Expenct non-:: IP in remote_ip6 */ - byte = (__u8 *)&ctx->remote_ip6; - if (byte[0] == 0 && byte[1] == 0 && - byte[2] == 0 && byte[3] == 0 && - byte[4] == 0 && byte[5] == 0 && - byte[6] == 0 && byte[7] == 0 && - byte[8] == 0 && byte[9] == 0 && - byte[10] == 0 && byte[11] == 0 && - byte[12] == 0 && byte[13] == 0 && - byte[14] == 0 && byte[15] == 0) + /* Expect non-:: IP in remote_ip6 */ + if (LSB(ctx->remote_ip6[0], 0) == 0 && LSB(ctx->remote_ip6[0], 1) == 0 && + LSB(ctx->remote_ip6[0], 2) == 0 && LSB(ctx->remote_ip6[0], 3) == 0 && + LSB(ctx->remote_ip6[1], 0) == 0 && LSB(ctx->remote_ip6[1], 1) == 0 && + LSB(ctx->remote_ip6[1], 2) == 0 && LSB(ctx->remote_ip6[1], 3) == 0 && + LSB(ctx->remote_ip6[2], 0) == 0 && LSB(ctx->remote_ip6[2], 1) == 0 && + LSB(ctx->remote_ip6[2], 2) == 0 && LSB(ctx->remote_ip6[2], 3) == 0 && + LSB(ctx->remote_ip6[3], 0) == 0 && LSB(ctx->remote_ip6[3], 1) == 0 && + LSB(ctx->remote_ip6[3], 2) == 0 && LSB(ctx->remote_ip6[3], 3) == 0) return SK_DROP; - half = (__u16 *)&ctx->remote_ip6; - if (half[0] == 0 && half[1] == 0 && - half[2] == 0 && half[3] == 0 && - half[4] == 0 && half[5] == 0 && - half[6] == 0 && half[7] == 0) + if (LSW(ctx->remote_ip6[0], 0) == 0 && LSW(ctx->remote_ip6[0], 1) == 0 && + LSW(ctx->remote_ip6[1], 0) == 0 && LSW(ctx->remote_ip6[1], 1) == 0 && + LSW(ctx->remote_ip6[2], 0) == 0 && LSW(ctx->remote_ip6[2], 1) == 0 && + LSW(ctx->remote_ip6[3], 0) == 0 && LSW(ctx->remote_ip6[3], 1) == 0) return SK_DROP; - /* Expect DST_IP6 in local_ip6 */ - byte = (__u8 *)&ctx->local_ip6; - if (byte[0] != ((DST_IP6[0] >> 0) & 0xff) || - byte[1] != ((DST_IP6[0] >> 8) & 0xff) || - byte[2] != ((DST_IP6[0] >> 16) & 0xff) || - byte[3] != ((DST_IP6[0] >> 24) & 0xff) || - byte[4] != ((DST_IP6[1] >> 0) & 0xff) || - byte[5] != ((DST_IP6[1] >> 8) & 0xff) || - byte[6] != ((DST_IP6[1] >> 16) & 0xff) || - byte[7] != ((DST_IP6[1] >> 24) & 0xff) || - byte[8] != ((DST_IP6[2] >> 0) & 0xff) || - byte[9] != ((DST_IP6[2] >> 8) & 0xff) || - byte[10] != ((DST_IP6[2] >> 16) & 0xff) || - byte[11] != ((DST_IP6[2] >> 24) & 0xff) || - byte[12] != ((DST_IP6[3] >> 0) & 0xff) || - byte[13] != ((DST_IP6[3] >> 8) & 0xff) || - byte[14] != ((DST_IP6[3] >> 16) & 0xff) || - byte[15] != ((DST_IP6[3] >> 24) & 0xff)) + if (LSB(ctx->local_ip6[0], 0) != ((DST_IP6[0] >> 0) & 0xff) || + LSB(ctx->local_ip6[0], 1) != ((DST_IP6[0] >> 8) & 0xff) || + LSB(ctx->local_ip6[0], 2) != ((DST_IP6[0] >> 16) & 0xff) || + LSB(ctx->local_ip6[0], 3) != ((DST_IP6[0] >> 24) & 0xff) || + LSB(ctx->local_ip6[1], 0) != ((DST_IP6[1] >> 0) & 0xff) || + LSB(ctx->local_ip6[1], 1) != ((DST_IP6[1] >> 8) & 0xff) || + LSB(ctx->local_ip6[1], 2) != ((DST_IP6[1] >> 16) & 0xff) || + LSB(ctx->local_ip6[1], 3) != ((DST_IP6[1] >> 24) & 0xff) || + LSB(ctx->local_ip6[2], 0) != ((DST_IP6[2] >> 0) & 0xff) || + LSB(ctx->local_ip6[2], 1) != ((DST_IP6[2] >> 8) & 0xff) || + LSB(ctx->local_ip6[2], 2) != ((DST_IP6[2] >> 16) & 0xff) || + LSB(ctx->local_ip6[2], 3) != ((DST_IP6[2] >> 24) & 0xff) || + LSB(ctx->local_ip6[3], 0) != ((DST_IP6[3] >> 0) & 0xff) || + LSB(ctx->local_ip6[3], 1) != ((DST_IP6[3] >> 8) & 0xff) || + LSB(ctx->local_ip6[3], 2) != ((DST_IP6[3] >> 16) & 0xff) || + LSB(ctx->local_ip6[3], 3) != ((DST_IP6[3] >> 24) & 0xff)) return SK_DROP; - half = (__u16 *)&ctx->local_ip6; - if (half[0] != ((DST_IP6[0] >> 0) & 0xffff) || - half[1] != ((DST_IP6[0] >> 16) & 0xffff) || - half[2] != ((DST_IP6[1] >> 0) & 0xffff) || - half[3] != ((DST_IP6[1] >> 16) & 0xffff) || - half[4] != ((DST_IP6[2] >> 0) & 0xffff) || - half[5] != ((DST_IP6[2] >> 16) & 0xffff) || - half[6] != ((DST_IP6[3] >> 0) & 0xffff) || - half[7] != ((DST_IP6[3] >> 16) & 0xffff)) + if (LSW(ctx->local_ip6[0], 0) != ((DST_IP6[0] >> 0) & 0xffff) || + LSW(ctx->local_ip6[0], 1) != ((DST_IP6[0] >> 16) & 0xffff) || + LSW(ctx->local_ip6[1], 0) != ((DST_IP6[1] >> 0) & 0xffff) || + LSW(ctx->local_ip6[1], 1) != ((DST_IP6[1] >> 16) & 0xffff) || + LSW(ctx->local_ip6[2], 0) != ((DST_IP6[2] >> 0) & 0xffff) || + LSW(ctx->local_ip6[2], 1) != ((DST_IP6[2] >> 16) & 0xffff) || + LSW(ctx->local_ip6[3], 0) != ((DST_IP6[3] >> 0) & 0xffff) || + LSW(ctx->local_ip6[3], 1) != ((DST_IP6[3] >> 16) & 0xffff)) return SK_DROP; } else { /* Expect :: IPs when family != AF_INET6 */ - byte = (__u8 *)&ctx->remote_ip6; - if (byte[0] != 0 || byte[1] != 0 || - byte[2] != 0 || byte[3] != 0 || - byte[4] != 0 || byte[5] != 0 || - byte[6] != 0 || byte[7] != 0 || - byte[8] != 0 || byte[9] != 0 || - byte[10] != 0 || byte[11] != 0 || - byte[12] != 0 || byte[13] != 0 || - byte[14] != 0 || byte[15] != 0) + if (LSB(ctx->remote_ip6[0], 0) != 0 || LSB(ctx->remote_ip6[0], 1) != 0 || + LSB(ctx->remote_ip6[0], 2) != 0 || LSB(ctx->remote_ip6[0], 3) != 0 || + LSB(ctx->remote_ip6[1], 0) != 0 || LSB(ctx->remote_ip6[1], 1) != 0 || + LSB(ctx->remote_ip6[1], 2) != 0 || LSB(ctx->remote_ip6[1], 3) != 0 || + LSB(ctx->remote_ip6[2], 0) != 0 || LSB(ctx->remote_ip6[2], 1) != 0 || + LSB(ctx->remote_ip6[2], 2) != 0 || LSB(ctx->remote_ip6[2], 3) != 0 || + LSB(ctx->remote_ip6[3], 0) != 0 || LSB(ctx->remote_ip6[3], 1) != 0 || + LSB(ctx->remote_ip6[3], 2) != 0 || LSB(ctx->remote_ip6[3], 3) != 0) return SK_DROP; - half = (__u16 *)&ctx->remote_ip6; - if (half[0] != 0 || half[1] != 0 || - half[2] != 0 || half[3] != 0 || - half[4] != 0 || half[5] != 0 || - half[6] != 0 || half[7] != 0) + if (LSW(ctx->remote_ip6[0], 0) != 0 || LSW(ctx->remote_ip6[0], 1) != 0 || + LSW(ctx->remote_ip6[1], 0) != 0 || LSW(ctx->remote_ip6[1], 1) != 0 || + LSW(ctx->remote_ip6[2], 0) != 0 || LSW(ctx->remote_ip6[2], 1) != 0 || + LSW(ctx->remote_ip6[3], 0) != 0 || LSW(ctx->remote_ip6[3], 1) != 0) return SK_DROP; - byte = (__u8 *)&ctx->local_ip6; - if (byte[0] != 0 || byte[1] != 0 || - byte[2] != 0 || byte[3] != 0 || - byte[4] != 0 || byte[5] != 0 || - byte[6] != 0 || byte[7] != 0 || - byte[8] != 0 || byte[9] != 0 || - byte[10] != 0 || byte[11] != 0 || - byte[12] != 0 || byte[13] != 0 || - byte[14] != 0 || byte[15] != 0) + if (LSB(ctx->local_ip6[0], 0) != 0 || LSB(ctx->local_ip6[0], 1) != 0 || + LSB(ctx->local_ip6[0], 2) != 0 || LSB(ctx->local_ip6[0], 3) != 0 || + LSB(ctx->local_ip6[1], 0) != 0 || LSB(ctx->local_ip6[1], 1) != 0 || + LSB(ctx->local_ip6[1], 2) != 0 || LSB(ctx->local_ip6[1], 3) != 0 || + LSB(ctx->local_ip6[2], 0) != 0 || LSB(ctx->local_ip6[2], 1) != 0 || + LSB(ctx->local_ip6[2], 2) != 0 || LSB(ctx->local_ip6[2], 3) != 0 || + LSB(ctx->local_ip6[3], 0) != 0 || LSB(ctx->local_ip6[3], 1) != 0 || + LSB(ctx->local_ip6[3], 2) != 0 || LSB(ctx->local_ip6[3], 3) != 0) return SK_DROP; - half = (__u16 *)&ctx->local_ip6; - if (half[0] != 0 || half[1] != 0 || - half[2] != 0 || half[3] != 0 || - half[4] != 0 || half[5] != 0 || - half[6] != 0 || half[7] != 0) + if (LSW(ctx->remote_ip6[0], 0) != 0 || LSW(ctx->remote_ip6[0], 1) != 0 || + LSW(ctx->remote_ip6[1], 0) != 0 || LSW(ctx->remote_ip6[1], 1) != 0 || + LSW(ctx->remote_ip6[2], 0) != 0 || LSW(ctx->remote_ip6[2], 1) != 0 || + LSW(ctx->remote_ip6[3], 0) != 0 || LSW(ctx->remote_ip6[3], 1) != 0) return SK_DROP; } diff --git a/tools/testing/selftests/bpf/progs/test_sock_fields_kern.c b/tools/testing/selftests/bpf/progs/test_sock_fields.c index 9bcaa37f476a..81b57b9aaaea 100644 --- a/tools/testing/selftests/bpf/progs/test_sock_fields_kern.c +++ b/tools/testing/selftests/bpf/progs/test_sock_fields.c @@ -7,19 +7,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_endian.h> - -enum bpf_addr_array_idx { - ADDR_SRV_IDX, - ADDR_CLI_IDX, - __NR_BPF_ADDR_ARRAY_IDX, -}; - -enum bpf_result_array_idx { - EGRESS_SRV_IDX, - EGRESS_CLI_IDX, - INGRESS_LISTEN_IDX, - __NR_BPF_RESULT_ARRAY_IDX, -}; +#include "bpf_tcp_helpers.h" enum bpf_linum_array_idx { EGRESS_LINUM_IDX, @@ -29,27 +17,6 @@ enum bpf_linum_array_idx { struct { __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(max_entries, __NR_BPF_ADDR_ARRAY_IDX); - __type(key, __u32); - __type(value, struct sockaddr_in6); -} addr_map SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(max_entries, __NR_BPF_RESULT_ARRAY_IDX); - __type(key, __u32); - __type(value, struct bpf_sock); -} sock_result_map SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(max_entries, __NR_BPF_RESULT_ARRAY_IDX); - __type(key, __u32); - __type(value, struct bpf_tcp_sock); -} tcp_sock_result_map SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_ARRAY); __uint(max_entries, __NR_BPF_LINUM_ARRAY_IDX); __type(key, __u32); __type(value, __u32); @@ -74,6 +41,17 @@ struct { __type(value, struct bpf_spinlock_cnt); } sk_pkt_out_cnt10 SEC(".maps"); +struct bpf_tcp_sock listen_tp = {}; +struct sockaddr_in6 srv_sa6 = {}; +struct bpf_tcp_sock cli_tp = {}; +struct bpf_tcp_sock srv_tp = {}; +struct bpf_sock listen_sk = {}; +struct bpf_sock srv_sk = {}; +struct bpf_sock cli_sk = {}; +__u64 parent_cg_id = 0; +__u64 child_cg_id = 0; +__u64 lsndtime = 0; + static bool is_loopback6(__u32 *a6) { return !a6[0] && !a6[1] && !a6[2] && a6[3] == bpf_htonl(1); @@ -130,62 +108,86 @@ static void tpcpy(struct bpf_tcp_sock *dst, dst->bytes_acked = src->bytes_acked; } -#define RETURN { \ +/* Always return CG_OK so that no pkt will be filtered out */ +#define CG_OK 1 + +#define RET_LOG() ({ \ linum = __LINE__; \ - bpf_map_update_elem(&linum_map, &linum_idx, &linum, 0); \ - return 1; \ -} + bpf_map_update_elem(&linum_map, &linum_idx, &linum, BPF_NOEXIST); \ + return CG_OK; \ +}) SEC("cgroup_skb/egress") int egress_read_sock_fields(struct __sk_buff *skb) { struct bpf_spinlock_cnt cli_cnt_init = { .lock = 0, .cnt = 0xeB9F }; - __u32 srv_idx = ADDR_SRV_IDX, cli_idx = ADDR_CLI_IDX, result_idx; struct bpf_spinlock_cnt *pkt_out_cnt, *pkt_out_cnt10; - struct sockaddr_in6 *srv_sa6, *cli_sa6; struct bpf_tcp_sock *tp, *tp_ret; struct bpf_sock *sk, *sk_ret; __u32 linum, linum_idx; + struct tcp_sock *ktp; linum_idx = EGRESS_LINUM_IDX; sk = skb->sk; - if (!sk || sk->state == 10) - RETURN; + if (!sk) + RET_LOG(); + /* Not the testing egress traffic or + * TCP_LISTEN (10) socket will be copied at the ingress side. + */ + if (sk->family != AF_INET6 || !is_loopback6(sk->src_ip6) || + sk->state == 10) + return CG_OK; + + if (sk->src_port == bpf_ntohs(srv_sa6.sin6_port)) { + /* Server socket */ + sk_ret = &srv_sk; + tp_ret = &srv_tp; + } else if (sk->dst_port == srv_sa6.sin6_port) { + /* Client socket */ + sk_ret = &cli_sk; + tp_ret = &cli_tp; + } else { + /* Not the testing egress traffic */ + return CG_OK; + } + + /* It must be a fullsock for cgroup_skb/egress prog */ sk = bpf_sk_fullsock(sk); - if (!sk || sk->family != AF_INET6 || sk->protocol != IPPROTO_TCP || - !is_loopback6(sk->src_ip6)) - RETURN; + if (!sk) + RET_LOG(); + + /* Not the testing egress traffic */ + if (sk->protocol != IPPROTO_TCP) + return CG_OK; tp = bpf_tcp_sock(sk); if (!tp) - RETURN; + RET_LOG(); - srv_sa6 = bpf_map_lookup_elem(&addr_map, &srv_idx); - cli_sa6 = bpf_map_lookup_elem(&addr_map, &cli_idx); - if (!srv_sa6 || !cli_sa6) - RETURN; + skcpy(sk_ret, sk); + tpcpy(tp_ret, tp); - if (sk->src_port == bpf_ntohs(srv_sa6->sin6_port)) - result_idx = EGRESS_SRV_IDX; - else if (sk->src_port == bpf_ntohs(cli_sa6->sin6_port)) - result_idx = EGRESS_CLI_IDX; - else - RETURN; + if (sk_ret == &srv_sk) { + ktp = bpf_skc_to_tcp_sock(sk); - sk_ret = bpf_map_lookup_elem(&sock_result_map, &result_idx); - tp_ret = bpf_map_lookup_elem(&tcp_sock_result_map, &result_idx); - if (!sk_ret || !tp_ret) - RETURN; + if (!ktp) + RET_LOG(); - skcpy(sk_ret, sk); - tpcpy(tp_ret, tp); + lsndtime = ktp->lsndtime; + + child_cg_id = bpf_sk_cgroup_id(ktp); + if (!child_cg_id) + RET_LOG(); + + parent_cg_id = bpf_sk_ancestor_cgroup_id(ktp, 2); + if (!parent_cg_id) + RET_LOG(); - if (result_idx == EGRESS_SRV_IDX) { /* The userspace has created it for srv sk */ - pkt_out_cnt = bpf_sk_storage_get(&sk_pkt_out_cnt, sk, 0, 0); - pkt_out_cnt10 = bpf_sk_storage_get(&sk_pkt_out_cnt10, sk, + pkt_out_cnt = bpf_sk_storage_get(&sk_pkt_out_cnt, ktp, 0, 0); + pkt_out_cnt10 = bpf_sk_storage_get(&sk_pkt_out_cnt10, ktp, 0, 0); } else { pkt_out_cnt = bpf_sk_storage_get(&sk_pkt_out_cnt, sk, @@ -197,7 +199,7 @@ int egress_read_sock_fields(struct __sk_buff *skb) } if (!pkt_out_cnt || !pkt_out_cnt10) - RETURN; + RET_LOG(); /* Even both cnt and cnt10 have lock defined in their BTF, * intentionally one cnt takes lock while one does not @@ -208,48 +210,44 @@ int egress_read_sock_fields(struct __sk_buff *skb) pkt_out_cnt10->cnt += 10; bpf_spin_unlock(&pkt_out_cnt10->lock); - RETURN; + return CG_OK; } SEC("cgroup_skb/ingress") int ingress_read_sock_fields(struct __sk_buff *skb) { - __u32 srv_idx = ADDR_SRV_IDX, result_idx = INGRESS_LISTEN_IDX; - struct bpf_tcp_sock *tp, *tp_ret; - struct bpf_sock *sk, *sk_ret; - struct sockaddr_in6 *srv_sa6; + struct bpf_tcp_sock *tp; __u32 linum, linum_idx; + struct bpf_sock *sk; linum_idx = INGRESS_LINUM_IDX; sk = skb->sk; - if (!sk || sk->family != AF_INET6 || !is_loopback6(sk->src_ip6)) - RETURN; + if (!sk) + RET_LOG(); - srv_sa6 = bpf_map_lookup_elem(&addr_map, &srv_idx); - if (!srv_sa6 || sk->src_port != bpf_ntohs(srv_sa6->sin6_port)) - RETURN; + /* Not the testing ingress traffic to the server */ + if (sk->family != AF_INET6 || !is_loopback6(sk->src_ip6) || + sk->src_port != bpf_ntohs(srv_sa6.sin6_port)) + return CG_OK; - if (sk->state != 10 && sk->state != 12) - RETURN; + /* Only interested in TCP_LISTEN */ + if (sk->state != 10) + return CG_OK; - sk = bpf_get_listener_sock(sk); + /* It must be a fullsock for cgroup_skb/ingress prog */ + sk = bpf_sk_fullsock(sk); if (!sk) - RETURN; + RET_LOG(); tp = bpf_tcp_sock(sk); if (!tp) - RETURN; - - sk_ret = bpf_map_lookup_elem(&sock_result_map, &result_idx); - tp_ret = bpf_map_lookup_elem(&tcp_sock_result_map, &result_idx); - if (!sk_ret || !tp_ret) - RETURN; + RET_LOG(); - skcpy(sk_ret, sk); - tpcpy(tp_ret, tp); + skcpy(&listen_sk, sk); + tpcpy(&listen_tp, tp); - RETURN; + return CG_OK; } char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tc_neigh.c b/tools/testing/selftests/bpf/progs/test_tc_neigh.c new file mode 100644 index 000000000000..889a72c3024f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_tc_neigh.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stdint.h> +#include <stdbool.h> + +#include <linux/bpf.h> +#include <linux/stddef.h> +#include <linux/pkt_cls.h> +#include <linux/if_ether.h> +#include <linux/in.h> +#include <linux/ip.h> +#include <linux/ipv6.h> + +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> + +#ifndef barrier_data +# define barrier_data(ptr) asm volatile("": :"r"(ptr) :"memory") +#endif + +#ifndef ctx_ptr +# define ctx_ptr(field) (void *)(long)(field) +#endif + +#define dst_to_src_tmp 0xeeddddeeU +#define src_to_dst_tmp 0xeeffffeeU + +#define ip4_src 0xac100164 /* 172.16.1.100 */ +#define ip4_dst 0xac100264 /* 172.16.2.100 */ + +#define ip6_src { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ + 0x00, 0x01, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe } +#define ip6_dst { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ + 0x00, 0x02, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe } + +#ifndef v6_equal +# define v6_equal(a, b) (a.s6_addr32[0] == b.s6_addr32[0] && \ + a.s6_addr32[1] == b.s6_addr32[1] && \ + a.s6_addr32[2] == b.s6_addr32[2] && \ + a.s6_addr32[3] == b.s6_addr32[3]) +#endif + +static __always_inline bool is_remote_ep_v4(struct __sk_buff *skb, + __be32 addr) +{ + void *data_end = ctx_ptr(skb->data_end); + void *data = ctx_ptr(skb->data); + struct iphdr *ip4h; + + if (data + sizeof(struct ethhdr) > data_end) + return false; + + ip4h = (struct iphdr *)(data + sizeof(struct ethhdr)); + if ((void *)(ip4h + 1) > data_end) + return false; + + return ip4h->daddr == addr; +} + +static __always_inline bool is_remote_ep_v6(struct __sk_buff *skb, + struct in6_addr addr) +{ + void *data_end = ctx_ptr(skb->data_end); + void *data = ctx_ptr(skb->data); + struct ipv6hdr *ip6h; + + if (data + sizeof(struct ethhdr) > data_end) + return false; + + ip6h = (struct ipv6hdr *)(data + sizeof(struct ethhdr)); + if ((void *)(ip6h + 1) > data_end) + return false; + + return v6_equal(ip6h->daddr, addr); +} + +SEC("chk_neigh") int tc_chk(struct __sk_buff *skb) +{ + void *data_end = ctx_ptr(skb->data_end); + void *data = ctx_ptr(skb->data); + __u32 *raw = data; + + if (data + sizeof(struct ethhdr) > data_end) + return TC_ACT_SHOT; + + return !raw[0] && !raw[1] && !raw[2] ? TC_ACT_SHOT : TC_ACT_OK; +} + +SEC("dst_ingress") int tc_dst(struct __sk_buff *skb) +{ + int idx = dst_to_src_tmp; + __u8 zero[ETH_ALEN * 2]; + bool redirect = false; + + switch (skb->protocol) { + case __bpf_constant_htons(ETH_P_IP): + redirect = is_remote_ep_v4(skb, __bpf_constant_htonl(ip4_src)); + break; + case __bpf_constant_htons(ETH_P_IPV6): + redirect = is_remote_ep_v6(skb, (struct in6_addr)ip6_src); + break; + } + + if (!redirect) + return TC_ACT_OK; + + barrier_data(&idx); + idx = bpf_ntohl(idx); + + __builtin_memset(&zero, 0, sizeof(zero)); + if (bpf_skb_store_bytes(skb, 0, &zero, sizeof(zero), 0) < 0) + return TC_ACT_SHOT; + + return bpf_redirect_neigh(idx, 0); +} + +SEC("src_ingress") int tc_src(struct __sk_buff *skb) +{ + int idx = src_to_dst_tmp; + __u8 zero[ETH_ALEN * 2]; + bool redirect = false; + + switch (skb->protocol) { + case __bpf_constant_htons(ETH_P_IP): + redirect = is_remote_ep_v4(skb, __bpf_constant_htonl(ip4_dst)); + break; + case __bpf_constant_htons(ETH_P_IPV6): + redirect = is_remote_ep_v6(skb, (struct in6_addr)ip6_dst); + break; + } + + if (!redirect) + return TC_ACT_OK; + + barrier_data(&idx); + idx = bpf_ntohl(idx); + + __builtin_memset(&zero, 0, sizeof(zero)); + if (bpf_skb_store_bytes(skb, 0, &zero, sizeof(zero), 0) < 0) + return TC_ACT_SHOT; + + return bpf_redirect_neigh(idx, 0); +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_trace_ext.c b/tools/testing/selftests/bpf/progs/test_trace_ext.c new file mode 100644 index 000000000000..d19a634d0e78 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_trace_ext.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2019 Facebook +#include <linux/bpf.h> +#include <stdbool.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> +#include <bpf/bpf_tracing.h> + +__u64 ext_called = 0; + +SEC("freplace/test_pkt_md_access") +int test_pkt_md_access_new(struct __sk_buff *skb) +{ + ext_called = skb->len; + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_trace_ext_tracing.c b/tools/testing/selftests/bpf/progs/test_trace_ext_tracing.c new file mode 100644 index 000000000000..52f3baf98f20 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_trace_ext_tracing.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +__u64 fentry_called = 0; + +SEC("fentry/test_pkt_md_access_new") +int BPF_PROG(fentry, struct sk_buff *skb) +{ + fentry_called = skb->len; + return 0; +} + +__u64 fexit_called = 0; + +SEC("fexit/test_pkt_md_access_new") +int BPF_PROG(fexit, struct sk_buff *skb) +{ + fexit_called = skb->len; + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index dbb820dde138..238f5f61189e 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -130,6 +130,69 @@ extern int test__join_cgroup(const char *path); #define CHECK_ATTR(condition, tag, format...) \ _CHECK(condition, tag, tattr.duration, format) +#define ASSERT_EQ(actual, expected, name) ({ \ + static int duration = 0; \ + typeof(actual) ___act = (actual); \ + typeof(expected) ___exp = (expected); \ + bool ___ok = ___act == ___exp; \ + CHECK(!___ok, (name), \ + "unexpected %s: actual %lld != expected %lld\n", \ + (name), (long long)(___act), (long long)(___exp)); \ + ___ok; \ +}) + +#define ASSERT_STREQ(actual, expected, name) ({ \ + static int duration = 0; \ + const char *___act = actual; \ + const char *___exp = expected; \ + bool ___ok = strcmp(___act, ___exp) == 0; \ + CHECK(!___ok, (name), \ + "unexpected %s: actual '%s' != expected '%s'\n", \ + (name), ___act, ___exp); \ + ___ok; \ +}) + +#define ASSERT_OK(res, name) ({ \ + static int duration = 0; \ + long long ___res = (res); \ + bool ___ok = ___res == 0; \ + CHECK(!___ok, (name), "unexpected error: %lld\n", ___res); \ + ___ok; \ +}) + +#define ASSERT_ERR(res, name) ({ \ + static int duration = 0; \ + long long ___res = (res); \ + bool ___ok = ___res < 0; \ + CHECK(!___ok, (name), "unexpected success: %lld\n", ___res); \ + ___ok; \ +}) + +#define ASSERT_NULL(ptr, name) ({ \ + static int duration = 0; \ + const void *___res = (ptr); \ + bool ___ok = !___res; \ + CHECK(!___ok, (name), "unexpected pointer: %p\n", ___res); \ + ___ok; \ +}) + +#define ASSERT_OK_PTR(ptr, name) ({ \ + static int duration = 0; \ + const void *___res = (ptr); \ + bool ___ok = !IS_ERR_OR_NULL(___res); \ + CHECK(!___ok, (name), \ + "unexpected error: %ld\n", PTR_ERR(___res)); \ + ___ok; \ +}) + +#define ASSERT_ERR_PTR(ptr, name) ({ \ + static int duration = 0; \ + const void *___res = (ptr); \ + bool ___ok = IS_ERR(___res) \ + CHECK(!___ok, (name), "unexpected pointer: %p\n", ___res); \ + ___ok; \ +}) + static inline __u64 ptr_to_u64(const void *ptr) { return (__u64) (unsigned long) ptr; diff --git a/tools/testing/selftests/bpf/test_sock_fields.c b/tools/testing/selftests/bpf/test_sock_fields.c deleted file mode 100644 index 6c9f269c396d..000000000000 --- a/tools/testing/selftests/bpf/test_sock_fields.c +++ /dev/null @@ -1,482 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2019 Facebook */ - -#include <sys/socket.h> -#include <sys/epoll.h> -#include <netinet/in.h> -#include <arpa/inet.h> -#include <unistd.h> -#include <stdlib.h> -#include <string.h> -#include <errno.h> - -#include <bpf/bpf.h> -#include <bpf/libbpf.h> - -#include "cgroup_helpers.h" -#include "bpf_rlimit.h" - -enum bpf_addr_array_idx { - ADDR_SRV_IDX, - ADDR_CLI_IDX, - __NR_BPF_ADDR_ARRAY_IDX, -}; - -enum bpf_result_array_idx { - EGRESS_SRV_IDX, - EGRESS_CLI_IDX, - INGRESS_LISTEN_IDX, - __NR_BPF_RESULT_ARRAY_IDX, -}; - -enum bpf_linum_array_idx { - EGRESS_LINUM_IDX, - INGRESS_LINUM_IDX, - __NR_BPF_LINUM_ARRAY_IDX, -}; - -struct bpf_spinlock_cnt { - struct bpf_spin_lock lock; - __u32 cnt; -}; - -#define CHECK(condition, tag, format...) ({ \ - int __ret = !!(condition); \ - if (__ret) { \ - printf("%s(%d):FAIL:%s ", __func__, __LINE__, tag); \ - printf(format); \ - printf("\n"); \ - exit(-1); \ - } \ -}) - -#define TEST_CGROUP "/test-bpf-sock-fields" -#define DATA "Hello BPF!" -#define DATA_LEN sizeof(DATA) - -static struct sockaddr_in6 srv_sa6, cli_sa6; -static int sk_pkt_out_cnt10_fd; -static int sk_pkt_out_cnt_fd; -static int linum_map_fd; -static int addr_map_fd; -static int tp_map_fd; -static int sk_map_fd; - -static __u32 addr_srv_idx = ADDR_SRV_IDX; -static __u32 addr_cli_idx = ADDR_CLI_IDX; - -static __u32 egress_srv_idx = EGRESS_SRV_IDX; -static __u32 egress_cli_idx = EGRESS_CLI_IDX; -static __u32 ingress_listen_idx = INGRESS_LISTEN_IDX; - -static __u32 egress_linum_idx = EGRESS_LINUM_IDX; -static __u32 ingress_linum_idx = INGRESS_LINUM_IDX; - -static void init_loopback6(struct sockaddr_in6 *sa6) -{ - memset(sa6, 0, sizeof(*sa6)); - sa6->sin6_family = AF_INET6; - sa6->sin6_addr = in6addr_loopback; -} - -static void print_sk(const struct bpf_sock *sk) -{ - char src_ip4[24], dst_ip4[24]; - char src_ip6[64], dst_ip6[64]; - - inet_ntop(AF_INET, &sk->src_ip4, src_ip4, sizeof(src_ip4)); - inet_ntop(AF_INET6, &sk->src_ip6, src_ip6, sizeof(src_ip6)); - inet_ntop(AF_INET, &sk->dst_ip4, dst_ip4, sizeof(dst_ip4)); - inet_ntop(AF_INET6, &sk->dst_ip6, dst_ip6, sizeof(dst_ip6)); - - printf("state:%u bound_dev_if:%u family:%u type:%u protocol:%u mark:%u priority:%u " - "src_ip4:%x(%s) src_ip6:%x:%x:%x:%x(%s) src_port:%u " - "dst_ip4:%x(%s) dst_ip6:%x:%x:%x:%x(%s) dst_port:%u\n", - sk->state, sk->bound_dev_if, sk->family, sk->type, sk->protocol, - sk->mark, sk->priority, - sk->src_ip4, src_ip4, - sk->src_ip6[0], sk->src_ip6[1], sk->src_ip6[2], sk->src_ip6[3], - src_ip6, sk->src_port, - sk->dst_ip4, dst_ip4, - sk->dst_ip6[0], sk->dst_ip6[1], sk->dst_ip6[2], sk->dst_ip6[3], - dst_ip6, ntohs(sk->dst_port)); -} - -static void print_tp(const struct bpf_tcp_sock *tp) -{ - printf("snd_cwnd:%u srtt_us:%u rtt_min:%u snd_ssthresh:%u rcv_nxt:%u " - "snd_nxt:%u snd:una:%u mss_cache:%u ecn_flags:%u " - "rate_delivered:%u rate_interval_us:%u packets_out:%u " - "retrans_out:%u total_retrans:%u segs_in:%u data_segs_in:%u " - "segs_out:%u data_segs_out:%u lost_out:%u sacked_out:%u " - "bytes_received:%llu bytes_acked:%llu\n", - tp->snd_cwnd, tp->srtt_us, tp->rtt_min, tp->snd_ssthresh, - tp->rcv_nxt, tp->snd_nxt, tp->snd_una, tp->mss_cache, - tp->ecn_flags, tp->rate_delivered, tp->rate_interval_us, - tp->packets_out, tp->retrans_out, tp->total_retrans, - tp->segs_in, tp->data_segs_in, tp->segs_out, - tp->data_segs_out, tp->lost_out, tp->sacked_out, - tp->bytes_received, tp->bytes_acked); -} - -static void check_result(void) -{ - struct bpf_tcp_sock srv_tp, cli_tp, listen_tp; - struct bpf_sock srv_sk, cli_sk, listen_sk; - __u32 ingress_linum, egress_linum; - int err; - - err = bpf_map_lookup_elem(linum_map_fd, &egress_linum_idx, - &egress_linum); - CHECK(err == -1, "bpf_map_lookup_elem(linum_map_fd)", - "err:%d errno:%d", err, errno); - - err = bpf_map_lookup_elem(linum_map_fd, &ingress_linum_idx, - &ingress_linum); - CHECK(err == -1, "bpf_map_lookup_elem(linum_map_fd)", - "err:%d errno:%d", err, errno); - - err = bpf_map_lookup_elem(sk_map_fd, &egress_srv_idx, &srv_sk); - CHECK(err == -1, "bpf_map_lookup_elem(sk_map_fd, &egress_srv_idx)", - "err:%d errno:%d", err, errno); - err = bpf_map_lookup_elem(tp_map_fd, &egress_srv_idx, &srv_tp); - CHECK(err == -1, "bpf_map_lookup_elem(tp_map_fd, &egress_srv_idx)", - "err:%d errno:%d", err, errno); - - err = bpf_map_lookup_elem(sk_map_fd, &egress_cli_idx, &cli_sk); - CHECK(err == -1, "bpf_map_lookup_elem(sk_map_fd, &egress_cli_idx)", - "err:%d errno:%d", err, errno); - err = bpf_map_lookup_elem(tp_map_fd, &egress_cli_idx, &cli_tp); - CHECK(err == -1, "bpf_map_lookup_elem(tp_map_fd, &egress_cli_idx)", - "err:%d errno:%d", err, errno); - - err = bpf_map_lookup_elem(sk_map_fd, &ingress_listen_idx, &listen_sk); - CHECK(err == -1, "bpf_map_lookup_elem(sk_map_fd, &ingress_listen_idx)", - "err:%d errno:%d", err, errno); - err = bpf_map_lookup_elem(tp_map_fd, &ingress_listen_idx, &listen_tp); - CHECK(err == -1, "bpf_map_lookup_elem(tp_map_fd, &ingress_listen_idx)", - "err:%d errno:%d", err, errno); - - printf("listen_sk: "); - print_sk(&listen_sk); - printf("\n"); - - printf("srv_sk: "); - print_sk(&srv_sk); - printf("\n"); - - printf("cli_sk: "); - print_sk(&cli_sk); - printf("\n"); - - printf("listen_tp: "); - print_tp(&listen_tp); - printf("\n"); - - printf("srv_tp: "); - print_tp(&srv_tp); - printf("\n"); - - printf("cli_tp: "); - print_tp(&cli_tp); - printf("\n"); - - CHECK(listen_sk.state != 10 || - listen_sk.family != AF_INET6 || - listen_sk.protocol != IPPROTO_TCP || - memcmp(listen_sk.src_ip6, &in6addr_loopback, - sizeof(listen_sk.src_ip6)) || - listen_sk.dst_ip6[0] || listen_sk.dst_ip6[1] || - listen_sk.dst_ip6[2] || listen_sk.dst_ip6[3] || - listen_sk.src_port != ntohs(srv_sa6.sin6_port) || - listen_sk.dst_port, - "Unexpected listen_sk", - "Check listen_sk output. ingress_linum:%u", - ingress_linum); - - CHECK(srv_sk.state == 10 || - !srv_sk.state || - srv_sk.family != AF_INET6 || - srv_sk.protocol != IPPROTO_TCP || - memcmp(srv_sk.src_ip6, &in6addr_loopback, - sizeof(srv_sk.src_ip6)) || - memcmp(srv_sk.dst_ip6, &in6addr_loopback, - sizeof(srv_sk.dst_ip6)) || - srv_sk.src_port != ntohs(srv_sa6.sin6_port) || - srv_sk.dst_port != cli_sa6.sin6_port, - "Unexpected srv_sk", "Check srv_sk output. egress_linum:%u", - egress_linum); - - CHECK(cli_sk.state == 10 || - !cli_sk.state || - cli_sk.family != AF_INET6 || - cli_sk.protocol != IPPROTO_TCP || - memcmp(cli_sk.src_ip6, &in6addr_loopback, - sizeof(cli_sk.src_ip6)) || - memcmp(cli_sk.dst_ip6, &in6addr_loopback, - sizeof(cli_sk.dst_ip6)) || - cli_sk.src_port != ntohs(cli_sa6.sin6_port) || - cli_sk.dst_port != srv_sa6.sin6_port, - "Unexpected cli_sk", "Check cli_sk output. egress_linum:%u", - egress_linum); - - CHECK(listen_tp.data_segs_out || - listen_tp.data_segs_in || - listen_tp.total_retrans || - listen_tp.bytes_acked, - "Unexpected listen_tp", "Check listen_tp output. ingress_linum:%u", - ingress_linum); - - CHECK(srv_tp.data_segs_out != 2 || - srv_tp.data_segs_in || - srv_tp.snd_cwnd != 10 || - srv_tp.total_retrans || - srv_tp.bytes_acked != 2 * DATA_LEN, - "Unexpected srv_tp", "Check srv_tp output. egress_linum:%u", - egress_linum); - - CHECK(cli_tp.data_segs_out || - cli_tp.data_segs_in != 2 || - cli_tp.snd_cwnd != 10 || - cli_tp.total_retrans || - cli_tp.bytes_received != 2 * DATA_LEN, - "Unexpected cli_tp", "Check cli_tp output. egress_linum:%u", - egress_linum); -} - -static void check_sk_pkt_out_cnt(int accept_fd, int cli_fd) -{ - struct bpf_spinlock_cnt pkt_out_cnt = {}, pkt_out_cnt10 = {}; - int err; - - pkt_out_cnt.cnt = ~0; - pkt_out_cnt10.cnt = ~0; - err = bpf_map_lookup_elem(sk_pkt_out_cnt_fd, &accept_fd, &pkt_out_cnt); - if (!err) - err = bpf_map_lookup_elem(sk_pkt_out_cnt10_fd, &accept_fd, - &pkt_out_cnt10); - - /* The bpf prog only counts for fullsock and - * passive conneciton did not become fullsock until 3WHS - * had been finished. - * The bpf prog only counted two data packet out but we - * specially init accept_fd's pkt_out_cnt by 2 in - * init_sk_storage(). Hence, 4 here. - */ - CHECK(err || pkt_out_cnt.cnt != 4 || pkt_out_cnt10.cnt != 40, - "bpf_map_lookup_elem(sk_pkt_out_cnt, &accept_fd)", - "err:%d errno:%d pkt_out_cnt:%u pkt_out_cnt10:%u", - err, errno, pkt_out_cnt.cnt, pkt_out_cnt10.cnt); - - pkt_out_cnt.cnt = ~0; - pkt_out_cnt10.cnt = ~0; - err = bpf_map_lookup_elem(sk_pkt_out_cnt_fd, &cli_fd, &pkt_out_cnt); - if (!err) - err = bpf_map_lookup_elem(sk_pkt_out_cnt10_fd, &cli_fd, - &pkt_out_cnt10); - /* Active connection is fullsock from the beginning. - * 1 SYN and 1 ACK during 3WHS - * 2 Acks on data packet. - * - * The bpf_prog initialized it to 0xeB9F. - */ - CHECK(err || pkt_out_cnt.cnt != 0xeB9F + 4 || - pkt_out_cnt10.cnt != 0xeB9F + 40, - "bpf_map_lookup_elem(sk_pkt_out_cnt, &cli_fd)", - "err:%d errno:%d pkt_out_cnt:%u pkt_out_cnt10:%u", - err, errno, pkt_out_cnt.cnt, pkt_out_cnt10.cnt); -} - -static void init_sk_storage(int sk_fd, __u32 pkt_out_cnt) -{ - struct bpf_spinlock_cnt scnt = {}; - int err; - - scnt.cnt = pkt_out_cnt; - err = bpf_map_update_elem(sk_pkt_out_cnt_fd, &sk_fd, &scnt, - BPF_NOEXIST); - CHECK(err, "bpf_map_update_elem(sk_pkt_out_cnt_fd)", - "err:%d errno:%d", err, errno); - - scnt.cnt *= 10; - err = bpf_map_update_elem(sk_pkt_out_cnt10_fd, &sk_fd, &scnt, - BPF_NOEXIST); - CHECK(err, "bpf_map_update_elem(sk_pkt_out_cnt10_fd)", - "err:%d errno:%d", err, errno); -} - -static void test(void) -{ - int listen_fd, cli_fd, accept_fd, epfd, err; - struct epoll_event ev; - socklen_t addrlen; - int i; - - addrlen = sizeof(struct sockaddr_in6); - ev.events = EPOLLIN; - - epfd = epoll_create(1); - CHECK(epfd == -1, "epoll_create()", "epfd:%d errno:%d", epfd, errno); - - /* Prepare listen_fd */ - listen_fd = socket(AF_INET6, SOCK_STREAM | SOCK_NONBLOCK, 0); - CHECK(listen_fd == -1, "socket()", "listen_fd:%d errno:%d", - listen_fd, errno); - - init_loopback6(&srv_sa6); - err = bind(listen_fd, (struct sockaddr *)&srv_sa6, sizeof(srv_sa6)); - CHECK(err, "bind(listen_fd)", "err:%d errno:%d", err, errno); - - err = getsockname(listen_fd, (struct sockaddr *)&srv_sa6, &addrlen); - CHECK(err, "getsockname(listen_fd)", "err:%d errno:%d", err, errno); - - err = listen(listen_fd, 1); - CHECK(err, "listen(listen_fd)", "err:%d errno:%d", err, errno); - - /* Prepare cli_fd */ - cli_fd = socket(AF_INET6, SOCK_STREAM | SOCK_NONBLOCK, 0); - CHECK(cli_fd == -1, "socket()", "cli_fd:%d errno:%d", cli_fd, errno); - - init_loopback6(&cli_sa6); - err = bind(cli_fd, (struct sockaddr *)&cli_sa6, sizeof(cli_sa6)); - CHECK(err, "bind(cli_fd)", "err:%d errno:%d", err, errno); - - err = getsockname(cli_fd, (struct sockaddr *)&cli_sa6, &addrlen); - CHECK(err, "getsockname(cli_fd)", "err:%d errno:%d", - err, errno); - - /* Update addr_map with srv_sa6 and cli_sa6 */ - err = bpf_map_update_elem(addr_map_fd, &addr_srv_idx, &srv_sa6, 0); - CHECK(err, "map_update", "err:%d errno:%d", err, errno); - - err = bpf_map_update_elem(addr_map_fd, &addr_cli_idx, &cli_sa6, 0); - CHECK(err, "map_update", "err:%d errno:%d", err, errno); - - /* Connect from cli_sa6 to srv_sa6 */ - err = connect(cli_fd, (struct sockaddr *)&srv_sa6, addrlen); - printf("srv_sa6.sin6_port:%u cli_sa6.sin6_port:%u\n\n", - ntohs(srv_sa6.sin6_port), ntohs(cli_sa6.sin6_port)); - CHECK(err && errno != EINPROGRESS, - "connect(cli_fd)", "err:%d errno:%d", err, errno); - - ev.data.fd = listen_fd; - err = epoll_ctl(epfd, EPOLL_CTL_ADD, listen_fd, &ev); - CHECK(err, "epoll_ctl(EPOLL_CTL_ADD, listen_fd)", "err:%d errno:%d", - err, errno); - - /* Accept the connection */ - /* Have some timeout in accept(listen_fd). Just in case. */ - err = epoll_wait(epfd, &ev, 1, 1000); - CHECK(err != 1 || ev.data.fd != listen_fd, - "epoll_wait(listen_fd)", - "err:%d errno:%d ev.data.fd:%d listen_fd:%d", - err, errno, ev.data.fd, listen_fd); - - accept_fd = accept(listen_fd, NULL, NULL); - CHECK(accept_fd == -1, "accept(listen_fd)", "accept_fd:%d errno:%d", - accept_fd, errno); - close(listen_fd); - - ev.data.fd = cli_fd; - err = epoll_ctl(epfd, EPOLL_CTL_ADD, cli_fd, &ev); - CHECK(err, "epoll_ctl(EPOLL_CTL_ADD, cli_fd)", "err:%d errno:%d", - err, errno); - - init_sk_storage(accept_fd, 2); - - for (i = 0; i < 2; i++) { - /* Send some data from accept_fd to cli_fd */ - err = send(accept_fd, DATA, DATA_LEN, 0); - CHECK(err != DATA_LEN, "send(accept_fd)", "err:%d errno:%d", - err, errno); - - /* Have some timeout in recv(cli_fd). Just in case. */ - err = epoll_wait(epfd, &ev, 1, 1000); - CHECK(err != 1 || ev.data.fd != cli_fd, - "epoll_wait(cli_fd)", "err:%d errno:%d ev.data.fd:%d cli_fd:%d", - err, errno, ev.data.fd, cli_fd); - - err = recv(cli_fd, NULL, 0, MSG_TRUNC); - CHECK(err, "recv(cli_fd)", "err:%d errno:%d", err, errno); - } - - check_sk_pkt_out_cnt(accept_fd, cli_fd); - - close(epfd); - close(accept_fd); - close(cli_fd); - - check_result(); -} - -int main(int argc, char **argv) -{ - struct bpf_prog_load_attr attr = { - .file = "test_sock_fields_kern.o", - .prog_type = BPF_PROG_TYPE_CGROUP_SKB, - .prog_flags = BPF_F_TEST_RND_HI32, - }; - int cgroup_fd, egress_fd, ingress_fd, err; - struct bpf_program *ingress_prog; - struct bpf_object *obj; - struct bpf_map *map; - - /* Create a cgroup, get fd, and join it */ - cgroup_fd = cgroup_setup_and_join(TEST_CGROUP); - CHECK(cgroup_fd < 0, "cgroup_setup_and_join()", - "cgroup_fd:%d errno:%d", cgroup_fd, errno); - atexit(cleanup_cgroup_environment); - - err = bpf_prog_load_xattr(&attr, &obj, &egress_fd); - CHECK(err, "bpf_prog_load_xattr()", "err:%d", err); - - ingress_prog = bpf_object__find_program_by_title(obj, - "cgroup_skb/ingress"); - CHECK(!ingress_prog, - "bpf_object__find_program_by_title(cgroup_skb/ingress)", - "not found"); - ingress_fd = bpf_program__fd(ingress_prog); - - err = bpf_prog_attach(egress_fd, cgroup_fd, BPF_CGROUP_INET_EGRESS, 0); - CHECK(err == -1, "bpf_prog_attach(CPF_CGROUP_INET_EGRESS)", - "err:%d errno%d", err, errno); - - err = bpf_prog_attach(ingress_fd, cgroup_fd, - BPF_CGROUP_INET_INGRESS, 0); - CHECK(err == -1, "bpf_prog_attach(CPF_CGROUP_INET_INGRESS)", - "err:%d errno%d", err, errno); - close(cgroup_fd); - - map = bpf_object__find_map_by_name(obj, "addr_map"); - CHECK(!map, "cannot find addr_map", "(null)"); - addr_map_fd = bpf_map__fd(map); - - map = bpf_object__find_map_by_name(obj, "sock_result_map"); - CHECK(!map, "cannot find sock_result_map", "(null)"); - sk_map_fd = bpf_map__fd(map); - - map = bpf_object__find_map_by_name(obj, "tcp_sock_result_map"); - CHECK(!map, "cannot find tcp_sock_result_map", "(null)"); - tp_map_fd = bpf_map__fd(map); - - map = bpf_object__find_map_by_name(obj, "linum_map"); - CHECK(!map, "cannot find linum_map", "(null)"); - linum_map_fd = bpf_map__fd(map); - - map = bpf_object__find_map_by_name(obj, "sk_pkt_out_cnt"); - CHECK(!map, "cannot find sk_pkt_out_cnt", "(null)"); - sk_pkt_out_cnt_fd = bpf_map__fd(map); - - map = bpf_object__find_map_by_name(obj, "sk_pkt_out_cnt10"); - CHECK(!map, "cannot find sk_pkt_out_cnt10", "(null)"); - sk_pkt_out_cnt10_fd = bpf_map__fd(map); - - test(); - - bpf_object__close(obj); - cleanup_cgroup_environment(); - - printf("PASS\n"); - - return 0; -} diff --git a/tools/testing/selftests/bpf/test_tc_neigh.sh b/tools/testing/selftests/bpf/test_tc_neigh.sh new file mode 100755 index 000000000000..31d8c3df8b24 --- /dev/null +++ b/tools/testing/selftests/bpf/test_tc_neigh.sh @@ -0,0 +1,168 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# This test sets up 3 netns (src <-> fwd <-> dst). There is no direct veth link +# between src and dst. The netns fwd has veth links to each src and dst. The +# client is in src and server in dst. The test installs a TC BPF program to each +# host facing veth in fwd which calls into bpf_redirect_peer() to perform the +# neigh addr population and redirect; it also installs a dropper prog on the +# egress side to drop skbs if neigh addrs were not populated. + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" + echo "FAIL" + exit 1 +fi + +# check that nc, dd, ping, ping6 and timeout are present +command -v nc >/dev/null 2>&1 || \ + { echo >&2 "nc is not available"; exit 1; } +command -v dd >/dev/null 2>&1 || \ + { echo >&2 "dd is not available"; exit 1; } +command -v timeout >/dev/null 2>&1 || \ + { echo >&2 "timeout is not available"; exit 1; } +command -v ping >/dev/null 2>&1 || \ + { echo >&2 "ping is not available"; exit 1; } +command -v ping6 >/dev/null 2>&1 || \ + { echo >&2 "ping6 is not available"; exit 1; } + +readonly GREEN='\033[0;92m' +readonly RED='\033[0;31m' +readonly NC='\033[0m' # No Color + +readonly PING_ARG="-c 3 -w 10 -q" + +readonly TIMEOUT=10 + +readonly NS_SRC="ns-src-$(mktemp -u XXXXXX)" +readonly NS_FWD="ns-fwd-$(mktemp -u XXXXXX)" +readonly NS_DST="ns-dst-$(mktemp -u XXXXXX)" + +readonly IP4_SRC="172.16.1.100" +readonly IP4_DST="172.16.2.100" + +readonly IP6_SRC="::1:dead:beef:cafe" +readonly IP6_DST="::2:dead:beef:cafe" + +readonly IP4_SLL="169.254.0.1" +readonly IP4_DLL="169.254.0.2" +readonly IP4_NET="169.254.0.0" + +cleanup() +{ + ip netns del ${NS_SRC} + ip netns del ${NS_FWD} + ip netns del ${NS_DST} +} + +trap cleanup EXIT + +set -e + +ip netns add "${NS_SRC}" +ip netns add "${NS_FWD}" +ip netns add "${NS_DST}" + +ip link add veth_src type veth peer name veth_src_fwd +ip link add veth_dst type veth peer name veth_dst_fwd + +ip link set veth_src netns ${NS_SRC} +ip link set veth_src_fwd netns ${NS_FWD} + +ip link set veth_dst netns ${NS_DST} +ip link set veth_dst_fwd netns ${NS_FWD} + +ip -netns ${NS_SRC} addr add ${IP4_SRC}/32 dev veth_src +ip -netns ${NS_DST} addr add ${IP4_DST}/32 dev veth_dst + +# The fwd netns automatically get a v6 LL address / routes, but also needs v4 +# one in order to start ARP probing. IP4_NET route is added to the endpoints +# so that the ARP processing will reply. + +ip -netns ${NS_FWD} addr add ${IP4_SLL}/32 dev veth_src_fwd +ip -netns ${NS_FWD} addr add ${IP4_DLL}/32 dev veth_dst_fwd + +ip -netns ${NS_SRC} addr add ${IP6_SRC}/128 dev veth_src nodad +ip -netns ${NS_DST} addr add ${IP6_DST}/128 dev veth_dst nodad + +ip -netns ${NS_SRC} link set dev veth_src up +ip -netns ${NS_FWD} link set dev veth_src_fwd up + +ip -netns ${NS_DST} link set dev veth_dst up +ip -netns ${NS_FWD} link set dev veth_dst_fwd up + +ip -netns ${NS_SRC} route add ${IP4_DST}/32 dev veth_src scope global +ip -netns ${NS_SRC} route add ${IP4_NET}/16 dev veth_src scope global +ip -netns ${NS_FWD} route add ${IP4_SRC}/32 dev veth_src_fwd scope global + +ip -netns ${NS_SRC} route add ${IP6_DST}/128 dev veth_src scope global +ip -netns ${NS_FWD} route add ${IP6_SRC}/128 dev veth_src_fwd scope global + +ip -netns ${NS_DST} route add ${IP4_SRC}/32 dev veth_dst scope global +ip -netns ${NS_DST} route add ${IP4_NET}/16 dev veth_dst scope global +ip -netns ${NS_FWD} route add ${IP4_DST}/32 dev veth_dst_fwd scope global + +ip -netns ${NS_DST} route add ${IP6_SRC}/128 dev veth_dst scope global +ip -netns ${NS_FWD} route add ${IP6_DST}/128 dev veth_dst_fwd scope global + +fmac_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/address) +fmac_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/address) + +ip -netns ${NS_SRC} neigh add ${IP4_DST} dev veth_src lladdr $fmac_src +ip -netns ${NS_DST} neigh add ${IP4_SRC} dev veth_dst lladdr $fmac_dst + +ip -netns ${NS_SRC} neigh add ${IP6_DST} dev veth_src lladdr $fmac_src +ip -netns ${NS_DST} neigh add ${IP6_SRC} dev veth_dst lladdr $fmac_dst + +veth_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/ifindex | awk '{printf "%08x\n", $1}') +veth_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/ifindex | awk '{printf "%08x\n", $1}') + +xxd -p < test_tc_neigh.o | sed "s/eeddddee/$veth_src/g" | xxd -r -p > test_tc_neigh.x.o +xxd -p < test_tc_neigh.x.o | sed "s/eeffffee/$veth_dst/g" | xxd -r -p > test_tc_neigh.y.o + +ip netns exec ${NS_FWD} tc qdisc add dev veth_src_fwd clsact +ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd ingress bpf da obj test_tc_neigh.y.o sec src_ingress +ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd egress bpf da obj test_tc_neigh.y.o sec chk_neigh + +ip netns exec ${NS_FWD} tc qdisc add dev veth_dst_fwd clsact +ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd ingress bpf da obj test_tc_neigh.y.o sec dst_ingress +ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd egress bpf da obj test_tc_neigh.y.o sec chk_neigh + +rm -f test_tc_neigh.x.o test_tc_neigh.y.o + +ip netns exec ${NS_DST} bash -c "nc -4 -l -p 9004 &" +ip netns exec ${NS_DST} bash -c "nc -6 -l -p 9006 &" + +set +e + +TEST="TCPv4 connectivity test" +ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP4_DST}/9004" +if [ $? -ne 0 ]; then + echo -e "${TEST}: ${RED}FAIL${NC}" + exit 1 +fi +echo -e "${TEST}: ${GREEN}PASS${NC}" + +TEST="TCPv6 connectivity test" +ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP6_DST}/9006" +if [ $? -ne 0 ]; then + echo -e "${TEST}: ${RED}FAIL${NC}" + exit 1 +fi +echo -e "${TEST}: ${GREEN}PASS${NC}" + +TEST="ICMPv4 connectivity test" +ip netns exec ${NS_SRC} ping $PING_ARG ${IP4_DST} +if [ $? -ne 0 ]; then + echo -e "${TEST}: ${RED}FAIL${NC}" + exit 1 +fi +echo -e "${TEST}: ${GREEN}PASS${NC}" + +TEST="ICMPv6 connectivity test" +ip netns exec ${NS_SRC} ping6 $PING_ARG ${IP6_DST} +if [ $? -ne 0 ]; then + echo -e "${TEST}: ${RED}FAIL${NC}" + exit 1 +fi +echo -e "${TEST}: ${GREEN}PASS${NC}" diff --git a/tools/testing/selftests/bpf/verifier/and.c b/tools/testing/selftests/bpf/verifier/and.c index d781bc86e100..ca8fdb1b3f01 100644 --- a/tools/testing/selftests/bpf/verifier/and.c +++ b/tools/testing/selftests/bpf/verifier/and.c @@ -48,3 +48,19 @@ .result = REJECT, .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, +{ + "check known subreg with unknown reg", + .insns = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_prandom_u32), + BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 32), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1), + BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFFFF1234), + /* Upper bits are unknown but AND above masks out 1 zero'ing lower bits */ + BPF_JMP32_IMM(BPF_JLT, BPF_REG_0, 1, 1), + BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1, 512), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .result = ACCEPT, + .retval = 0 +}, diff --git a/tools/testing/selftests/bpf/verifier/ref_tracking.c b/tools/testing/selftests/bpf/verifier/ref_tracking.c index 056e0273bf12..006b5bd99c08 100644 --- a/tools/testing/selftests/bpf/verifier/ref_tracking.c +++ b/tools/testing/selftests/bpf/verifier/ref_tracking.c @@ -854,3 +854,50 @@ .errstr = "Unreleased reference", .result = REJECT, }, +{ + "reference tracking: bpf_sk_release(btf_tcp_sock)", + .insns = { + BPF_SK_LOOKUP(sk_lookup_tcp), + BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_6, BPF_REG_0), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), + BPF_EMIT_CALL(BPF_FUNC_skc_to_tcp_sock), + BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), + BPF_EMIT_CALL(BPF_FUNC_sk_release), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), + BPF_EMIT_CALL(BPF_FUNC_sk_release), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_SCHED_CLS, + .result = ACCEPT, + .result_unpriv = REJECT, + .errstr_unpriv = "unknown func", +}, +{ + "reference tracking: use ptr from bpf_skc_to_tcp_sock() after release", + .insns = { + BPF_SK_LOOKUP(sk_lookup_tcp), + BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_6, BPF_REG_0), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), + BPF_EMIT_CALL(BPF_FUNC_skc_to_tcp_sock), + BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), + BPF_EMIT_CALL(BPF_FUNC_sk_release), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_7, BPF_REG_0), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), + BPF_EMIT_CALL(BPF_FUNC_sk_release), + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_7, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_SCHED_CLS, + .result = REJECT, + .errstr = "invalid mem access", + .result_unpriv = REJECT, + .errstr_unpriv = "unknown func", +}, diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh index 6d1790b5de7a..e9f8718af979 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh @@ -147,17 +147,26 @@ switch_create() # Make sure that ingress quotas are smaller than egress so that there is # room for both streams of traffic to be admitted to shared buffer. + devlink_pool_size_thtype_save 0 devlink_pool_size_thtype_set 0 dynamic 10000000 + devlink_pool_size_thtype_save 4 devlink_pool_size_thtype_set 4 dynamic 10000000 + devlink_port_pool_th_save $swp1 0 devlink_port_pool_th_set $swp1 0 6 + devlink_tc_bind_pool_th_save $swp1 1 ingress devlink_tc_bind_pool_th_set $swp1 1 ingress 0 6 + devlink_port_pool_th_save $swp2 0 devlink_port_pool_th_set $swp2 0 6 + devlink_tc_bind_pool_th_save $swp2 2 ingress devlink_tc_bind_pool_th_set $swp2 2 ingress 0 6 + devlink_tc_bind_pool_th_save $swp3 1 egress devlink_tc_bind_pool_th_set $swp3 1 egress 4 7 + devlink_tc_bind_pool_th_save $swp3 2 egress devlink_tc_bind_pool_th_set $swp3 2 egress 4 7 + devlink_port_pool_th_save $swp3 4 devlink_port_pool_th_set $swp3 4 7 } diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_headroom.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_headroom.sh new file mode 100755 index 000000000000..27de3d9ed08e --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_headroom.sh @@ -0,0 +1,379 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +ALL_TESTS=" + test_defaults + test_dcb_ets + test_mtu + test_pfc + test_int_buf + test_tc_priomap + test_tc_mtu + test_tc_sizes + test_tc_int_buf +" + +lib_dir=$(dirname $0)/../../../net/forwarding + +NUM_NETIFS=0 +source $lib_dir/lib.sh +source $lib_dir/devlink_lib.sh +source qos_lib.sh + +swp=$NETIF_NO_CABLE + +cleanup() +{ + pre_cleanup +} + +get_prio_pg() +{ + __mlnx_qos -i $swp | sed -n '/^PFC/,/^[^[:space:]]/p' | + grep buffer | sed 's/ \+/ /g' | cut -d' ' -f 2- +} + +get_prio_pfc() +{ + __mlnx_qos -i $swp | sed -n '/^PFC/,/^[^[:space:]]/p' | + grep enabled | sed 's/ \+/ /g' | cut -d' ' -f 2- +} + +get_prio_tc() +{ + __mlnx_qos -i $swp | sed -n '/^tc/,$p' | + awk '/^tc/ { TC = $2 } + /priority:/ { PRIO[$2]=TC } + END { + for (i in PRIO) + printf("%d ", PRIO[i]) + }' +} + +get_buf_size() +{ + local idx=$1; shift + + __mlnx_qos -i $swp | grep Receive | sed 's/.*: //' | cut -d, -f $((idx + 1)) +} + +get_tot_size() +{ + __mlnx_qos -i $swp | grep Receive | sed 's/.*total_size=//' +} + +check_prio_pg() +{ + local expect=$1; shift + + local current=$(get_prio_pg) + test "$current" = "$expect" + check_err $? "prio2buffer is '$current', expected '$expect'" +} + +check_prio_pfc() +{ + local expect=$1; shift + + local current=$(get_prio_pfc) + test "$current" = "$expect" + check_err $? "prio PFC is '$current', expected '$expect'" +} + +check_prio_tc() +{ + local expect=$1; shift + + local current=$(get_prio_tc) + test "$current" = "$expect" + check_err $? "prio_tc is '$current', expected '$expect'" +} + +__check_buf_size() +{ + local idx=$1; shift + local expr=$1; shift + local what=$1; shift + + local current=$(get_buf_size $idx) + ((current $expr)) + check_err $? "${what}buffer $idx size is '$current', expected '$expr'" + echo $current +} + +check_buf_size() +{ + __check_buf_size "$@" > /dev/null +} + +test_defaults() +{ + RET=0 + + check_prio_pg "0 0 0 0 0 0 0 0 " + check_prio_tc "0 0 0 0 0 0 0 0 " + check_prio_pfc "0 0 0 0 0 0 0 0 " + + log_test "Default headroom configuration" +} + +test_dcb_ets() +{ + RET=0 + + __mlnx_qos -i $swp --prio_tc=0,2,4,6,1,3,5,7 > /dev/null + + check_prio_pg "0 2 4 6 1 3 5 7 " + check_prio_tc "0 2 4 6 1 3 5 7 " + check_prio_pfc "0 0 0 0 0 0 0 0 " + + __mlnx_qos -i $swp --prio_tc=0,0,0,0,0,0,0,0 > /dev/null + + check_prio_pg "0 0 0 0 0 0 0 0 " + check_prio_tc "0 0 0 0 0 0 0 0 " + + __mlnx_qos -i $swp --prio2buffer=1,3,5,7,0,2,4,6 &> /dev/null + check_fail $? "prio2buffer accepted in DCB mode" + + log_test "Configuring headroom through ETS" +} + +test_mtu() +{ + local what=$1; shift + local buf0size_2 + local buf0size + + RET=0 + buf0size=$(__check_buf_size 0 "> 0") + + mtu_set $swp 3000 + buf0size_2=$(__check_buf_size 0 "> $buf0size" "MTU 3000: ") + mtu_restore $swp + + mtu_set $swp 6000 + check_buf_size 0 "> $buf0size_2" "MTU 6000: " + mtu_restore $swp + + check_buf_size 0 "== $buf0size" + + log_test "${what}MTU impacts buffer size" +} + +test_tc_mtu() +{ + # In TC mode, MTU still impacts the threshold below which a buffer is + # not permitted to go. + + tc qdisc replace dev $swp root handle 1: bfifo limit 1.5M + test_mtu "TC: " + tc qdisc delete dev $swp root +} + +test_pfc() +{ + RET=0 + + __mlnx_qos -i $swp --prio_tc=0,0,0,0,0,1,2,3 > /dev/null + + local buf0size=$(get_buf_size 0) + local buf1size=$(get_buf_size 1) + local buf2size=$(get_buf_size 2) + local buf3size=$(get_buf_size 3) + check_buf_size 0 "> 0" + check_buf_size 1 "> 0" + check_buf_size 2 "> 0" + check_buf_size 3 "> 0" + check_buf_size 4 "== 0" + check_buf_size 5 "== 0" + check_buf_size 6 "== 0" + check_buf_size 7 "== 0" + + log_test "Buffer size sans PFC" + + RET=0 + + __mlnx_qos -i $swp --pfc=0,0,0,0,0,1,1,1 --cable_len=0 > /dev/null + + check_prio_pg "0 0 0 0 0 1 2 3 " + check_prio_pfc "0 0 0 0 0 1 1 1 " + check_buf_size 0 "== $buf0size" + check_buf_size 1 "> $buf1size" + check_buf_size 2 "> $buf2size" + check_buf_size 3 "> $buf3size" + + local buf1size=$(get_buf_size 1) + check_buf_size 2 "== $buf1size" + check_buf_size 3 "== $buf1size" + + log_test "PFC: Cable length 0" + + RET=0 + + __mlnx_qos -i $swp --pfc=0,0,0,0,0,1,1,1 --cable_len=1000 > /dev/null + + check_buf_size 0 "== $buf0size" + check_buf_size 1 "> $buf1size" + check_buf_size 2 "> $buf1size" + check_buf_size 3 "> $buf1size" + + log_test "PFC: Cable length 1000" + + RET=0 + + __mlnx_qos -i $swp --pfc=0,0,0,0,0,0,0,0 --cable_len=0 > /dev/null + __mlnx_qos -i $swp --prio_tc=0,0,0,0,0,0,0,0 > /dev/null + + check_prio_pg "0 0 0 0 0 0 0 0 " + check_prio_tc "0 0 0 0 0 0 0 0 " + check_buf_size 0 "> 0" + check_buf_size 1 "== 0" + check_buf_size 2 "== 0" + check_buf_size 3 "== 0" + check_buf_size 4 "== 0" + check_buf_size 5 "== 0" + check_buf_size 6 "== 0" + check_buf_size 7 "== 0" + + log_test "PFC: Restore defaults" +} + +test_tc_priomap() +{ + RET=0 + + __mlnx_qos -i $swp --prio_tc=0,1,2,3,4,5,6,7 > /dev/null + check_prio_pg "0 1 2 3 4 5 6 7 " + + tc qdisc replace dev $swp root handle 1: bfifo limit 1.5M + check_prio_pg "0 0 0 0 0 0 0 0 " + + __mlnx_qos -i $swp --prio2buffer=1,3,5,7,0,2,4,6 > /dev/null + check_prio_pg "1 3 5 7 0 2 4 6 " + + tc qdisc delete dev $swp root + check_prio_pg "0 1 2 3 4 5 6 7 " + + # Clean up. + tc qdisc replace dev $swp root handle 1: bfifo limit 1.5M + __mlnx_qos -i $swp --prio2buffer=0,0,0,0,0,0,0,0 > /dev/null + tc qdisc delete dev $swp root + __mlnx_qos -i $swp --prio_tc=0,0,0,0,0,0,0,0 > /dev/null + + log_test "TC: priomap" +} + +test_tc_sizes() +{ + local cell_size=$(devlink_cell_size_get) + local size=$((cell_size * 1000)) + + RET=0 + + __mlnx_qos -i $swp --buffer_size=$size,0,0,0,0,0,0,0 &> /dev/null + check_fail $? "buffer_size should fail before qdisc is added" + + tc qdisc replace dev $swp root handle 1: bfifo limit 1.5M + + __mlnx_qos -i $swp --buffer_size=$size,0,0,0,0,0,0,0 > /dev/null + check_err $? "buffer_size should pass after qdisc is added" + check_buf_size 0 "== $size" "set size: " + + mtu_set $swp 6000 + check_buf_size 0 "== $size" "set MTU: " + mtu_restore $swp + + __mlnx_qos -i $swp --buffer_size=0,0,0,0,0,0,0,0 > /dev/null + + # After replacing the qdisc for the same kind, buffer_size still has to + # work. + tc qdisc replace dev $swp root handle 1: bfifo limit 1M + + __mlnx_qos -i $swp --buffer_size=$size,0,0,0,0,0,0,0 > /dev/null + check_buf_size 0 "== $size" "post replace, set size: " + + __mlnx_qos -i $swp --buffer_size=0,0,0,0,0,0,0,0 > /dev/null + + # Likewise after replacing for a different kind. + tc qdisc replace dev $swp root handle 2: prio bands 8 + + __mlnx_qos -i $swp --buffer_size=$size,0,0,0,0,0,0,0 > /dev/null + check_buf_size 0 "== $size" "post replace different kind, set size: " + + tc qdisc delete dev $swp root + + __mlnx_qos -i $swp --buffer_size=$size,0,0,0,0,0,0,0 &> /dev/null + check_fail $? "buffer_size should fail after qdisc is deleted" + + log_test "TC: buffer size" +} + +test_int_buf() +{ + local what=$1; shift + + RET=0 + + local buf0size=$(get_buf_size 0) + local tot_size=$(get_tot_size) + + # Size of internal buffer and buffer 9. + local dsize=$((tot_size - buf0size)) + + tc qdisc add dev $swp clsact + tc filter add dev $swp egress matchall skip_sw action mirred egress mirror dev $swp + + local buf0size_2=$(get_buf_size 0) + local tot_size_2=$(get_tot_size) + local dsize_2=$((tot_size_2 - buf0size_2)) + + # Egress SPAN should have added to the "invisible" buffer configuration. + ((dsize_2 > dsize)) + check_err $? "Invisible buffers account for '$dsize_2', expected '> $dsize'" + + mtu_set $swp 3000 + + local buf0size_3=$(get_buf_size 0) + local tot_size_3=$(get_tot_size) + local dsize_3=$((tot_size_3 - buf0size_3)) + + # MTU change might change buffer 0, which will show at total, but the + # hidden buffers should stay the same size. + ((dsize_3 == dsize_2)) + check_err $? "MTU change: Invisible buffers account for '$dsize_3', expected '== $dsize_2'" + + mtu_restore $swp + tc qdisc del dev $swp clsact + + # After SPAN removal, hidden buffers should be back to the original sizes. + local buf0size_4=$(get_buf_size 0) + local tot_size_4=$(get_tot_size) + local dsize_4=$((tot_size_4 - buf0size_4)) + ((dsize_4 == dsize)) + check_err $? "SPAN removed: Invisible buffers account for '$dsize_4', expected '== $dsize'" + + log_test "${what}internal buffer size" +} + +test_tc_int_buf() +{ + local cell_size=$(devlink_cell_size_get) + local size=$((cell_size * 1000)) + + tc qdisc replace dev $swp root handle 1: bfifo limit 1.5M + test_int_buf "TC: " + + __mlnx_qos -i $swp --buffer_size=$size,0,0,0,0,0,0,0 > /dev/null + test_int_buf "TC+buffsize: " + + __mlnx_qos -i $swp --buffer_size=0,0,0,0,0,0,0,0 > /dev/null + tc qdisc delete dev $swp root +} + +trap cleanup EXIT + +bail_on_lldpad +setup_wait +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_lib.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_lib.sh index faa51012cdac..0bf76f13c030 100644 --- a/tools/testing/selftests/drivers/net/mlxsw/qos_lib.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_lib.sh @@ -82,3 +82,17 @@ bail_on_lldpad() fi fi } + +__mlnx_qos() +{ + local err + + mlnx_qos "$@" 2>/dev/null + err=$? + + if ((err)); then + echo "Error ($err) in mlnx_qos $@" >/dev/stderr + fi + + return $err +} diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh index b025daea062d..8f164c80e215 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh @@ -145,12 +145,17 @@ switch_create() # Make sure that ingress quotas are smaller than egress so that there is # room for both streams of traffic to be admitted to shared buffer. + devlink_port_pool_th_save $swp1 0 devlink_port_pool_th_set $swp1 0 5 + devlink_tc_bind_pool_th_save $swp1 0 ingress devlink_tc_bind_pool_th_set $swp1 0 ingress 0 5 + devlink_port_pool_th_save $swp2 0 devlink_port_pool_th_set $swp2 0 5 + devlink_tc_bind_pool_th_save $swp2 1 ingress devlink_tc_bind_pool_th_set $swp2 1 ingress 0 5 + devlink_port_pool_th_save $swp3 4 devlink_port_pool_th_set $swp3 4 12 } diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_pfc.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_pfc.sh new file mode 100755 index 000000000000..4d900bc1f76c --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_pfc.sh @@ -0,0 +1,403 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# This test injects a 10-MB burst of traffic with VLAN tag and 802.1p priority +# of 1. This stream is consistently prioritized as priority 1, is put to PG +# buffer 1, and scheduled at TC 1. +# +# - the stream first ingresses through $swp1, where it is forwarded to $swp3 +# +# - then it ingresses through $swp4. Here it is put to a lossless buffer and put +# to a small pool ("PFC pool"). The traffic is forwarded to $swp2, which is +# shaped, and thus the PFC pool eventually fills, therefore the headroom +# fills, and $swp3 is paused. +# +# - since $swp3 now can't send traffic, the traffic ingressing $swp1 is kept at +# a pool ("overflow pool"). The overflow pool needs to be large enough to +# contain the whole burst. +# +# - eventually the PFC pool gets some traffic out, headroom therefore gets some +# traffic to the pool, and $swp3 is unpaused again. This way the traffic is +# gradually forwarded from the overflow pool, through the PFC pool, out of +# $swp2, and eventually to $h2. +# +# - if PFC works, all lossless flow packets that ingress through $swp1 should +# also be seen ingressing $h2. If it doesn't, there will be drops due to +# discrepancy between the speeds of $swp1 and $h2. +# +# - it should all play out relatively quickly, so that SLL and HLL will not +# cause drops. +# +# +-----------------------+ +# | H1 | +# | + $h1.111 | +# | | 192.0.2.33/28 | +# | | | +# | + $h1 | +# +---|-------------------+ +--------------------+ +# | | | +# +---|----------------------|--------------------|---------------------------+ +# | + $swp1 $swp3 + + $swp4 | +# | | iPOOL1 iPOOL0 | | iPOOL2 | +# | | ePOOL4 ePOOL5 | | ePOOL4 | +# | | 1Gbps | | 1Gbps | +# | | PFC:enabled=1 | | PFC:enabled=1 | +# | +-|----------------------|-+ +-|------------------------+ | +# | | + $swp1.111 $swp3.111 + | | + $swp4.111 | | +# | | | | | | +# | | BR1 | | BR2 | | +# | | | | | | +# | | | | + $swp2.111 | | +# | +--------------------------+ +---------|----------------+ | +# | | | +# | iPOOL0: 500KB dynamic | | +# | iPOOL1: 10MB static | | +# | iPOOL2: 1MB static + $swp2 | +# | ePOOL4: 500KB dynamic | iPOOL0 | +# | ePOOL5: 10MB static | ePOOL6 | +# | ePOOL6: "infinite" static | 200Mbps shaper | +# +-------------------------------------------------------|-------------------+ +# | +# +---|-------------------+ +# | + $h2 H2 | +# | | | +# | + $h2.111 | +# | 192.0.2.34/28 | +# +-----------------------+ +# +# iPOOL0+ePOOL4 is a helper pool for control traffic etc. +# iPOOL1+ePOOL5 are overflow pools. +# iPOOL2+ePOOL6 are PFC pools. + +ALL_TESTS=" + ping_ipv4 + test_qos_pfc +" + +lib_dir=$(dirname $0)/../../../net/forwarding + +NUM_NETIFS=6 +source $lib_dir/lib.sh +source $lib_dir/devlink_lib.sh +source qos_lib.sh + +_1KB=1000 +_100KB=$((100 * _1KB)) +_500KB=$((500 * _1KB)) +_1MB=$((1000 * _1KB)) +_10MB=$((10 * _1MB)) + +h1_create() +{ + simple_if_init $h1 + mtu_set $h1 10000 + + vlan_create $h1 111 v$h1 192.0.2.33/28 +} + +h1_destroy() +{ + vlan_destroy $h1 111 + + mtu_restore $h1 + simple_if_fini $h1 +} + +h2_create() +{ + simple_if_init $h2 + mtu_set $h2 10000 + + vlan_create $h2 111 v$h2 192.0.2.34/28 +} + +h2_destroy() +{ + vlan_destroy $h2 111 + + mtu_restore $h2 + simple_if_fini $h2 +} + +switch_create() +{ + # pools + # ----- + + devlink_pool_size_thtype_save 0 + devlink_pool_size_thtype_save 4 + devlink_pool_size_thtype_save 1 + devlink_pool_size_thtype_save 5 + devlink_pool_size_thtype_save 2 + devlink_pool_size_thtype_save 6 + + devlink_port_pool_th_save $swp1 1 + devlink_port_pool_th_save $swp2 6 + devlink_port_pool_th_save $swp3 5 + devlink_port_pool_th_save $swp4 2 + + devlink_tc_bind_pool_th_save $swp1 1 ingress + devlink_tc_bind_pool_th_save $swp2 1 egress + devlink_tc_bind_pool_th_save $swp3 1 egress + devlink_tc_bind_pool_th_save $swp4 1 ingress + + # Control traffic pools. Just reduce the size. Keep them dynamic so that + # we don't need to change all the uninteresting quotas. + devlink_pool_size_thtype_set 0 dynamic $_500KB + devlink_pool_size_thtype_set 4 dynamic $_500KB + + # Overflow pools. + devlink_pool_size_thtype_set 1 static $_10MB + devlink_pool_size_thtype_set 5 static $_10MB + + # PFC pools. As per the writ, the size of egress PFC pool should be + # infinice, but actually it just needs to be large enough to not matter + # in practice, so reuse the 10MB limit. + devlink_pool_size_thtype_set 2 static $_1MB + devlink_pool_size_thtype_set 6 static $_10MB + + # $swp1 + # ----- + + ip link set dev $swp1 up + mtu_set $swp1 10000 + vlan_create $swp1 111 + ip link set dev $swp1.111 type vlan ingress-qos-map 0:0 1:1 + + devlink_port_pool_th_set $swp1 1 $_10MB + devlink_tc_bind_pool_th_set $swp1 1 ingress 1 $_10MB + + # Configure qdisc so that we can configure PG and therefore pool + # assignment. + tc qdisc replace dev $swp1 root handle 1: \ + ets bands 8 strict 8 priomap 7 6 + __mlnx_qos -i $swp1 --prio2buffer=0,1,0,0,0,0,0,0 >/dev/null + + # $swp2 + # ----- + + ip link set dev $swp2 up + mtu_set $swp2 10000 + vlan_create $swp2 111 + ip link set dev $swp2.111 type vlan egress-qos-map 0:0 1:1 + + devlink_port_pool_th_set $swp2 6 $_10MB + devlink_tc_bind_pool_th_set $swp2 1 egress 6 $_10MB + + # prio 0->TC0 (band 7), 1->TC1 (band 6). TC1 is shaped. + tc qdisc replace dev $swp2 root handle 1: \ + ets bands 8 strict 8 priomap 7 6 + tc qdisc replace dev $swp2 parent 1:7 handle 17: \ + tbf rate 200Mbit burst 131072 limit 1M + + # $swp3 + # ----- + + ip link set dev $swp3 up + mtu_set $swp3 10000 + vlan_create $swp3 111 + ip link set dev $swp3.111 type vlan egress-qos-map 0:0 1:1 + + devlink_port_pool_th_set $swp3 5 $_10MB + devlink_tc_bind_pool_th_set $swp3 1 egress 5 $_10MB + + # prio 0->TC0 (band 7), 1->TC1 (band 6) + tc qdisc replace dev $swp3 root handle 1: \ + ets bands 8 strict 8 priomap 7 6 + + # Need to enable PFC so that PAUSE takes effect. Therefore need to put + # the lossless prio into a buffer of its own. Don't bother with buffer + # sizes though, there is not going to be any pressure in the "backward" + # direction. + __mlnx_qos -i $swp3 --prio2buffer=0,1,0,0,0,0,0,0 >/dev/null + __mlnx_qos -i $swp3 --pfc=0,1,0,0,0,0,0,0 >/dev/null + + # $swp4 + # ----- + + ip link set dev $swp4 up + mtu_set $swp4 10000 + vlan_create $swp4 111 + ip link set dev $swp4.111 type vlan ingress-qos-map 0:0 1:1 + + devlink_port_pool_th_set $swp4 2 $_1MB + devlink_tc_bind_pool_th_set $swp4 1 ingress 2 $_1MB + + # Configure qdisc so that we can hand-tune headroom. + tc qdisc replace dev $swp4 root handle 1: \ + ets bands 8 strict 8 priomap 7 6 + __mlnx_qos -i $swp4 --prio2buffer=0,1,0,0,0,0,0,0 >/dev/null + __mlnx_qos -i $swp4 --pfc=0,1,0,0,0,0,0,0 >/dev/null + # PG0 will get autoconfigured to Xoff, give PG1 arbitrarily 100K, which + # is (-2*MTU) about 80K of delay provision. + __mlnx_qos -i $swp3 --buffer_size=0,$_100KB,0,0,0,0,0,0 >/dev/null + + # bridges + # ------- + + ip link add name br1 type bridge vlan_filtering 0 + ip link set dev $swp1.111 master br1 + ip link set dev $swp3.111 master br1 + ip link set dev br1 up + + ip link add name br2 type bridge vlan_filtering 0 + ip link set dev $swp2.111 master br2 + ip link set dev $swp4.111 master br2 + ip link set dev br2 up +} + +switch_destroy() +{ + # Do this first so that we can reset the limits to values that are only + # valid for the original static / dynamic setting. + devlink_pool_size_thtype_restore 6 + devlink_pool_size_thtype_restore 5 + devlink_pool_size_thtype_restore 4 + devlink_pool_size_thtype_restore 2 + devlink_pool_size_thtype_restore 1 + devlink_pool_size_thtype_restore 0 + + # bridges + # ------- + + ip link set dev br2 down + ip link set dev $swp4.111 nomaster + ip link set dev $swp2.111 nomaster + ip link del dev br2 + + ip link set dev br1 down + ip link set dev $swp3.111 nomaster + ip link set dev $swp1.111 nomaster + ip link del dev br1 + + # $swp4 + # ----- + + __mlnx_qos -i $swp4 --buffer_size=0,0,0,0,0,0,0,0 >/dev/null + __mlnx_qos -i $swp4 --pfc=0,0,0,0,0,0,0,0 >/dev/null + __mlnx_qos -i $swp4 --prio2buffer=0,0,0,0,0,0,0,0 >/dev/null + tc qdisc del dev $swp4 root + + devlink_tc_bind_pool_th_restore $swp4 1 ingress + devlink_port_pool_th_restore $swp4 2 + + vlan_destroy $swp4 111 + mtu_restore $swp4 + ip link set dev $swp4 down + + # $swp3 + # ----- + + __mlnx_qos -i $swp3 --pfc=0,0,0,0,0,0,0,0 >/dev/null + __mlnx_qos -i $swp3 --prio2buffer=0,0,0,0,0,0,0,0 >/dev/null + tc qdisc del dev $swp3 root + + devlink_tc_bind_pool_th_restore $swp3 1 egress + devlink_port_pool_th_restore $swp3 5 + + vlan_destroy $swp3 111 + mtu_restore $swp3 + ip link set dev $swp3 down + + # $swp2 + # ----- + + tc qdisc del dev $swp2 parent 1:7 + tc qdisc del dev $swp2 root + + devlink_tc_bind_pool_th_restore $swp2 1 egress + devlink_port_pool_th_restore $swp2 6 + + vlan_destroy $swp2 111 + mtu_restore $swp2 + ip link set dev $swp2 down + + # $swp1 + # ----- + + __mlnx_qos -i $swp1 --prio2buffer=0,0,0,0,0,0,0,0 >/dev/null + tc qdisc del dev $swp1 root + + devlink_tc_bind_pool_th_restore $swp1 1 ingress + devlink_port_pool_th_restore $swp1 1 + + vlan_destroy $swp1 111 + mtu_restore $swp1 + ip link set dev $swp1 down +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + swp3=${NETIFS[p5]} + swp4=${NETIFS[p6]} + + h2mac=$(mac_get $h2) + + vrf_prepare + + h1_create + h2_create + switch_create +} + +cleanup() +{ + pre_cleanup + + switch_destroy + h2_destroy + h1_destroy + + vrf_cleanup +} + +ping_ipv4() +{ + ping_test $h1 192.0.2.34 +} + +test_qos_pfc() +{ + RET=0 + + # 10M pool, each packet is 8K of payload + headers + local pkts=$((_10MB / 8050)) + local size=$((pkts * 8050)) + local in0=$(ethtool_stats_get $swp1 rx_octets_prio_1) + local out0=$(ethtool_stats_get $swp2 tx_octets_prio_1) + + $MZ $h1 -p 8000 -Q 1:111 -A 192.0.2.33 -B 192.0.2.34 \ + -a own -b $h2mac -c $pkts -t udp -q + sleep 2 + + local in1=$(ethtool_stats_get $swp1 rx_octets_prio_1) + local out1=$(ethtool_stats_get $swp2 tx_octets_prio_1) + + local din=$((in1 - in0)) + local dout=$((out1 - out0)) + + local pct_in=$((din * 100 / size)) + + ((pct_in > 95 && pct_in < 105)) + check_err $? "Relative ingress out of expected bounds, $pct_in% should be 100%" + + ((dout == din)) + check_err $? "$((din - dout)) bytes out of $din ingressed got lost" + + log_test "PFC" +} + +trap cleanup EXIT + +bail_on_lldpad +setup_prepare +setup_wait +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh index 94c37124a840..af64bc9ea8ab 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh @@ -27,11 +27,17 @@ switch_create() # amount of traffic that is admitted to the shared buffers. This makes # sure that there is always enough traffic of all types to select from # for the DWRR process. + devlink_port_pool_th_save $swp1 0 devlink_port_pool_th_set $swp1 0 12 + devlink_tc_bind_pool_th_save $swp1 0 ingress devlink_tc_bind_pool_th_set $swp1 0 ingress 0 12 + devlink_port_pool_th_save $swp2 4 devlink_port_pool_th_set $swp2 4 12 + devlink_tc_bind_pool_th_save $swp2 7 egress devlink_tc_bind_pool_th_set $swp2 7 egress 4 5 + devlink_tc_bind_pool_th_save $swp2 6 egress devlink_tc_bind_pool_th_set $swp2 6 egress 4 5 + devlink_tc_bind_pool_th_save $swp2 5 egress devlink_tc_bind_pool_th_set $swp2 5 egress 4 5 # Note: sch_ets_core.sh uses VLAN ingress-qos-map to assign packet diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh index 517297a14ecf..b0cb1aaffdda 100644 --- a/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh @@ -208,6 +208,7 @@ switch_create() ip link set dev br2_11 up local size=$(devlink_pool_size_thtype 0 | cut -d' ' -f 1) + devlink_port_pool_th_save $swp3 8 devlink_port_pool_th_set $swp3 8 $size } diff --git a/tools/testing/selftests/drivers/net/netdevsim/udp_tunnel_nic.sh b/tools/testing/selftests/drivers/net/netdevsim/udp_tunnel_nic.sh index ba1d53b9f815..1b08e042cf94 100644..100755 --- a/tools/testing/selftests/drivers/net/netdevsim/udp_tunnel_nic.sh +++ b/tools/testing/selftests/drivers/net/netdevsim/udp_tunnel_nic.sh @@ -7,6 +7,7 @@ NSIM_DEV_SYS=/sys/bus/netdevsim/devices/netdevsim$NSIM_ID NSIM_DEV_DFS=/sys/kernel/debug/netdevsim/netdevsim$NSIM_ID NSIM_NETDEV= HAS_ETHTOOL= +STATIC_ENTRIES= EXIT_STATUS=0 num_cases=0 num_errors=0 @@ -193,6 +194,21 @@ function check_tables { sleep 0.02 ((retries--)) done + + if [ -n "$HAS_ETHTOOL" -a -n "${STATIC_ENTRIES[0]}" ]; then + fail=0 + for i in "${!STATIC_ENTRIES[@]}"; do + pp_expected=`pre_ethtool ${STATIC_ENTRIES[i]}` + cnt=$(ethtool --show-tunnels $NSIM_NETDEV | grep -c "$pp_expected") + if [ $cnt -ne 1 ]; then + err_cnt "ethtool static entry: $pfx - $msg" + echo " check_table: ethtool does not contain '$pp_expected'" + ethtool --show-tunnels $NSIM_NETDEV + fail=1 + fi + done + [ $fail == 0 ] && pass_cnt + fi } function print_table { @@ -775,6 +791,157 @@ for port in 0 1; do exp1=( 0 0 0 0 ) done +cleanup_nsim + +# shared port tables +pfx="table sharing" + +echo $NSIM_ID > /sys/bus/netdevsim/new_device +echo 0 > $NSIM_DEV_SYS/del_port + +echo 0 > $NSIM_DEV_DFS/udp_ports_open_only +echo 1 > $NSIM_DEV_DFS/udp_ports_sleep +echo 1 > $NSIM_DEV_DFS/udp_ports_shared + +old_netdevs=$(ls /sys/class/net) +echo 1 > $NSIM_DEV_SYS/new_port +NSIM_NETDEV=`get_netdev_name old_netdevs` +old_netdevs=$(ls /sys/class/net) +echo 2 > $NSIM_DEV_SYS/new_port +NSIM_NETDEV2=`get_netdev_name old_netdevs` + +msg="VxLAN v4 devices" +exp0=( `mke 4789 1` 0 0 0 ) +exp1=( 0 0 0 0 ) +new_vxlan vxlan0 4789 $NSIM_NETDEV +new_vxlan vxlan1 4789 $NSIM_NETDEV2 + +msg="VxLAN v4 devices go down" +exp0=( 0 0 0 0 ) +ifconfig vxlan1 down +ifconfig vxlan0 down +check_tables + +for ifc in vxlan0 vxlan1; do + ifconfig $ifc up +done + +msg="VxLAN v6 device" +exp0=( `mke 4789 1` `mke 4790 1` 0 0 ) +new_vxlan vxlanC 4790 $NSIM_NETDEV 6 + +msg="Geneve device" +exp1=( `mke 6081 2` 0 0 0 ) +new_geneve gnv0 6081 + +msg="NIC device goes down" +ifconfig $NSIM_NETDEV down +check_tables + +msg="NIC device goes up again" +ifconfig $NSIM_NETDEV up +check_tables + +for i in `seq 2`; do + msg="turn feature off - 1, rep $i" + ethtool -K $NSIM_NETDEV rx-udp_tunnel-port-offload off + check_tables + + msg="turn feature off - 2, rep $i" + exp0=( 0 0 0 0 ) + exp1=( 0 0 0 0 ) + ethtool -K $NSIM_NETDEV2 rx-udp_tunnel-port-offload off + check_tables + + msg="turn feature on - 1, rep $i" + exp0=( `mke 4789 1` `mke 4790 1` 0 0 ) + exp1=( `mke 6081 2` 0 0 0 ) + ethtool -K $NSIM_NETDEV rx-udp_tunnel-port-offload on + check_tables + + msg="turn feature on - 2, rep $i" + ethtool -K $NSIM_NETDEV2 rx-udp_tunnel-port-offload on + check_tables +done + +msg="tunnels destroyed 1" +cleanup_tuns +exp0=( 0 0 0 0 ) +exp1=( 0 0 0 0 ) +check_tables + +overflow_table0 "overflow NIC table" + +msg="re-add a port" + +echo 2 > $NSIM_DEV_SYS/del_port +echo 2 > $NSIM_DEV_SYS/new_port +check_tables + +msg="replace VxLAN in overflow table" +exp0=( `mke 10000 1` `mke 10004 1` `mke 10002 1` `mke 10003 1` ) +del_dev vxlan1 + +msg="vacate VxLAN in overflow table" +exp0=( `mke 10000 1` `mke 10004 1` 0 `mke 10003 1` ) +del_dev vxlan2 + +echo 1 > $NSIM_DEV_DFS/ports/$port/udp_ports_reset +check_tables + +msg="tunnels destroyed 2" +cleanup_tuns +exp0=( 0 0 0 0 ) +exp1=( 0 0 0 0 ) +check_tables + +echo 1 > $NSIM_DEV_SYS/del_port +echo 2 > $NSIM_DEV_SYS/del_port + +cleanup_nsim + +# Static IANA port +pfx="static IANA vxlan" + +echo $NSIM_ID > /sys/bus/netdevsim/new_device +echo 0 > $NSIM_DEV_SYS/del_port + +echo 1 > $NSIM_DEV_DFS/udp_ports_static_iana_vxlan +STATIC_ENTRIES=( `mke 4789 1` ) + +port=1 +old_netdevs=$(ls /sys/class/net) +echo $port > $NSIM_DEV_SYS/new_port +NSIM_NETDEV=`get_netdev_name old_netdevs` + +msg="check empty" +exp0=( 0 0 0 0 ) +exp1=( 0 0 0 0 ) +check_tables + +msg="add on static port" +new_vxlan vxlan0 4789 $NSIM_NETDEV +new_vxlan vxlan1 4789 $NSIM_NETDEV + +msg="add on different port" +exp0=( `mke 4790 1` 0 0 0 ) +new_vxlan vxlan2 4790 $NSIM_NETDEV + +cleanup_tuns + +msg="tunnels destroyed" +exp0=( 0 0 0 0 ) +exp1=( 0 0 0 0 ) +check_tables + +msg="different type" +new_geneve gnv0 4789 + +cleanup_tuns +cleanup_nsim + +# END + modprobe -r netdevsim if [ $num_errors -eq 0 ]; then diff --git a/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh b/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh new file mode 100755 index 000000000000..71a538add08a --- /dev/null +++ b/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh @@ -0,0 +1,273 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# Copyright 2020 NXP Semiconductors + +WAIT_TIME=1 +NUM_NETIFS=4 +lib_dir=$(dirname $0)/../../../net/forwarding +source $lib_dir/tc_common.sh +source $lib_dir/lib.sh + +require_command tcpdump + +# +# +---------------------------------------------+ +# | DUT ports Generator ports | +# | +--------+ +--------+ +--------+ +--------+ | +# | | | | | | | | | | +# | | eth0 | | eth1 | | eth2 | | eth3 | | +# | | | | | | | | | | +# +-+--------+-+--------+-+--------+-+--------+-+ +# | | | | +# | | | | +# | +-----------+ | +# | | +# +--------------------------------+ + +eth0=${NETIFS[p1]} +eth1=${NETIFS[p2]} +eth2=${NETIFS[p3]} +eth3=${NETIFS[p4]} + +eth0_mac="de:ad:be:ef:00:00" +eth1_mac="de:ad:be:ef:00:01" +eth2_mac="de:ad:be:ef:00:02" +eth3_mac="de:ad:be:ef:00:03" + +# Helpers to map a VCAP IS1 and VCAP IS2 lookup and policy to a chain number +# used by the kernel driver. The numbers are: +# VCAP IS1 lookup 0: 10000 +# VCAP IS1 lookup 1: 11000 +# VCAP IS1 lookup 2: 12000 +# VCAP IS2 lookup 0 policy 0: 20000 +# VCAP IS2 lookup 0 policy 1: 20001 +# VCAP IS2 lookup 0 policy 255: 20255 +# VCAP IS2 lookup 1 policy 0: 21000 +# VCAP IS2 lookup 1 policy 1: 21001 +# VCAP IS2 lookup 1 policy 255: 21255 +IS1() +{ + local lookup=$1 + + echo $((10000 + 1000 * lookup)) +} + +IS2() +{ + local lookup=$1 + local pag=$2 + + echo $((20000 + 1000 * lookup + pag)) +} + +ES0() +{ + echo 0 +} + +# The Ocelot switches have a fixed ingress pipeline composed of: +# +# +----------------------------------------------+ +-----------------------------------------+ +# | VCAP IS1 | | VCAP IS2 | +# | | | | +# | +----------+ +----------+ +----------+ | | +----------+ +----------+ | +# | | Lookup 0 | | Lookup 1 | | Lookup 2 | | --+------> PAG 0: | Lookup 0 | -> | Lookup 1 | | +# | +----------+ -> +----------+ -> +----------+ | | | +----------+ +----------+ | +# | |key&action| |key&action| |key&action| | | | |key&action| |key&action| | +# | |key&action| |key&action| |key&action| | | | | .. | | .. | | +# | | .. | | .. | | .. | | | | +----------+ +----------+ | +# | +----------+ +----------+ +----------+ | | | | +# | selects PAG | | | +----------+ +----------+ | +# +----------------------------------------------+ +------> PAG 1: | Lookup 0 | -> | Lookup 1 | | +# | | +----------+ +----------+ | +# | | |key&action| |key&action| | +# | | | .. | | .. | | +# | | +----------+ +----------+ | +# | | ... | +# | | | +# | | +----------+ +----------+ | +# +----> PAG 254: | Lookup 0 | -> | Lookup 1 | | +# | | +----------+ +----------+ | +# | | |key&action| |key&action| | +# | | | .. | | .. | | +# | | +----------+ +----------+ | +# | | | +# | | +----------+ +----------+ | +# +----> PAG 255: | Lookup 0 | -> | Lookup 1 | | +# | +----------+ +----------+ | +# | |key&action| |key&action| | +# | | .. | | .. | | +# | +----------+ +----------+ | +# +-----------------------------------------+ +# +# Both the VCAP IS1 (Ingress Stage 1) and IS2 (Ingress Stage 2) are indexed +# (looked up) multiple times: IS1 3 times, and IS2 2 times. Each filter +# (key and action pair) can be configured to only match during the first, or +# second, etc, lookup. +# +# During one TCAM lookup, the filter processing stops at the first entry that +# matches, then the pipeline jumps to the next lookup. +# The driver maps each individual lookup of each individual ingress TCAM to a +# separate chain number. For correct rule offloading, it is mandatory that each +# filter installed in one TCAM is terminated by a non-optional GOTO action to +# the next lookup from the fixed pipeline. +# +# A chain can only be used if there is a GOTO action correctly set up from the +# prior lookup in the processing pipeline. Setting up all chains is not +# mandatory. + +# NOTE: VCAP IS1 currently uses only S1_NORMAL half keys and VCAP IS2 +# dynamically chooses between MAC_ETYPE, ARP, IP4_TCP_UDP, IP4_OTHER, which are +# all half keys as well. + +create_tcam_skeleton() +{ + local eth=$1 + + tc qdisc add dev $eth clsact + + # VCAP IS1 is the Ingress Classification TCAM and can offload the + # following actions: + # - skbedit priority + # - vlan pop + # - vlan modify + # - goto (only in lookup 2, the last IS1 lookup) + tc filter add dev $eth ingress chain 0 pref 49152 flower \ + skip_sw action goto chain $(IS1 0) + tc filter add dev $eth ingress chain $(IS1 0) pref 49152 \ + flower skip_sw action goto chain $(IS1 1) + tc filter add dev $eth ingress chain $(IS1 1) pref 49152 \ + flower skip_sw action goto chain $(IS1 2) + tc filter add dev $eth ingress chain $(IS1 2) pref 49152 \ + flower skip_sw action goto chain $(IS2 0 0) + + # VCAP IS2 is the Security Enforcement ingress TCAM and can offload the + # following actions: + # - trap + # - drop + # - police + # The two VCAP IS2 lookups can be segmented into up to 256 groups of + # rules, called Policies. A Policy is selected through the Policy + # Association Group (PAG) action of VCAP IS1 (which is the + # GOTO offload). + tc filter add dev $eth ingress chain $(IS2 0 0) pref 49152 \ + flower skip_sw action goto chain $(IS2 1 0) +} + +setup_prepare() +{ + create_tcam_skeleton $eth0 + + ip link add br0 type bridge + ip link set $eth0 master br0 + ip link set $eth1 master br0 + ip link set br0 up + + ip link add link $eth3 name $eth3.100 type vlan id 100 + ip link set $eth3.100 up + + tc filter add dev $eth0 ingress chain $(IS1 1) pref 1 \ + protocol 802.1Q flower skip_sw vlan_id 100 \ + action vlan pop \ + action goto chain $(IS1 2) + + tc filter add dev $eth0 egress chain $(ES0) pref 1 \ + flower skip_sw indev $eth1 \ + action vlan push protocol 802.1Q id 100 + + tc filter add dev $eth0 ingress chain $(IS1 0) \ + protocol ipv4 flower skip_sw src_ip 10.1.1.2 \ + action skbedit priority 7 \ + action goto chain $(IS1 1) + + tc filter add dev $eth0 ingress chain $(IS2 0 0) \ + protocol ipv4 flower skip_sw ip_proto udp dst_port 5201 \ + action police rate 50mbit burst 64k \ + action goto chain $(IS2 1 0) +} + +cleanup() +{ + ip link del $eth3.100 + tc qdisc del dev $eth0 clsact + ip link del br0 +} + +test_vlan_pop() +{ + printf "Testing VLAN pop.. " + + tcpdump_start $eth2 + + # Work around Mausezahn VLAN builder bug + # (https://github.com/netsniff-ng/netsniff-ng/issues/225) by using + # an 8021q upper + $MZ $eth3.100 -q -c 1 -p 64 -a $eth3_mac -b $eth2_mac -t ip + + sleep 1 + + tcpdump_stop + + if tcpdump_show | grep -q "$eth3_mac > $eth2_mac, ethertype IPv4"; then + echo "OK" + else + echo "FAIL" + fi + + tcpdump_cleanup +} + +test_vlan_push() +{ + printf "Testing VLAN push.. " + + tcpdump_start $eth3.100 + + $MZ $eth2 -q -c 1 -p 64 -a $eth2_mac -b $eth3_mac -t ip + + sleep 1 + + tcpdump_stop + + if tcpdump_show | grep -q "$eth2_mac > $eth3_mac"; then + echo "OK" + else + echo "FAIL" + fi + + tcpdump_cleanup +} + +test_skbedit_priority() +{ + local num_pkts=100 + + printf "Testing frame prioritization.. " + + before=$(ethtool_stats_get $eth0 'rx_green_prio_7') + + $MZ $eth3 -q -c $num_pkts -p 64 -a $eth3_mac -b $eth2_mac -t ip -A 10.1.1.2 + + after=$(ethtool_stats_get $eth0 'rx_green_prio_7') + + if [ $((after - before)) = $num_pkts ]; then + echo "OK" + else + echo "FAIL" + fi +} + +trap cleanup EXIT + +ALL_TESTS=" + test_vlan_pop + test_vlan_push + test_skbedit_priority +" + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/.gitignore b/tools/testing/selftests/net/.gitignore index 742c499328b2..61ae899cfc17 100644 --- a/tools/testing/selftests/net/.gitignore +++ b/tools/testing/selftests/net/.gitignore @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only +ipsec msg_zerocopy socket psock_fanout diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index 9491bbaa0831..4773ce72edbd 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile @@ -19,6 +19,7 @@ TEST_PROGS += txtimestamp.sh TEST_PROGS += vrf-xfrm-tests.sh TEST_PROGS += rxtimestamp.sh TEST_PROGS += devlink_port_split.py +TEST_PROGS += drop_monitor_tests.sh TEST_PROGS_EXTENDED := in_netns.sh TEST_GEN_FILES = socket nettest TEST_GEN_FILES += psock_fanout psock_tpacket msg_zerocopy reuseport_addr_any @@ -29,6 +30,7 @@ TEST_GEN_FILES += tcp_fastopen_backup_key TEST_GEN_FILES += fin_ack_lat TEST_GEN_FILES += reuseaddr_ports_exhausted TEST_GEN_FILES += hwtstamp_config rxtimestamp timestamping txtimestamp +TEST_GEN_FILES += ipsec TEST_GEN_PROGS = reuseport_bpf reuseport_bpf_cpu reuseport_bpf_numa TEST_GEN_PROGS += reuseport_dualstack reuseaddr_conflict tls diff --git a/tools/testing/selftests/net/config b/tools/testing/selftests/net/config index 5a57ea02802d..43649242adc0 100644 --- a/tools/testing/selftests/net/config +++ b/tools/testing/selftests/net/config @@ -30,3 +30,6 @@ CONFIG_NET_SCH_ETF=m CONFIG_NET_SCH_NETEM=y CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_KALLSYMS=y +CONFIG_TRACEPOINTS=y +CONFIG_NET_DROP_MONITOR=m +CONFIG_NETDEVSIM=m diff --git a/tools/testing/selftests/net/drop_monitor_tests.sh b/tools/testing/selftests/net/drop_monitor_tests.sh new file mode 100755 index 000000000000..b7650e30d18b --- /dev/null +++ b/tools/testing/selftests/net/drop_monitor_tests.sh @@ -0,0 +1,215 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# This test is for checking drop monitor functionality. + +ret=0 +# Kselftest framework requirement - SKIP code is 4. +ksft_skip=4 + +# all tests in this script. Can be overridden with -t option +TESTS=" + sw_drops + hw_drops +" + +IP="ip -netns ns1" +TC="tc -netns ns1" +DEVLINK="devlink -N ns1" +NS_EXEC="ip netns exec ns1" +NETDEVSIM_PATH=/sys/bus/netdevsim/ +DEV_ADDR=1337 +DEV=netdevsim${DEV_ADDR} +DEVLINK_DEV=netdevsim/${DEV} + +log_test() +{ + local rc=$1 + local expected=$2 + local msg="$3" + + if [ ${rc} -eq ${expected} ]; then + printf " TEST: %-60s [ OK ]\n" "${msg}" + nsuccess=$((nsuccess+1)) + else + ret=1 + nfail=$((nfail+1)) + printf " TEST: %-60s [FAIL]\n" "${msg}" + fi +} + +setup() +{ + modprobe netdevsim &> /dev/null + + set -e + ip netns add ns1 + $IP link add dummy10 up type dummy + + $NS_EXEC echo "$DEV_ADDR 1" > ${NETDEVSIM_PATH}/new_device + udevadm settle + local netdev=$($NS_EXEC ls ${NETDEVSIM_PATH}/devices/${DEV}/net/) + $IP link set dev $netdev up + + set +e +} + +cleanup() +{ + $NS_EXEC echo "$DEV_ADDR" > ${NETDEVSIM_PATH}/del_device + ip netns del ns1 +} + +sw_drops_test() +{ + echo + echo "Software drops test" + + setup + + local dir=$(mktemp -d) + + $TC qdisc add dev dummy10 clsact + $TC filter add dev dummy10 egress pref 1 handle 101 proto ip \ + flower dst_ip 192.0.2.10 action drop + + $NS_EXEC mausezahn dummy10 -a 00:11:22:33:44:55 -b 00:aa:bb:cc:dd:ee \ + -A 192.0.2.1 -B 192.0.2.10 -t udp sp=12345,dp=54321 -c 0 -q \ + -d 100msec & + timeout 5 dwdump -o sw -w ${dir}/packets.pcap + (( $(tshark -r ${dir}/packets.pcap \ + -Y 'ip.dst == 192.0.2.10' 2> /dev/null | wc -l) != 0)) + log_test $? 0 "Capturing active software drops" + + rm ${dir}/packets.pcap + + { kill %% && wait %%; } 2>/dev/null + timeout 5 dwdump -o sw -w ${dir}/packets.pcap + (( $(tshark -r ${dir}/packets.pcap \ + -Y 'ip.dst == 192.0.2.10' 2> /dev/null | wc -l) == 0)) + log_test $? 0 "Capturing inactive software drops" + + rm -r $dir + + cleanup +} + +hw_drops_test() +{ + echo + echo "Hardware drops test" + + setup + + local dir=$(mktemp -d) + + $DEVLINK trap set $DEVLINK_DEV trap blackhole_route action trap + timeout 5 dwdump -o hw -w ${dir}/packets.pcap + (( $(tshark -r ${dir}/packets.pcap \ + -Y 'net_dm.hw_trap_name== blackhole_route' 2> /dev/null \ + | wc -l) != 0)) + log_test $? 0 "Capturing active hardware drops" + + rm ${dir}/packets.pcap + + $DEVLINK trap set $DEVLINK_DEV trap blackhole_route action drop + timeout 5 dwdump -o hw -w ${dir}/packets.pcap + (( $(tshark -r ${dir}/packets.pcap \ + -Y 'net_dm.hw_trap_name== blackhole_route' 2> /dev/null \ + | wc -l) == 0)) + log_test $? 0 "Capturing inactive hardware drops" + + rm -r $dir + + cleanup +} + +################################################################################ +# usage + +usage() +{ + cat <<EOF +usage: ${0##*/} OPTS + + -t <test> Test(s) to run (default: all) + (options: $TESTS) +EOF +} + +################################################################################ +# main + +while getopts ":t:h" opt; do + case $opt in + t) TESTS=$OPTARG;; + h) usage; exit 0;; + *) usage; exit 1;; + esac +done + +if [ "$(id -u)" -ne 0 ];then + echo "SKIP: Need root privileges" + exit $ksft_skip; +fi + +if [ ! -x "$(command -v ip)" ]; then + echo "SKIP: Could not run test without ip tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v devlink)" ]; then + echo "SKIP: Could not run test without devlink tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v tshark)" ]; then + echo "SKIP: Could not run test without tshark tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v dwdump)" ]; then + echo "SKIP: Could not run test without dwdump tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v udevadm)" ]; then + echo "SKIP: Could not run test without udevadm tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v timeout)" ]; then + echo "SKIP: Could not run test without timeout tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v mausezahn)" ]; then + echo "SKIP: Could not run test without mausezahn tool" + exit $ksft_skip +fi + +tshark -G fields 2> /dev/null | grep -q net_dm +if [ $? -ne 0 ]; then + echo "SKIP: tshark too old, missing net_dm dissector" + exit $ksft_skip +fi + +# start clean +cleanup &> /dev/null + +for t in $TESTS +do + case $t in + sw_drops|sw) sw_drops_test;; + hw_drops|hw) hw_drops_test;; + + help) echo "Test names: $TESTS"; exit 0;; + esac +done + +if [ "$TESTS" != "none" ]; then + printf "\nTests passed: %3d\n" ${nsuccess} + printf "Tests failed: %3d\n" ${nfail} +fi + +exit $ret diff --git a/tools/testing/selftests/net/forwarding/devlink_lib.sh b/tools/testing/selftests/net/forwarding/devlink_lib.sh index 75fe24bcb9cd..9c12c4fd3afc 100644 --- a/tools/testing/selftests/net/forwarding/devlink_lib.sh +++ b/tools/testing/selftests/net/forwarding/devlink_lib.sh @@ -5,7 +5,7 @@ # Defines if [[ ! -v DEVLINK_DEV ]]; then - DEVLINK_DEV=$(devlink port show "${NETIFS[p1]}" -j \ + DEVLINK_DEV=$(devlink port show "${NETIFS[p1]:-$NETIF_NO_CABLE}" -j \ | jq -r '.port | keys[]' | cut -d/ -f-2) if [ -z "$DEVLINK_DEV" ]; then echo "SKIP: ${NETIFS[p1]} has no devlink device registered for it" @@ -117,6 +117,12 @@ devlink_reload() declare -A DEVLINK_ORIG +# Changing pool type from static to dynamic causes reinterpretation of threshold +# values. They therefore need to be saved before pool type is changed, then the +# pool type can be changed, and then the new values need to be set up. Therefore +# instead of saving the current state implicitly in the _set call, provide +# functions for all three primitives: save, set, and restore. + devlink_port_pool_threshold() { local port=$1; shift @@ -126,14 +132,21 @@ devlink_port_pool_threshold() | jq '.port_pool."'"$port"'"[].threshold' } -devlink_port_pool_th_set() +devlink_port_pool_th_save() { local port=$1; shift local pool=$1; shift - local th=$1; shift local key="port_pool($port,$pool).threshold" DEVLINK_ORIG[$key]=$(devlink_port_pool_threshold $port $pool) +} + +devlink_port_pool_th_set() +{ + local port=$1; shift + local pool=$1; shift + local th=$1; shift + devlink sb port pool set $port pool $pool th $th } @@ -142,8 +155,13 @@ devlink_port_pool_th_restore() local port=$1; shift local pool=$1; shift local key="port_pool($port,$pool).threshold" + local -a orig=(${DEVLINK_ORIG[$key]}) - devlink sb port pool set $port pool $pool th ${DEVLINK_ORIG[$key]} + if [[ -z $orig ]]; then + echo "WARNING: Mismatched devlink_port_pool_th_restore" + else + devlink sb port pool set $port pool $pool th $orig + fi } devlink_pool_size_thtype() @@ -154,14 +172,20 @@ devlink_pool_size_thtype() | jq -r '.pool[][] | (.size, .thtype)' } +devlink_pool_size_thtype_save() +{ + local pool=$1; shift + local key="pool($pool).size_thtype" + + DEVLINK_ORIG[$key]=$(devlink_pool_size_thtype $pool) +} + devlink_pool_size_thtype_set() { local pool=$1; shift local thtype=$1; shift local size=$1; shift - local key="pool($pool).size_thtype" - DEVLINK_ORIG[$key]=$(devlink_pool_size_thtype $pool) devlink sb pool set "$DEVLINK_DEV" pool $pool size $size thtype $thtype } @@ -171,8 +195,12 @@ devlink_pool_size_thtype_restore() local key="pool($pool).size_thtype" local -a orig=(${DEVLINK_ORIG[$key]}) - devlink sb pool set "$DEVLINK_DEV" pool $pool \ - size ${orig[0]} thtype ${orig[1]} + if [[ -z ${orig[0]} ]]; then + echo "WARNING: Mismatched devlink_pool_size_thtype_restore" + else + devlink sb pool set "$DEVLINK_DEV" pool $pool \ + size ${orig[0]} thtype ${orig[1]} + fi } devlink_tc_bind_pool_th() @@ -185,6 +213,16 @@ devlink_tc_bind_pool_th() | jq -r '.tc_bind[][] | (.pool, .threshold)' } +devlink_tc_bind_pool_th_save() +{ + local port=$1; shift + local tc=$1; shift + local dir=$1; shift + local key="tc_bind($port,$dir,$tc).pool_th" + + DEVLINK_ORIG[$key]=$(devlink_tc_bind_pool_th $port $tc $dir) +} + devlink_tc_bind_pool_th_set() { local port=$1; shift @@ -192,9 +230,7 @@ devlink_tc_bind_pool_th_set() local dir=$1; shift local pool=$1; shift local th=$1; shift - local key="tc_bind($port,$dir,$tc).pool_th" - DEVLINK_ORIG[$key]=$(devlink_tc_bind_pool_th $port $tc $dir) devlink sb tc bind set $port tc $tc type $dir pool $pool th $th } @@ -206,8 +242,12 @@ devlink_tc_bind_pool_th_restore() local key="tc_bind($port,$dir,$tc).pool_th" local -a orig=(${DEVLINK_ORIG[$key]}) - devlink sb tc bind set $port tc $tc type $dir \ - pool ${orig[0]} th ${orig[1]} + if [[ -z ${orig[0]} ]]; then + echo "WARNING: Mismatched devlink_tc_bind_pool_th_restore" + else + devlink sb tc bind set $port tc $tc type $dir \ + pool ${orig[0]} th ${orig[1]} + fi } devlink_traps_num_get() @@ -509,3 +549,9 @@ devlink_cpu_port_get() echo "$DEVLINK_DEV/$cpu_dl_port_num" } + +devlink_cell_size_get() +{ + devlink sb pool show "$DEVLINK_DEV" pool 0 -j \ + | jq '.pool[][].cell_size' +} diff --git a/tools/testing/selftests/net/forwarding/lib.sh b/tools/testing/selftests/net/forwarding/lib.sh index 977fc2b326a2..927f9ba49e08 100644 --- a/tools/testing/selftests/net/forwarding/lib.sh +++ b/tools/testing/selftests/net/forwarding/lib.sh @@ -1227,3 +1227,46 @@ stop_traffic() # Suppress noise from killing mausezahn. { kill %% && wait %%; } 2>/dev/null } + +tcpdump_start() +{ + local if_name=$1; shift + local ns=$1; shift + + capfile=$(mktemp) + capout=$(mktemp) + + if [ -z $ns ]; then + ns_cmd="" + else + ns_cmd="ip netns exec ${ns}" + fi + + if [ -z $SUDO_USER ] ; then + capuser="" + else + capuser="-Z $SUDO_USER" + fi + + $ns_cmd tcpdump -e -n -Q in -i $if_name \ + -s 65535 -B 32768 $capuser -w $capfile > "$capout" 2>&1 & + cappid=$! + + sleep 1 +} + +tcpdump_stop() +{ + $ns_cmd kill $cappid + sleep 1 +} + +tcpdump_cleanup() +{ + rm $capfile $capout +} + +tcpdump_show() +{ + tcpdump -e -n -r $capfile 2>&1 +} diff --git a/tools/testing/selftests/net/ipsec.c b/tools/testing/selftests/net/ipsec.c new file mode 100644 index 000000000000..17ced7d6ce25 --- /dev/null +++ b/tools/testing/selftests/net/ipsec.c @@ -0,0 +1,2195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ipsec.c - Check xfrm on veth inside a net-ns. + * Copyright (c) 2018 Dmitry Safonov + */ + +#define _GNU_SOURCE + +#include <arpa/inet.h> +#include <asm/types.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <linux/limits.h> +#include <linux/netlink.h> +#include <linux/random.h> +#include <linux/rtnetlink.h> +#include <linux/veth.h> +#include <linux/xfrm.h> +#include <netinet/in.h> +#include <net/if.h> +#include <sched.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <time.h> +#include <unistd.h> + +#include "../kselftest.h" + +#define printk(fmt, ...) \ + ksft_print_msg("%d[%u] " fmt "\n", getpid(), __LINE__, ##__VA_ARGS__) + +#define pr_err(fmt, ...) printk(fmt ": %m", ##__VA_ARGS__) + +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) +#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) + +#define IPV4_STR_SZ 16 /* xxx.xxx.xxx.xxx is longest + \0 */ +#define MAX_PAYLOAD 2048 +#define XFRM_ALGO_KEY_BUF_SIZE 512 +#define MAX_PROCESSES (1 << 14) /* /16 mask divided by /30 subnets */ +#define INADDR_A ((in_addr_t) 0x0a000000) /* 10.0.0.0 */ +#define INADDR_B ((in_addr_t) 0xc0a80000) /* 192.168.0.0 */ + +/* /30 mask for one veth connection */ +#define PREFIX_LEN 30 +#define child_ip(nr) (4*nr + 1) +#define grchild_ip(nr) (4*nr + 2) + +#define VETH_FMT "ktst-%d" +#define VETH_LEN 12 + +static int nsfd_parent = -1; +static int nsfd_childa = -1; +static int nsfd_childb = -1; +static long page_size; + +/* + * ksft_cnt is static in kselftest, so isn't shared with children. + * We have to send a test result back to parent and count there. + * results_fd is a pipe with test feedback from children. + */ +static int results_fd[2]; + +const unsigned int ping_delay_nsec = 50 * 1000 * 1000; +const unsigned int ping_timeout = 300; +const unsigned int ping_count = 100; +const unsigned int ping_success = 80; + +static void randomize_buffer(void *buf, size_t buflen) +{ + int *p = (int *)buf; + size_t words = buflen / sizeof(int); + size_t leftover = buflen % sizeof(int); + + if (!buflen) + return; + + while (words--) + *p++ = rand(); + + if (leftover) { + int tmp = rand(); + + memcpy(buf + buflen - leftover, &tmp, leftover); + } + + return; +} + +static int unshare_open(void) +{ + const char *netns_path = "/proc/self/ns/net"; + int fd; + + if (unshare(CLONE_NEWNET) != 0) { + pr_err("unshare()"); + return -1; + } + + fd = open(netns_path, O_RDONLY); + if (fd <= 0) { + pr_err("open(%s)", netns_path); + return -1; + } + + return fd; +} + +static int switch_ns(int fd) +{ + if (setns(fd, CLONE_NEWNET)) { + pr_err("setns()"); + return -1; + } + return 0; +} + +/* + * Running the test inside a new parent net namespace to bother less + * about cleanup on error-path. + */ +static int init_namespaces(void) +{ + nsfd_parent = unshare_open(); + if (nsfd_parent <= 0) + return -1; + + nsfd_childa = unshare_open(); + if (nsfd_childa <= 0) + return -1; + + if (switch_ns(nsfd_parent)) + return -1; + + nsfd_childb = unshare_open(); + if (nsfd_childb <= 0) + return -1; + + if (switch_ns(nsfd_parent)) + return -1; + return 0; +} + +static int netlink_sock(int *sock, uint32_t *seq_nr, int proto) +{ + if (*sock > 0) { + seq_nr++; + return 0; + } + + *sock = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, proto); + if (*sock <= 0) { + pr_err("socket(AF_NETLINK)"); + return -1; + } + + randomize_buffer(seq_nr, sizeof(*seq_nr)); + + return 0; +} + +static inline struct rtattr *rtattr_hdr(struct nlmsghdr *nh) +{ + return (struct rtattr *)((char *)(nh) + RTA_ALIGN((nh)->nlmsg_len)); +} + +static int rtattr_pack(struct nlmsghdr *nh, size_t req_sz, + unsigned short rta_type, const void *payload, size_t size) +{ + /* NLMSG_ALIGNTO == RTA_ALIGNTO, nlmsg_len already aligned */ + struct rtattr *attr = rtattr_hdr(nh); + size_t nl_size = RTA_ALIGN(nh->nlmsg_len) + RTA_LENGTH(size); + + if (req_sz < nl_size) { + printk("req buf is too small: %zu < %zu", req_sz, nl_size); + return -1; + } + nh->nlmsg_len = nl_size; + + attr->rta_len = RTA_LENGTH(size); + attr->rta_type = rta_type; + memcpy(RTA_DATA(attr), payload, size); + + return 0; +} + +static struct rtattr *_rtattr_begin(struct nlmsghdr *nh, size_t req_sz, + unsigned short rta_type, const void *payload, size_t size) +{ + struct rtattr *ret = rtattr_hdr(nh); + + if (rtattr_pack(nh, req_sz, rta_type, payload, size)) + return 0; + + return ret; +} + +static inline struct rtattr *rtattr_begin(struct nlmsghdr *nh, size_t req_sz, + unsigned short rta_type) +{ + return _rtattr_begin(nh, req_sz, rta_type, 0, 0); +} + +static inline void rtattr_end(struct nlmsghdr *nh, struct rtattr *attr) +{ + char *nlmsg_end = (char *)nh + nh->nlmsg_len; + + attr->rta_len = nlmsg_end - (char *)attr; +} + +static int veth_pack_peerb(struct nlmsghdr *nh, size_t req_sz, + const char *peer, int ns) +{ + struct ifinfomsg pi; + struct rtattr *peer_attr; + + memset(&pi, 0, sizeof(pi)); + pi.ifi_family = AF_UNSPEC; + pi.ifi_change = 0xFFFFFFFF; + + peer_attr = _rtattr_begin(nh, req_sz, VETH_INFO_PEER, &pi, sizeof(pi)); + if (!peer_attr) + return -1; + + if (rtattr_pack(nh, req_sz, IFLA_IFNAME, peer, strlen(peer))) + return -1; + + if (rtattr_pack(nh, req_sz, IFLA_NET_NS_FD, &ns, sizeof(ns))) + return -1; + + rtattr_end(nh, peer_attr); + + return 0; +} + +static int netlink_check_answer(int sock) +{ + struct nlmsgerror { + struct nlmsghdr hdr; + int error; + struct nlmsghdr orig_msg; + } answer; + + if (recv(sock, &answer, sizeof(answer), 0) < 0) { + pr_err("recv()"); + return -1; + } else if (answer.hdr.nlmsg_type != NLMSG_ERROR) { + printk("expected NLMSG_ERROR, got %d", (int)answer.hdr.nlmsg_type); + return -1; + } else if (answer.error) { + printk("NLMSG_ERROR: %d: %s", + answer.error, strerror(-answer.error)); + return answer.error; + } + + return 0; +} + +static int veth_add(int sock, uint32_t seq, const char *peera, int ns_a, + const char *peerb, int ns_b) +{ + uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE; + struct { + struct nlmsghdr nh; + struct ifinfomsg info; + char attrbuf[MAX_PAYLOAD]; + } req; + const char veth_type[] = "veth"; + struct rtattr *link_info, *info_data; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)); + req.nh.nlmsg_type = RTM_NEWLINK; + req.nh.nlmsg_flags = flags; + req.nh.nlmsg_seq = seq; + req.info.ifi_family = AF_UNSPEC; + req.info.ifi_change = 0xFFFFFFFF; + + if (rtattr_pack(&req.nh, sizeof(req), IFLA_IFNAME, peera, strlen(peera))) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), IFLA_NET_NS_FD, &ns_a, sizeof(ns_a))) + return -1; + + link_info = rtattr_begin(&req.nh, sizeof(req), IFLA_LINKINFO); + if (!link_info) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), IFLA_INFO_KIND, veth_type, sizeof(veth_type))) + return -1; + + info_data = rtattr_begin(&req.nh, sizeof(req), IFLA_INFO_DATA); + if (!info_data) + return -1; + + if (veth_pack_peerb(&req.nh, sizeof(req), peerb, ns_b)) + return -1; + + rtattr_end(&req.nh, info_data); + rtattr_end(&req.nh, link_info); + + if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + return -1; + } + return netlink_check_answer(sock); +} + +static int ip4_addr_set(int sock, uint32_t seq, const char *intf, + struct in_addr addr, uint8_t prefix) +{ + uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE; + struct { + struct nlmsghdr nh; + struct ifaddrmsg info; + char attrbuf[MAX_PAYLOAD]; + } req; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)); + req.nh.nlmsg_type = RTM_NEWADDR; + req.nh.nlmsg_flags = flags; + req.nh.nlmsg_seq = seq; + req.info.ifa_family = AF_INET; + req.info.ifa_prefixlen = prefix; + req.info.ifa_index = if_nametoindex(intf); + +#ifdef DEBUG + { + char addr_str[IPV4_STR_SZ] = {}; + + strncpy(addr_str, inet_ntoa(addr), IPV4_STR_SZ - 1); + + printk("ip addr set %s", addr_str); + } +#endif + + if (rtattr_pack(&req.nh, sizeof(req), IFA_LOCAL, &addr, sizeof(addr))) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), IFA_ADDRESS, &addr, sizeof(addr))) + return -1; + + if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + return -1; + } + return netlink_check_answer(sock); +} + +static int link_set_up(int sock, uint32_t seq, const char *intf) +{ + struct { + struct nlmsghdr nh; + struct ifinfomsg info; + char attrbuf[MAX_PAYLOAD]; + } req; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)); + req.nh.nlmsg_type = RTM_NEWLINK; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.nh.nlmsg_seq = seq; + req.info.ifi_family = AF_UNSPEC; + req.info.ifi_change = 0xFFFFFFFF; + req.info.ifi_index = if_nametoindex(intf); + req.info.ifi_flags = IFF_UP; + req.info.ifi_change = IFF_UP; + + if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + return -1; + } + return netlink_check_answer(sock); +} + +static int ip4_route_set(int sock, uint32_t seq, const char *intf, + struct in_addr src, struct in_addr dst) +{ + struct { + struct nlmsghdr nh; + struct rtmsg rt; + char attrbuf[MAX_PAYLOAD]; + } req; + unsigned int index = if_nametoindex(intf); + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.rt)); + req.nh.nlmsg_type = RTM_NEWROUTE; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE; + req.nh.nlmsg_seq = seq; + req.rt.rtm_family = AF_INET; + req.rt.rtm_dst_len = 32; + req.rt.rtm_table = RT_TABLE_MAIN; + req.rt.rtm_protocol = RTPROT_BOOT; + req.rt.rtm_scope = RT_SCOPE_LINK; + req.rt.rtm_type = RTN_UNICAST; + + if (rtattr_pack(&req.nh, sizeof(req), RTA_DST, &dst, sizeof(dst))) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), RTA_PREFSRC, &src, sizeof(src))) + return -1; + + if (rtattr_pack(&req.nh, sizeof(req), RTA_OIF, &index, sizeof(index))) + return -1; + + if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + return -1; + } + + return netlink_check_answer(sock); +} + +static int tunnel_set_route(int route_sock, uint32_t *route_seq, char *veth, + struct in_addr tunsrc, struct in_addr tundst) +{ + if (ip4_addr_set(route_sock, (*route_seq)++, "lo", + tunsrc, PREFIX_LEN)) { + printk("Failed to set ipv4 addr"); + return -1; + } + + if (ip4_route_set(route_sock, (*route_seq)++, veth, tunsrc, tundst)) { + printk("Failed to set ipv4 route"); + return -1; + } + + return 0; +} + +static int init_child(int nsfd, char *veth, unsigned int src, unsigned int dst) +{ + struct in_addr intsrc = inet_makeaddr(INADDR_B, src); + struct in_addr tunsrc = inet_makeaddr(INADDR_A, src); + struct in_addr tundst = inet_makeaddr(INADDR_A, dst); + int route_sock = -1, ret = -1; + uint32_t route_seq; + + if (switch_ns(nsfd)) + return -1; + + if (netlink_sock(&route_sock, &route_seq, NETLINK_ROUTE)) { + printk("Failed to open netlink route socket in child"); + return -1; + } + + if (ip4_addr_set(route_sock, route_seq++, veth, intsrc, PREFIX_LEN)) { + printk("Failed to set ipv4 addr"); + goto err; + } + + if (link_set_up(route_sock, route_seq++, veth)) { + printk("Failed to bring up %s", veth); + goto err; + } + + if (tunnel_set_route(route_sock, &route_seq, veth, tunsrc, tundst)) { + printk("Failed to add tunnel route on %s", veth); + goto err; + } + ret = 0; + +err: + close(route_sock); + return ret; +} + +#define ALGO_LEN 64 +enum desc_type { + CREATE_TUNNEL = 0, + ALLOCATE_SPI, + MONITOR_ACQUIRE, + EXPIRE_STATE, + EXPIRE_POLICY, +}; +const char *desc_name[] = { + "create tunnel", + "alloc spi", + "monitor acquire", + "expire state", + "expire policy" +}; +struct xfrm_desc { + enum desc_type type; + uint8_t proto; + char a_algo[ALGO_LEN]; + char e_algo[ALGO_LEN]; + char c_algo[ALGO_LEN]; + char ae_algo[ALGO_LEN]; + unsigned int icv_len; + /* unsigned key_len; */ +}; + +enum msg_type { + MSG_ACK = 0, + MSG_EXIT, + MSG_PING, + MSG_XFRM_PREPARE, + MSG_XFRM_ADD, + MSG_XFRM_DEL, + MSG_XFRM_CLEANUP, +}; + +struct test_desc { + enum msg_type type; + union { + struct { + in_addr_t reply_ip; + unsigned int port; + } ping; + struct xfrm_desc xfrm_desc; + } body; +}; + +struct test_result { + struct xfrm_desc desc; + unsigned int res; +}; + +static void write_test_result(unsigned int res, struct xfrm_desc *d) +{ + struct test_result tr = {}; + ssize_t ret; + + tr.desc = *d; + tr.res = res; + + ret = write(results_fd[1], &tr, sizeof(tr)); + if (ret != sizeof(tr)) + pr_err("Failed to write the result in pipe %zd", ret); +} + +static void write_msg(int fd, struct test_desc *msg, bool exit_of_fail) +{ + ssize_t bytes = write(fd, msg, sizeof(*msg)); + + /* Make sure that write/read is atomic to a pipe */ + BUILD_BUG_ON(sizeof(struct test_desc) > PIPE_BUF); + + if (bytes < 0) { + pr_err("write()"); + if (exit_of_fail) + exit(KSFT_FAIL); + } + if (bytes != sizeof(*msg)) { + pr_err("sent part of the message %zd/%zu", bytes, sizeof(*msg)); + if (exit_of_fail) + exit(KSFT_FAIL); + } +} + +static void read_msg(int fd, struct test_desc *msg, bool exit_of_fail) +{ + ssize_t bytes = read(fd, msg, sizeof(*msg)); + + if (bytes < 0) { + pr_err("read()"); + if (exit_of_fail) + exit(KSFT_FAIL); + } + if (bytes != sizeof(*msg)) { + pr_err("got incomplete message %zd/%zu", bytes, sizeof(*msg)); + if (exit_of_fail) + exit(KSFT_FAIL); + } +} + +static int udp_ping_init(struct in_addr listen_ip, unsigned int u_timeout, + unsigned int *server_port, int sock[2]) +{ + struct sockaddr_in server; + struct timeval t = { .tv_sec = 0, .tv_usec = u_timeout }; + socklen_t s_len = sizeof(server); + + sock[0] = socket(AF_INET, SOCK_DGRAM, 0); + if (sock[0] < 0) { + pr_err("socket()"); + return -1; + } + + server.sin_family = AF_INET; + server.sin_port = 0; + memcpy(&server.sin_addr.s_addr, &listen_ip, sizeof(struct in_addr)); + + if (bind(sock[0], (struct sockaddr *)&server, s_len)) { + pr_err("bind()"); + goto err_close_server; + } + + if (getsockname(sock[0], (struct sockaddr *)&server, &s_len)) { + pr_err("getsockname()"); + goto err_close_server; + } + + *server_port = ntohs(server.sin_port); + + if (setsockopt(sock[0], SOL_SOCKET, SO_RCVTIMEO, (const char *)&t, sizeof t)) { + pr_err("setsockopt()"); + goto err_close_server; + } + + sock[1] = socket(AF_INET, SOCK_DGRAM, 0); + if (sock[1] < 0) { + pr_err("socket()"); + goto err_close_server; + } + + return 0; + +err_close_server: + close(sock[0]); + return -1; +} + +static int udp_ping_send(int sock[2], in_addr_t dest_ip, unsigned int port, + char *buf, size_t buf_len) +{ + struct sockaddr_in server; + const struct sockaddr *dest_addr = (struct sockaddr *)&server; + char *sock_buf[buf_len]; + ssize_t r_bytes, s_bytes; + + server.sin_family = AF_INET; + server.sin_port = htons(port); + server.sin_addr.s_addr = dest_ip; + + s_bytes = sendto(sock[1], buf, buf_len, 0, dest_addr, sizeof(server)); + if (s_bytes < 0) { + pr_err("sendto()"); + return -1; + } else if (s_bytes != buf_len) { + printk("send part of the message: %zd/%zu", s_bytes, sizeof(server)); + return -1; + } + + r_bytes = recv(sock[0], sock_buf, buf_len, 0); + if (r_bytes < 0) { + if (errno != EAGAIN) + pr_err("recv()"); + return -1; + } else if (r_bytes == 0) { /* EOF */ + printk("EOF on reply to ping"); + return -1; + } else if (r_bytes != buf_len || memcmp(buf, sock_buf, buf_len)) { + printk("ping reply packet is corrupted %zd/%zu", r_bytes, buf_len); + return -1; + } + + return 0; +} + +static int udp_ping_reply(int sock[2], in_addr_t dest_ip, unsigned int port, + char *buf, size_t buf_len) +{ + struct sockaddr_in server; + const struct sockaddr *dest_addr = (struct sockaddr *)&server; + char *sock_buf[buf_len]; + ssize_t r_bytes, s_bytes; + + server.sin_family = AF_INET; + server.sin_port = htons(port); + server.sin_addr.s_addr = dest_ip; + + r_bytes = recv(sock[0], sock_buf, buf_len, 0); + if (r_bytes < 0) { + if (errno != EAGAIN) + pr_err("recv()"); + return -1; + } + if (r_bytes == 0) { /* EOF */ + printk("EOF on reply to ping"); + return -1; + } + if (r_bytes != buf_len || memcmp(buf, sock_buf, buf_len)) { + printk("ping reply packet is corrupted %zd/%zu", r_bytes, buf_len); + return -1; + } + + s_bytes = sendto(sock[1], buf, buf_len, 0, dest_addr, sizeof(server)); + if (s_bytes < 0) { + pr_err("sendto()"); + return -1; + } else if (s_bytes != buf_len) { + printk("send part of the message: %zd/%zu", s_bytes, sizeof(server)); + return -1; + } + + return 0; +} + +typedef int (*ping_f)(int sock[2], in_addr_t dest_ip, unsigned int port, + char *buf, size_t buf_len); +static int do_ping(int cmd_fd, char *buf, size_t buf_len, struct in_addr from, + bool init_side, int d_port, in_addr_t to, ping_f func) +{ + struct test_desc msg; + unsigned int s_port, i, ping_succeeded = 0; + int ping_sock[2]; + char to_str[IPV4_STR_SZ] = {}, from_str[IPV4_STR_SZ] = {}; + + if (udp_ping_init(from, ping_timeout, &s_port, ping_sock)) { + printk("Failed to init ping"); + return -1; + } + + memset(&msg, 0, sizeof(msg)); + msg.type = MSG_PING; + msg.body.ping.port = s_port; + memcpy(&msg.body.ping.reply_ip, &from, sizeof(from)); + + write_msg(cmd_fd, &msg, 0); + if (init_side) { + /* The other end sends ip to ping */ + read_msg(cmd_fd, &msg, 0); + if (msg.type != MSG_PING) + return -1; + to = msg.body.ping.reply_ip; + d_port = msg.body.ping.port; + } + + for (i = 0; i < ping_count ; i++) { + struct timespec sleep_time = { + .tv_sec = 0, + .tv_nsec = ping_delay_nsec, + }; + + ping_succeeded += !func(ping_sock, to, d_port, buf, page_size); + nanosleep(&sleep_time, 0); + } + + close(ping_sock[0]); + close(ping_sock[1]); + + strncpy(to_str, inet_ntoa(*(struct in_addr *)&to), IPV4_STR_SZ - 1); + strncpy(from_str, inet_ntoa(from), IPV4_STR_SZ - 1); + + if (ping_succeeded < ping_success) { + printk("ping (%s) %s->%s failed %u/%u times", + init_side ? "send" : "reply", from_str, to_str, + ping_count - ping_succeeded, ping_count); + return -1; + } + +#ifdef DEBUG + printk("ping (%s) %s->%s succeeded %u/%u times", + init_side ? "send" : "reply", from_str, to_str, + ping_succeeded, ping_count); +#endif + + return 0; +} + +static int xfrm_fill_key(char *name, char *buf, + size_t buf_len, unsigned int *key_len) +{ + /* TODO: use set/map instead */ + if (strncmp(name, "digest_null", ALGO_LEN) == 0) + *key_len = 0; + else if (strncmp(name, "ecb(cipher_null)", ALGO_LEN) == 0) + *key_len = 0; + else if (strncmp(name, "cbc(des)", ALGO_LEN) == 0) + *key_len = 64; + else if (strncmp(name, "hmac(md5)", ALGO_LEN) == 0) + *key_len = 128; + else if (strncmp(name, "cmac(aes)", ALGO_LEN) == 0) + *key_len = 128; + else if (strncmp(name, "xcbc(aes)", ALGO_LEN) == 0) + *key_len = 128; + else if (strncmp(name, "cbc(cast5)", ALGO_LEN) == 0) + *key_len = 128; + else if (strncmp(name, "cbc(serpent)", ALGO_LEN) == 0) + *key_len = 128; + else if (strncmp(name, "hmac(sha1)", ALGO_LEN) == 0) + *key_len = 160; + else if (strncmp(name, "hmac(rmd160)", ALGO_LEN) == 0) + *key_len = 160; + else if (strncmp(name, "cbc(des3_ede)", ALGO_LEN) == 0) + *key_len = 192; + else if (strncmp(name, "hmac(sha256)", ALGO_LEN) == 0) + *key_len = 256; + else if (strncmp(name, "cbc(aes)", ALGO_LEN) == 0) + *key_len = 256; + else if (strncmp(name, "cbc(camellia)", ALGO_LEN) == 0) + *key_len = 256; + else if (strncmp(name, "cbc(twofish)", ALGO_LEN) == 0) + *key_len = 256; + else if (strncmp(name, "rfc3686(ctr(aes))", ALGO_LEN) == 0) + *key_len = 288; + else if (strncmp(name, "hmac(sha384)", ALGO_LEN) == 0) + *key_len = 384; + else if (strncmp(name, "cbc(blowfish)", ALGO_LEN) == 0) + *key_len = 448; + else if (strncmp(name, "hmac(sha512)", ALGO_LEN) == 0) + *key_len = 512; + else if (strncmp(name, "rfc4106(gcm(aes))-128", ALGO_LEN) == 0) + *key_len = 160; + else if (strncmp(name, "rfc4543(gcm(aes))-128", ALGO_LEN) == 0) + *key_len = 160; + else if (strncmp(name, "rfc4309(ccm(aes))-128", ALGO_LEN) == 0) + *key_len = 152; + else if (strncmp(name, "rfc4106(gcm(aes))-192", ALGO_LEN) == 0) + *key_len = 224; + else if (strncmp(name, "rfc4543(gcm(aes))-192", ALGO_LEN) == 0) + *key_len = 224; + else if (strncmp(name, "rfc4309(ccm(aes))-192", ALGO_LEN) == 0) + *key_len = 216; + else if (strncmp(name, "rfc4106(gcm(aes))-256", ALGO_LEN) == 0) + *key_len = 288; + else if (strncmp(name, "rfc4543(gcm(aes))-256", ALGO_LEN) == 0) + *key_len = 288; + else if (strncmp(name, "rfc4309(ccm(aes))-256", ALGO_LEN) == 0) + *key_len = 280; + else if (strncmp(name, "rfc7539(chacha20,poly1305)-128", ALGO_LEN) == 0) + *key_len = 0; + + if (*key_len > buf_len) { + printk("Can't pack a key - too big for buffer"); + return -1; + } + + randomize_buffer(buf, *key_len); + + return 0; +} + +static int xfrm_state_pack_algo(struct nlmsghdr *nh, size_t req_sz, + struct xfrm_desc *desc) +{ + struct { + union { + struct xfrm_algo alg; + struct xfrm_algo_aead aead; + struct xfrm_algo_auth auth; + } u; + char buf[XFRM_ALGO_KEY_BUF_SIZE]; + } alg = {}; + size_t alen, elen, clen, aelen; + unsigned short type; + + alen = strlen(desc->a_algo); + elen = strlen(desc->e_algo); + clen = strlen(desc->c_algo); + aelen = strlen(desc->ae_algo); + + /* Verify desc */ + switch (desc->proto) { + case IPPROTO_AH: + if (!alen || elen || clen || aelen) { + printk("BUG: buggy ah desc"); + return -1; + } + strncpy(alg.u.alg.alg_name, desc->a_algo, ALGO_LEN - 1); + if (xfrm_fill_key(desc->a_algo, alg.u.alg.alg_key, + sizeof(alg.buf), &alg.u.alg.alg_key_len)) + return -1; + type = XFRMA_ALG_AUTH; + break; + case IPPROTO_COMP: + if (!clen || elen || alen || aelen) { + printk("BUG: buggy comp desc"); + return -1; + } + strncpy(alg.u.alg.alg_name, desc->c_algo, ALGO_LEN - 1); + if (xfrm_fill_key(desc->c_algo, alg.u.alg.alg_key, + sizeof(alg.buf), &alg.u.alg.alg_key_len)) + return -1; + type = XFRMA_ALG_COMP; + break; + case IPPROTO_ESP: + if (!((alen && elen) ^ aelen) || clen) { + printk("BUG: buggy esp desc"); + return -1; + } + if (aelen) { + alg.u.aead.alg_icv_len = desc->icv_len; + strncpy(alg.u.aead.alg_name, desc->ae_algo, ALGO_LEN - 1); + if (xfrm_fill_key(desc->ae_algo, alg.u.aead.alg_key, + sizeof(alg.buf), &alg.u.aead.alg_key_len)) + return -1; + type = XFRMA_ALG_AEAD; + } else { + + strncpy(alg.u.alg.alg_name, desc->e_algo, ALGO_LEN - 1); + type = XFRMA_ALG_CRYPT; + if (xfrm_fill_key(desc->e_algo, alg.u.alg.alg_key, + sizeof(alg.buf), &alg.u.alg.alg_key_len)) + return -1; + if (rtattr_pack(nh, req_sz, type, &alg, sizeof(alg))) + return -1; + + strncpy(alg.u.alg.alg_name, desc->a_algo, ALGO_LEN); + type = XFRMA_ALG_AUTH; + if (xfrm_fill_key(desc->a_algo, alg.u.alg.alg_key, + sizeof(alg.buf), &alg.u.alg.alg_key_len)) + return -1; + } + break; + default: + printk("BUG: unknown proto in desc"); + return -1; + } + + if (rtattr_pack(nh, req_sz, type, &alg, sizeof(alg))) + return -1; + + return 0; +} + +static inline uint32_t gen_spi(struct in_addr src) +{ + return htonl(inet_lnaof(src)); +} + +static int xfrm_state_add(int xfrm_sock, uint32_t seq, uint32_t spi, + struct in_addr src, struct in_addr dst, + struct xfrm_desc *desc) +{ + struct { + struct nlmsghdr nh; + struct xfrm_usersa_info info; + char attrbuf[MAX_PAYLOAD]; + } req; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)); + req.nh.nlmsg_type = XFRM_MSG_NEWSA; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.nh.nlmsg_seq = seq; + + /* Fill selector. */ + memcpy(&req.info.sel.daddr, &dst, sizeof(dst)); + memcpy(&req.info.sel.saddr, &src, sizeof(src)); + req.info.sel.family = AF_INET; + req.info.sel.prefixlen_d = PREFIX_LEN; + req.info.sel.prefixlen_s = PREFIX_LEN; + + /* Fill id */ + memcpy(&req.info.id.daddr, &dst, sizeof(dst)); + /* Note: zero-spi cannot be deleted */ + req.info.id.spi = spi; + req.info.id.proto = desc->proto; + + memcpy(&req.info.saddr, &src, sizeof(src)); + + /* Fill lifteme_cfg */ + req.info.lft.soft_byte_limit = XFRM_INF; + req.info.lft.hard_byte_limit = XFRM_INF; + req.info.lft.soft_packet_limit = XFRM_INF; + req.info.lft.hard_packet_limit = XFRM_INF; + + req.info.family = AF_INET; + req.info.mode = XFRM_MODE_TUNNEL; + + if (xfrm_state_pack_algo(&req.nh, sizeof(req), desc)) + return -1; + + if (send(xfrm_sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + return -1; + } + + return netlink_check_answer(xfrm_sock); +} + +static bool xfrm_usersa_found(struct xfrm_usersa_info *info, uint32_t spi, + struct in_addr src, struct in_addr dst, + struct xfrm_desc *desc) +{ + if (memcmp(&info->sel.daddr, &dst, sizeof(dst))) + return false; + + if (memcmp(&info->sel.saddr, &src, sizeof(src))) + return false; + + if (info->sel.family != AF_INET || + info->sel.prefixlen_d != PREFIX_LEN || + info->sel.prefixlen_s != PREFIX_LEN) + return false; + + if (info->id.spi != spi || info->id.proto != desc->proto) + return false; + + if (memcmp(&info->id.daddr, &dst, sizeof(dst))) + return false; + + if (memcmp(&info->saddr, &src, sizeof(src))) + return false; + + if (info->lft.soft_byte_limit != XFRM_INF || + info->lft.hard_byte_limit != XFRM_INF || + info->lft.soft_packet_limit != XFRM_INF || + info->lft.hard_packet_limit != XFRM_INF) + return false; + + if (info->family != AF_INET || info->mode != XFRM_MODE_TUNNEL) + return false; + + /* XXX: check xfrm algo, see xfrm_state_pack_algo(). */ + + return true; +} + +static int xfrm_state_check(int xfrm_sock, uint32_t seq, uint32_t spi, + struct in_addr src, struct in_addr dst, + struct xfrm_desc *desc) +{ + struct { + struct nlmsghdr nh; + char attrbuf[MAX_PAYLOAD]; + } req; + struct { + struct nlmsghdr nh; + union { + struct xfrm_usersa_info info; + int error; + }; + char attrbuf[MAX_PAYLOAD]; + } answer; + struct xfrm_address_filter filter = {}; + bool found = false; + + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(0); + req.nh.nlmsg_type = XFRM_MSG_GETSA; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP; + req.nh.nlmsg_seq = seq; + + /* + * Add dump filter by source address as there may be other tunnels + * in this netns (if tests run in parallel). + */ + filter.family = AF_INET; + filter.splen = 0x1f; /* 0xffffffff mask see addr_match() */ + memcpy(&filter.saddr, &src, sizeof(src)); + if (rtattr_pack(&req.nh, sizeof(req), XFRMA_ADDRESS_FILTER, + &filter, sizeof(filter))) + return -1; + + if (send(xfrm_sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + return -1; + } + + while (1) { + if (recv(xfrm_sock, &answer, sizeof(answer), 0) < 0) { + pr_err("recv()"); + return -1; + } + if (answer.nh.nlmsg_type == NLMSG_ERROR) { + printk("NLMSG_ERROR: %d: %s", + answer.error, strerror(-answer.error)); + return -1; + } else if (answer.nh.nlmsg_type == NLMSG_DONE) { + if (found) + return 0; + printk("didn't find allocated xfrm state in dump"); + return -1; + } else if (answer.nh.nlmsg_type == XFRM_MSG_NEWSA) { + if (xfrm_usersa_found(&answer.info, spi, src, dst, desc)) + found = true; + } + } +} + +static int xfrm_set(int xfrm_sock, uint32_t *seq, + struct in_addr src, struct in_addr dst, + struct in_addr tunsrc, struct in_addr tundst, + struct xfrm_desc *desc) +{ + int err; + + err = xfrm_state_add(xfrm_sock, (*seq)++, gen_spi(src), src, dst, desc); + if (err) { + printk("Failed to add xfrm state"); + return -1; + } + + err = xfrm_state_add(xfrm_sock, (*seq)++, gen_spi(src), dst, src, desc); + if (err) { + printk("Failed to add xfrm state"); + return -1; + } + + /* Check dumps for XFRM_MSG_GETSA */ + err = xfrm_state_check(xfrm_sock, (*seq)++, gen_spi(src), src, dst, desc); + err |= xfrm_state_check(xfrm_sock, (*seq)++, gen_spi(src), dst, src, desc); + if (err) { + printk("Failed to check xfrm state"); + return -1; + } + + return 0; +} + +static int xfrm_policy_add(int xfrm_sock, uint32_t seq, uint32_t spi, + struct in_addr src, struct in_addr dst, uint8_t dir, + struct in_addr tunsrc, struct in_addr tundst, uint8_t proto) +{ + struct { + struct nlmsghdr nh; + struct xfrm_userpolicy_info info; + char attrbuf[MAX_PAYLOAD]; + } req; + struct xfrm_user_tmpl tmpl; + + memset(&req, 0, sizeof(req)); + memset(&tmpl, 0, sizeof(tmpl)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.info)); + req.nh.nlmsg_type = XFRM_MSG_NEWPOLICY; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.nh.nlmsg_seq = seq; + + /* Fill selector. */ + memcpy(&req.info.sel.daddr, &dst, sizeof(tundst)); + memcpy(&req.info.sel.saddr, &src, sizeof(tunsrc)); + req.info.sel.family = AF_INET; + req.info.sel.prefixlen_d = PREFIX_LEN; + req.info.sel.prefixlen_s = PREFIX_LEN; + + /* Fill lifteme_cfg */ + req.info.lft.soft_byte_limit = XFRM_INF; + req.info.lft.hard_byte_limit = XFRM_INF; + req.info.lft.soft_packet_limit = XFRM_INF; + req.info.lft.hard_packet_limit = XFRM_INF; + + req.info.dir = dir; + + /* Fill tmpl */ + memcpy(&tmpl.id.daddr, &dst, sizeof(dst)); + /* Note: zero-spi cannot be deleted */ + tmpl.id.spi = spi; + tmpl.id.proto = proto; + tmpl.family = AF_INET; + memcpy(&tmpl.saddr, &src, sizeof(src)); + tmpl.mode = XFRM_MODE_TUNNEL; + tmpl.aalgos = (~(uint32_t)0); + tmpl.ealgos = (~(uint32_t)0); + tmpl.calgos = (~(uint32_t)0); + + if (rtattr_pack(&req.nh, sizeof(req), XFRMA_TMPL, &tmpl, sizeof(tmpl))) + return -1; + + if (send(xfrm_sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + return -1; + } + + return netlink_check_answer(xfrm_sock); +} + +static int xfrm_prepare(int xfrm_sock, uint32_t *seq, + struct in_addr src, struct in_addr dst, + struct in_addr tunsrc, struct in_addr tundst, uint8_t proto) +{ + if (xfrm_policy_add(xfrm_sock, (*seq)++, gen_spi(src), src, dst, + XFRM_POLICY_OUT, tunsrc, tundst, proto)) { + printk("Failed to add xfrm policy"); + return -1; + } + + if (xfrm_policy_add(xfrm_sock, (*seq)++, gen_spi(src), dst, src, + XFRM_POLICY_IN, tunsrc, tundst, proto)) { + printk("Failed to add xfrm policy"); + return -1; + } + + return 0; +} + +static int xfrm_policy_del(int xfrm_sock, uint32_t seq, + struct in_addr src, struct in_addr dst, uint8_t dir, + struct in_addr tunsrc, struct in_addr tundst) +{ + struct { + struct nlmsghdr nh; + struct xfrm_userpolicy_id id; + char attrbuf[MAX_PAYLOAD]; + } req; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.id)); + req.nh.nlmsg_type = XFRM_MSG_DELPOLICY; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.nh.nlmsg_seq = seq; + + /* Fill id */ + memcpy(&req.id.sel.daddr, &dst, sizeof(tundst)); + memcpy(&req.id.sel.saddr, &src, sizeof(tunsrc)); + req.id.sel.family = AF_INET; + req.id.sel.prefixlen_d = PREFIX_LEN; + req.id.sel.prefixlen_s = PREFIX_LEN; + req.id.dir = dir; + + if (send(xfrm_sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + return -1; + } + + return netlink_check_answer(xfrm_sock); +} + +static int xfrm_cleanup(int xfrm_sock, uint32_t *seq, + struct in_addr src, struct in_addr dst, + struct in_addr tunsrc, struct in_addr tundst) +{ + if (xfrm_policy_del(xfrm_sock, (*seq)++, src, dst, + XFRM_POLICY_OUT, tunsrc, tundst)) { + printk("Failed to add xfrm policy"); + return -1; + } + + if (xfrm_policy_del(xfrm_sock, (*seq)++, dst, src, + XFRM_POLICY_IN, tunsrc, tundst)) { + printk("Failed to add xfrm policy"); + return -1; + } + + return 0; +} + +static int xfrm_state_del(int xfrm_sock, uint32_t seq, uint32_t spi, + struct in_addr src, struct in_addr dst, uint8_t proto) +{ + struct { + struct nlmsghdr nh; + struct xfrm_usersa_id id; + char attrbuf[MAX_PAYLOAD]; + } req; + xfrm_address_t saddr = {}; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.id)); + req.nh.nlmsg_type = XFRM_MSG_DELSA; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.nh.nlmsg_seq = seq; + + memcpy(&req.id.daddr, &dst, sizeof(dst)); + req.id.family = AF_INET; + req.id.proto = proto; + /* Note: zero-spi cannot be deleted */ + req.id.spi = spi; + + memcpy(&saddr, &src, sizeof(src)); + if (rtattr_pack(&req.nh, sizeof(req), XFRMA_SRCADDR, &saddr, sizeof(saddr))) + return -1; + + if (send(xfrm_sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + return -1; + } + + return netlink_check_answer(xfrm_sock); +} + +static int xfrm_delete(int xfrm_sock, uint32_t *seq, + struct in_addr src, struct in_addr dst, + struct in_addr tunsrc, struct in_addr tundst, uint8_t proto) +{ + if (xfrm_state_del(xfrm_sock, (*seq)++, gen_spi(src), src, dst, proto)) { + printk("Failed to remove xfrm state"); + return -1; + } + + if (xfrm_state_del(xfrm_sock, (*seq)++, gen_spi(src), dst, src, proto)) { + printk("Failed to remove xfrm state"); + return -1; + } + + return 0; +} + +static int xfrm_state_allocspi(int xfrm_sock, uint32_t *seq, + uint32_t spi, uint8_t proto) +{ + struct { + struct nlmsghdr nh; + struct xfrm_userspi_info spi; + } req; + struct { + struct nlmsghdr nh; + union { + struct xfrm_usersa_info info; + int error; + }; + } answer; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.spi)); + req.nh.nlmsg_type = XFRM_MSG_ALLOCSPI; + req.nh.nlmsg_flags = NLM_F_REQUEST; + req.nh.nlmsg_seq = (*seq)++; + + req.spi.info.family = AF_INET; + req.spi.min = spi; + req.spi.max = spi; + req.spi.info.id.proto = proto; + + if (send(xfrm_sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + return KSFT_FAIL; + } + + if (recv(xfrm_sock, &answer, sizeof(answer), 0) < 0) { + pr_err("recv()"); + return KSFT_FAIL; + } else if (answer.nh.nlmsg_type == XFRM_MSG_NEWSA) { + uint32_t new_spi = htonl(answer.info.id.spi); + + if (new_spi != spi) { + printk("allocated spi is different from requested: %#x != %#x", + new_spi, spi); + return KSFT_FAIL; + } + return KSFT_PASS; + } else if (answer.nh.nlmsg_type != NLMSG_ERROR) { + printk("expected NLMSG_ERROR, got %d", (int)answer.nh.nlmsg_type); + return KSFT_FAIL; + } + + printk("NLMSG_ERROR: %d: %s", answer.error, strerror(-answer.error)); + return (answer.error) ? KSFT_FAIL : KSFT_PASS; +} + +static int netlink_sock_bind(int *sock, uint32_t *seq, int proto, uint32_t groups) +{ + struct sockaddr_nl snl = {}; + socklen_t addr_len; + int ret = -1; + + snl.nl_family = AF_NETLINK; + snl.nl_groups = groups; + + if (netlink_sock(sock, seq, proto)) { + printk("Failed to open xfrm netlink socket"); + return -1; + } + + if (bind(*sock, (struct sockaddr *)&snl, sizeof(snl)) < 0) { + pr_err("bind()"); + goto out_close; + } + + addr_len = sizeof(snl); + if (getsockname(*sock, (struct sockaddr *)&snl, &addr_len) < 0) { + pr_err("getsockname()"); + goto out_close; + } + if (addr_len != sizeof(snl)) { + printk("Wrong address length %d", addr_len); + goto out_close; + } + if (snl.nl_family != AF_NETLINK) { + printk("Wrong address family %d", snl.nl_family); + goto out_close; + } + return 0; + +out_close: + close(*sock); + return ret; +} + +static int xfrm_monitor_acquire(int xfrm_sock, uint32_t *seq, unsigned int nr) +{ + struct { + struct nlmsghdr nh; + union { + struct xfrm_user_acquire acq; + int error; + }; + char attrbuf[MAX_PAYLOAD]; + } req; + struct xfrm_user_tmpl xfrm_tmpl = {}; + int xfrm_listen = -1, ret = KSFT_FAIL; + uint32_t seq_listen; + + if (netlink_sock_bind(&xfrm_listen, &seq_listen, NETLINK_XFRM, XFRMNLGRP_ACQUIRE)) + return KSFT_FAIL; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.acq)); + req.nh.nlmsg_type = XFRM_MSG_ACQUIRE; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.nh.nlmsg_seq = (*seq)++; + + req.acq.policy.sel.family = AF_INET; + req.acq.aalgos = 0xfeed; + req.acq.ealgos = 0xbaad; + req.acq.calgos = 0xbabe; + + xfrm_tmpl.family = AF_INET; + xfrm_tmpl.id.proto = IPPROTO_ESP; + if (rtattr_pack(&req.nh, sizeof(req), XFRMA_TMPL, &xfrm_tmpl, sizeof(xfrm_tmpl))) + goto out_close; + + if (send(xfrm_sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + goto out_close; + } + + if (recv(xfrm_sock, &req, sizeof(req), 0) < 0) { + pr_err("recv()"); + goto out_close; + } else if (req.nh.nlmsg_type != NLMSG_ERROR) { + printk("expected NLMSG_ERROR, got %d", (int)req.nh.nlmsg_type); + goto out_close; + } + + if (req.error) { + printk("NLMSG_ERROR: %d: %s", req.error, strerror(-req.error)); + ret = req.error; + goto out_close; + } + + if (recv(xfrm_listen, &req, sizeof(req), 0) < 0) { + pr_err("recv()"); + goto out_close; + } + + if (req.acq.aalgos != 0xfeed || req.acq.ealgos != 0xbaad + || req.acq.calgos != 0xbabe) { + printk("xfrm_user_acquire has changed %x %x %x", + req.acq.aalgos, req.acq.ealgos, req.acq.calgos); + goto out_close; + } + + ret = KSFT_PASS; +out_close: + close(xfrm_listen); + return ret; +} + +static int xfrm_expire_state(int xfrm_sock, uint32_t *seq, + unsigned int nr, struct xfrm_desc *desc) +{ + struct { + struct nlmsghdr nh; + union { + struct xfrm_user_expire expire; + int error; + }; + } req; + struct in_addr src, dst; + int xfrm_listen = -1, ret = KSFT_FAIL; + uint32_t seq_listen; + + src = inet_makeaddr(INADDR_B, child_ip(nr)); + dst = inet_makeaddr(INADDR_B, grchild_ip(nr)); + + if (xfrm_state_add(xfrm_sock, (*seq)++, gen_spi(src), src, dst, desc)) { + printk("Failed to add xfrm state"); + return KSFT_FAIL; + } + + if (netlink_sock_bind(&xfrm_listen, &seq_listen, NETLINK_XFRM, XFRMNLGRP_EXPIRE)) + return KSFT_FAIL; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.expire)); + req.nh.nlmsg_type = XFRM_MSG_EXPIRE; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.nh.nlmsg_seq = (*seq)++; + + memcpy(&req.expire.state.id.daddr, &dst, sizeof(dst)); + req.expire.state.id.spi = gen_spi(src); + req.expire.state.id.proto = desc->proto; + req.expire.state.family = AF_INET; + req.expire.hard = 0xff; + + if (send(xfrm_sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + goto out_close; + } + + if (recv(xfrm_sock, &req, sizeof(req), 0) < 0) { + pr_err("recv()"); + goto out_close; + } else if (req.nh.nlmsg_type != NLMSG_ERROR) { + printk("expected NLMSG_ERROR, got %d", (int)req.nh.nlmsg_type); + goto out_close; + } + + if (req.error) { + printk("NLMSG_ERROR: %d: %s", req.error, strerror(-req.error)); + ret = req.error; + goto out_close; + } + + if (recv(xfrm_listen, &req, sizeof(req), 0) < 0) { + pr_err("recv()"); + goto out_close; + } + + if (req.expire.hard != 0x1) { + printk("expire.hard is not set: %x", req.expire.hard); + goto out_close; + } + + ret = KSFT_PASS; +out_close: + close(xfrm_listen); + return ret; +} + +static int xfrm_expire_policy(int xfrm_sock, uint32_t *seq, + unsigned int nr, struct xfrm_desc *desc) +{ + struct { + struct nlmsghdr nh; + union { + struct xfrm_user_polexpire expire; + int error; + }; + } req; + struct in_addr src, dst, tunsrc, tundst; + int xfrm_listen = -1, ret = KSFT_FAIL; + uint32_t seq_listen; + + src = inet_makeaddr(INADDR_B, child_ip(nr)); + dst = inet_makeaddr(INADDR_B, grchild_ip(nr)); + tunsrc = inet_makeaddr(INADDR_A, child_ip(nr)); + tundst = inet_makeaddr(INADDR_A, grchild_ip(nr)); + + if (xfrm_policy_add(xfrm_sock, (*seq)++, gen_spi(src), src, dst, + XFRM_POLICY_OUT, tunsrc, tundst, desc->proto)) { + printk("Failed to add xfrm policy"); + return KSFT_FAIL; + } + + if (netlink_sock_bind(&xfrm_listen, &seq_listen, NETLINK_XFRM, XFRMNLGRP_EXPIRE)) + return KSFT_FAIL; + + memset(&req, 0, sizeof(req)); + req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(req.expire)); + req.nh.nlmsg_type = XFRM_MSG_POLEXPIRE; + req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; + req.nh.nlmsg_seq = (*seq)++; + + /* Fill selector. */ + memcpy(&req.expire.pol.sel.daddr, &dst, sizeof(tundst)); + memcpy(&req.expire.pol.sel.saddr, &src, sizeof(tunsrc)); + req.expire.pol.sel.family = AF_INET; + req.expire.pol.sel.prefixlen_d = PREFIX_LEN; + req.expire.pol.sel.prefixlen_s = PREFIX_LEN; + req.expire.pol.dir = XFRM_POLICY_OUT; + req.expire.hard = 0xff; + + if (send(xfrm_sock, &req, req.nh.nlmsg_len, 0) < 0) { + pr_err("send()"); + goto out_close; + } + + if (recv(xfrm_sock, &req, sizeof(req), 0) < 0) { + pr_err("recv()"); + goto out_close; + } else if (req.nh.nlmsg_type != NLMSG_ERROR) { + printk("expected NLMSG_ERROR, got %d", (int)req.nh.nlmsg_type); + goto out_close; + } + + if (req.error) { + printk("NLMSG_ERROR: %d: %s", req.error, strerror(-req.error)); + ret = req.error; + goto out_close; + } + + if (recv(xfrm_listen, &req, sizeof(req), 0) < 0) { + pr_err("recv()"); + goto out_close; + } + + if (req.expire.hard != 0x1) { + printk("expire.hard is not set: %x", req.expire.hard); + goto out_close; + } + + ret = KSFT_PASS; +out_close: + close(xfrm_listen); + return ret; +} + +static int child_serv(int xfrm_sock, uint32_t *seq, + unsigned int nr, int cmd_fd, void *buf, struct xfrm_desc *desc) +{ + struct in_addr src, dst, tunsrc, tundst; + struct test_desc msg; + int ret = KSFT_FAIL; + + src = inet_makeaddr(INADDR_B, child_ip(nr)); + dst = inet_makeaddr(INADDR_B, grchild_ip(nr)); + tunsrc = inet_makeaddr(INADDR_A, child_ip(nr)); + tundst = inet_makeaddr(INADDR_A, grchild_ip(nr)); + + /* UDP pinging without xfrm */ + if (do_ping(cmd_fd, buf, page_size, src, true, 0, 0, udp_ping_send)) { + printk("ping failed before setting xfrm"); + return KSFT_FAIL; + } + + memset(&msg, 0, sizeof(msg)); + msg.type = MSG_XFRM_PREPARE; + memcpy(&msg.body.xfrm_desc, desc, sizeof(*desc)); + write_msg(cmd_fd, &msg, 1); + + if (xfrm_prepare(xfrm_sock, seq, src, dst, tunsrc, tundst, desc->proto)) { + printk("failed to prepare xfrm"); + goto cleanup; + } + + memset(&msg, 0, sizeof(msg)); + msg.type = MSG_XFRM_ADD; + memcpy(&msg.body.xfrm_desc, desc, sizeof(*desc)); + write_msg(cmd_fd, &msg, 1); + if (xfrm_set(xfrm_sock, seq, src, dst, tunsrc, tundst, desc)) { + printk("failed to set xfrm"); + goto delete; + } + + /* UDP pinging with xfrm tunnel */ + if (do_ping(cmd_fd, buf, page_size, tunsrc, + true, 0, 0, udp_ping_send)) { + printk("ping failed for xfrm"); + goto delete; + } + + ret = KSFT_PASS; +delete: + /* xfrm delete */ + memset(&msg, 0, sizeof(msg)); + msg.type = MSG_XFRM_DEL; + memcpy(&msg.body.xfrm_desc, desc, sizeof(*desc)); + write_msg(cmd_fd, &msg, 1); + + if (xfrm_delete(xfrm_sock, seq, src, dst, tunsrc, tundst, desc->proto)) { + printk("failed ping to remove xfrm"); + ret = KSFT_FAIL; + } + +cleanup: + memset(&msg, 0, sizeof(msg)); + msg.type = MSG_XFRM_CLEANUP; + memcpy(&msg.body.xfrm_desc, desc, sizeof(*desc)); + write_msg(cmd_fd, &msg, 1); + if (xfrm_cleanup(xfrm_sock, seq, src, dst, tunsrc, tundst)) { + printk("failed ping to cleanup xfrm"); + ret = KSFT_FAIL; + } + return ret; +} + +static int child_f(unsigned int nr, int test_desc_fd, int cmd_fd, void *buf) +{ + struct xfrm_desc desc; + struct test_desc msg; + int xfrm_sock = -1; + uint32_t seq; + + if (switch_ns(nsfd_childa)) + exit(KSFT_FAIL); + + if (netlink_sock(&xfrm_sock, &seq, NETLINK_XFRM)) { + printk("Failed to open xfrm netlink socket"); + exit(KSFT_FAIL); + } + + /* Check that seq sock is ready, just for sure. */ + memset(&msg, 0, sizeof(msg)); + msg.type = MSG_ACK; + write_msg(cmd_fd, &msg, 1); + read_msg(cmd_fd, &msg, 1); + if (msg.type != MSG_ACK) { + printk("Ack failed"); + exit(KSFT_FAIL); + } + + for (;;) { + ssize_t received = read(test_desc_fd, &desc, sizeof(desc)); + int ret; + + if (received == 0) /* EOF */ + break; + + if (received != sizeof(desc)) { + pr_err("read() returned %zd", received); + exit(KSFT_FAIL); + } + + switch (desc.type) { + case CREATE_TUNNEL: + ret = child_serv(xfrm_sock, &seq, nr, + cmd_fd, buf, &desc); + break; + case ALLOCATE_SPI: + ret = xfrm_state_allocspi(xfrm_sock, &seq, + -1, desc.proto); + break; + case MONITOR_ACQUIRE: + ret = xfrm_monitor_acquire(xfrm_sock, &seq, nr); + break; + case EXPIRE_STATE: + ret = xfrm_expire_state(xfrm_sock, &seq, nr, &desc); + break; + case EXPIRE_POLICY: + ret = xfrm_expire_policy(xfrm_sock, &seq, nr, &desc); + break; + default: + printk("Unknown desc type %d", desc.type); + exit(KSFT_FAIL); + } + write_test_result(ret, &desc); + } + + close(xfrm_sock); + + msg.type = MSG_EXIT; + write_msg(cmd_fd, &msg, 1); + exit(KSFT_PASS); +} + +static void grand_child_serv(unsigned int nr, int cmd_fd, void *buf, + struct test_desc *msg, int xfrm_sock, uint32_t *seq) +{ + struct in_addr src, dst, tunsrc, tundst; + bool tun_reply; + struct xfrm_desc *desc = &msg->body.xfrm_desc; + + src = inet_makeaddr(INADDR_B, grchild_ip(nr)); + dst = inet_makeaddr(INADDR_B, child_ip(nr)); + tunsrc = inet_makeaddr(INADDR_A, grchild_ip(nr)); + tundst = inet_makeaddr(INADDR_A, child_ip(nr)); + + switch (msg->type) { + case MSG_EXIT: + exit(KSFT_PASS); + case MSG_ACK: + write_msg(cmd_fd, msg, 1); + break; + case MSG_PING: + tun_reply = memcmp(&dst, &msg->body.ping.reply_ip, sizeof(in_addr_t)); + /* UDP pinging without xfrm */ + if (do_ping(cmd_fd, buf, page_size, tun_reply ? tunsrc : src, + false, msg->body.ping.port, + msg->body.ping.reply_ip, udp_ping_reply)) { + printk("ping failed before setting xfrm"); + } + break; + case MSG_XFRM_PREPARE: + if (xfrm_prepare(xfrm_sock, seq, src, dst, tunsrc, tundst, + desc->proto)) { + xfrm_cleanup(xfrm_sock, seq, src, dst, tunsrc, tundst); + printk("failed to prepare xfrm"); + } + break; + case MSG_XFRM_ADD: + if (xfrm_set(xfrm_sock, seq, src, dst, tunsrc, tundst, desc)) { + xfrm_cleanup(xfrm_sock, seq, src, dst, tunsrc, tundst); + printk("failed to set xfrm"); + } + break; + case MSG_XFRM_DEL: + if (xfrm_delete(xfrm_sock, seq, src, dst, tunsrc, tundst, + desc->proto)) { + xfrm_cleanup(xfrm_sock, seq, src, dst, tunsrc, tundst); + printk("failed to remove xfrm"); + } + break; + case MSG_XFRM_CLEANUP: + if (xfrm_cleanup(xfrm_sock, seq, src, dst, tunsrc, tundst)) { + printk("failed to cleanup xfrm"); + } + break; + default: + printk("got unknown msg type %d", msg->type); + }; +} + +static int grand_child_f(unsigned int nr, int cmd_fd, void *buf) +{ + struct test_desc msg; + int xfrm_sock = -1; + uint32_t seq; + + if (switch_ns(nsfd_childb)) + exit(KSFT_FAIL); + + if (netlink_sock(&xfrm_sock, &seq, NETLINK_XFRM)) { + printk("Failed to open xfrm netlink socket"); + exit(KSFT_FAIL); + } + + do { + read_msg(cmd_fd, &msg, 1); + grand_child_serv(nr, cmd_fd, buf, &msg, xfrm_sock, &seq); + } while (1); + + close(xfrm_sock); + exit(KSFT_FAIL); +} + +static int start_child(unsigned int nr, char *veth, int test_desc_fd[2]) +{ + int cmd_sock[2]; + void *data_map; + pid_t child; + + if (init_child(nsfd_childa, veth, child_ip(nr), grchild_ip(nr))) + return -1; + + if (init_child(nsfd_childb, veth, grchild_ip(nr), child_ip(nr))) + return -1; + + child = fork(); + if (child < 0) { + pr_err("fork()"); + return -1; + } else if (child) { + /* in parent - selftest */ + return switch_ns(nsfd_parent); + } + + if (close(test_desc_fd[1])) { + pr_err("close()"); + return -1; + } + + /* child */ + data_map = mmap(0, page_size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (data_map == MAP_FAILED) { + pr_err("mmap()"); + return -1; + } + + randomize_buffer(data_map, page_size); + + if (socketpair(PF_LOCAL, SOCK_SEQPACKET, 0, cmd_sock)) { + pr_err("socketpair()"); + return -1; + } + + child = fork(); + if (child < 0) { + pr_err("fork()"); + return -1; + } else if (child) { + if (close(cmd_sock[0])) { + pr_err("close()"); + return -1; + } + return child_f(nr, test_desc_fd[0], cmd_sock[1], data_map); + } + if (close(cmd_sock[1])) { + pr_err("close()"); + return -1; + } + return grand_child_f(nr, cmd_sock[0], data_map); +} + +static void exit_usage(char **argv) +{ + printk("Usage: %s [nr_process]", argv[0]); + exit(KSFT_FAIL); +} + +static int __write_desc(int test_desc_fd, struct xfrm_desc *desc) +{ + ssize_t ret; + + ret = write(test_desc_fd, desc, sizeof(*desc)); + + if (ret == sizeof(*desc)) + return 0; + + pr_err("Writing test's desc failed %ld", ret); + + return -1; +} + +static int write_desc(int proto, int test_desc_fd, + char *a, char *e, char *c, char *ae) +{ + struct xfrm_desc desc = {}; + + desc.type = CREATE_TUNNEL; + desc.proto = proto; + + if (a) + strncpy(desc.a_algo, a, ALGO_LEN - 1); + if (e) + strncpy(desc.e_algo, e, ALGO_LEN - 1); + if (c) + strncpy(desc.c_algo, c, ALGO_LEN - 1); + if (ae) + strncpy(desc.ae_algo, ae, ALGO_LEN - 1); + + return __write_desc(test_desc_fd, &desc); +} + +int proto_list[] = { IPPROTO_AH, IPPROTO_COMP, IPPROTO_ESP }; +char *ah_list[] = { + "digest_null", "hmac(md5)", "hmac(sha1)", "hmac(sha256)", + "hmac(sha384)", "hmac(sha512)", "hmac(rmd160)", + "xcbc(aes)", "cmac(aes)" +}; +char *comp_list[] = { + "deflate", +#if 0 + /* No compression backend realization */ + "lzs", "lzjh" +#endif +}; +char *e_list[] = { + "ecb(cipher_null)", "cbc(des)", "cbc(des3_ede)", "cbc(cast5)", + "cbc(blowfish)", "cbc(aes)", "cbc(serpent)", "cbc(camellia)", + "cbc(twofish)", "rfc3686(ctr(aes))" +}; +char *ae_list[] = { +#if 0 + /* not implemented */ + "rfc4106(gcm(aes))", "rfc4309(ccm(aes))", "rfc4543(gcm(aes))", + "rfc7539esp(chacha20,poly1305)" +#endif +}; + +const unsigned int proto_plan = ARRAY_SIZE(ah_list) + ARRAY_SIZE(comp_list) \ + + (ARRAY_SIZE(ah_list) * ARRAY_SIZE(e_list)) \ + + ARRAY_SIZE(ae_list); + +static int write_proto_plan(int fd, int proto) +{ + unsigned int i; + + switch (proto) { + case IPPROTO_AH: + for (i = 0; i < ARRAY_SIZE(ah_list); i++) { + if (write_desc(proto, fd, ah_list[i], 0, 0, 0)) + return -1; + } + break; + case IPPROTO_COMP: + for (i = 0; i < ARRAY_SIZE(comp_list); i++) { + if (write_desc(proto, fd, 0, 0, comp_list[i], 0)) + return -1; + } + break; + case IPPROTO_ESP: + for (i = 0; i < ARRAY_SIZE(ah_list); i++) { + int j; + + for (j = 0; j < ARRAY_SIZE(e_list); j++) { + if (write_desc(proto, fd, ah_list[i], + e_list[j], 0, 0)) + return -1; + } + } + for (i = 0; i < ARRAY_SIZE(ae_list); i++) { + if (write_desc(proto, fd, 0, 0, 0, ae_list[i])) + return -1; + } + break; + default: + printk("BUG: Specified unknown proto %d", proto); + return -1; + } + + return 0; +} + +/* + * Some structures in xfrm uapi header differ in size between + * 64-bit and 32-bit ABI: + * + * 32-bit UABI | 64-bit UABI + * -------------------------------------|------------------------------------- + * sizeof(xfrm_usersa_info) = 220 | sizeof(xfrm_usersa_info) = 224 + * sizeof(xfrm_userpolicy_info) = 164 | sizeof(xfrm_userpolicy_info) = 168 + * sizeof(xfrm_userspi_info) = 228 | sizeof(xfrm_userspi_info) = 232 + * sizeof(xfrm_user_acquire) = 276 | sizeof(xfrm_user_acquire) = 280 + * sizeof(xfrm_user_expire) = 224 | sizeof(xfrm_user_expire) = 232 + * sizeof(xfrm_user_polexpire) = 168 | sizeof(xfrm_user_polexpire) = 176 + * + * Check the affected by the UABI difference structures. + */ +const unsigned int compat_plan = 4; +static int write_compat_struct_tests(int test_desc_fd) +{ + struct xfrm_desc desc = {}; + + desc.type = ALLOCATE_SPI; + desc.proto = IPPROTO_AH; + strncpy(desc.a_algo, ah_list[0], ALGO_LEN - 1); + + if (__write_desc(test_desc_fd, &desc)) + return -1; + + desc.type = MONITOR_ACQUIRE; + if (__write_desc(test_desc_fd, &desc)) + return -1; + + desc.type = EXPIRE_STATE; + if (__write_desc(test_desc_fd, &desc)) + return -1; + + desc.type = EXPIRE_POLICY; + if (__write_desc(test_desc_fd, &desc)) + return -1; + + return 0; +} + +static int write_test_plan(int test_desc_fd) +{ + unsigned int i; + pid_t child; + + child = fork(); + if (child < 0) { + pr_err("fork()"); + return -1; + } + if (child) { + if (close(test_desc_fd)) + printk("close(): %m"); + return 0; + } + + if (write_compat_struct_tests(test_desc_fd)) + exit(KSFT_FAIL); + + for (i = 0; i < ARRAY_SIZE(proto_list); i++) { + if (write_proto_plan(test_desc_fd, proto_list[i])) + exit(KSFT_FAIL); + } + + exit(KSFT_PASS); +} + +static int children_cleanup(void) +{ + unsigned ret = KSFT_PASS; + + while (1) { + int status; + pid_t p = wait(&status); + + if ((p < 0) && errno == ECHILD) + break; + + if (p < 0) { + pr_err("wait()"); + return KSFT_FAIL; + } + + if (!WIFEXITED(status)) { + ret = KSFT_FAIL; + continue; + } + + if (WEXITSTATUS(status) == KSFT_FAIL) + ret = KSFT_FAIL; + } + + return ret; +} + +typedef void (*print_res)(const char *, ...); + +static int check_results(void) +{ + struct test_result tr = {}; + struct xfrm_desc *d = &tr.desc; + int ret = KSFT_PASS; + + while (1) { + ssize_t received = read(results_fd[0], &tr, sizeof(tr)); + print_res result; + + if (received == 0) /* EOF */ + break; + + if (received != sizeof(tr)) { + pr_err("read() returned %zd", received); + return KSFT_FAIL; + } + + switch (tr.res) { + case KSFT_PASS: + result = ksft_test_result_pass; + break; + case KSFT_FAIL: + default: + result = ksft_test_result_fail; + ret = KSFT_FAIL; + } + + result(" %s: [%u, '%s', '%s', '%s', '%s', %u]\n", + desc_name[d->type], (unsigned int)d->proto, d->a_algo, + d->e_algo, d->c_algo, d->ae_algo, d->icv_len); + } + + return ret; +} + +int main(int argc, char **argv) +{ + unsigned int nr_process = 1; + int route_sock = -1, ret = KSFT_SKIP; + int test_desc_fd[2]; + uint32_t route_seq; + unsigned int i; + + if (argc > 2) + exit_usage(argv); + + if (argc > 1) { + char *endptr; + + errno = 0; + nr_process = strtol(argv[1], &endptr, 10); + if ((errno == ERANGE && (nr_process == LONG_MAX || nr_process == LONG_MIN)) + || (errno != 0 && nr_process == 0) + || (endptr == argv[1]) || (*endptr != '\0')) { + printk("Failed to parse [nr_process]"); + exit_usage(argv); + } + + if (nr_process > MAX_PROCESSES || !nr_process) { + printk("nr_process should be between [1; %u]", + MAX_PROCESSES); + exit_usage(argv); + } + } + + srand(time(NULL)); + page_size = sysconf(_SC_PAGESIZE); + if (page_size < 1) + ksft_exit_skip("sysconf(): %m\n"); + + if (pipe2(test_desc_fd, O_DIRECT) < 0) + ksft_exit_skip("pipe(): %m\n"); + + if (pipe2(results_fd, O_DIRECT) < 0) + ksft_exit_skip("pipe(): %m\n"); + + if (init_namespaces()) + ksft_exit_skip("Failed to create namespaces\n"); + + if (netlink_sock(&route_sock, &route_seq, NETLINK_ROUTE)) + ksft_exit_skip("Failed to open netlink route socket\n"); + + for (i = 0; i < nr_process; i++) { + char veth[VETH_LEN]; + + snprintf(veth, VETH_LEN, VETH_FMT, i); + + if (veth_add(route_sock, route_seq++, veth, nsfd_childa, veth, nsfd_childb)) { + close(route_sock); + ksft_exit_fail_msg("Failed to create veth device"); + } + + if (start_child(i, veth, test_desc_fd)) { + close(route_sock); + ksft_exit_fail_msg("Child %u failed to start", i); + } + } + + if (close(route_sock) || close(test_desc_fd[0]) || close(results_fd[1])) + ksft_exit_fail_msg("close(): %m"); + + ksft_set_plan(proto_plan + compat_plan); + + if (write_test_plan(test_desc_fd[1])) + ksft_exit_fail_msg("Failed to write test plan to pipe"); + + ret = check_results(); + + if (children_cleanup() == KSFT_FAIL) + exit(KSFT_FAIL); + + exit(ret); +} |