diff options
699 files changed, 26645 insertions, 7333 deletions
@@ -627,6 +627,10 @@ S: 48287 Sawleaf S: Fremont, California 94539 S: USA +N: Tomas Cech +E: sleep_walker@suse.com +D: arm/palm treo support + N: Florent Chabaud E: florent.chabaud@polytechnique.org D: software suspend diff --git a/Documentation/ABI/testing/sysfs-devices-platform-soc-ipa b/Documentation/ABI/testing/sysfs-devices-platform-soc-ipa index c56dcf15bf29..364b1ba41242 100644 --- a/Documentation/ABI/testing/sysfs-devices-platform-soc-ipa +++ b/Documentation/ABI/testing/sysfs-devices-platform-soc-ipa @@ -46,33 +46,69 @@ Description: that is supported by the hardware. The possible values are "MAPv4" or "MAPv5". +What: .../XXXXXXX.ipa/endpoint_id/ +Date: July 2022 +KernelVersion: v5.19 +Contact: Alex Elder <elder@kernel.org> +Description: + The .../XXXXXXX.ipa/endpoint_id/ directory contains + attributes that define IDs associated with IPA + endpoints. The "rx" or "tx" in an endpoint name is + from the perspective of the AP. An endpoint ID is a + small unsigned integer. + +What: .../XXXXXXX.ipa/endpoint_id/modem_rx +Date: July 2022 +KernelVersion: v5.19 +Contact: Alex Elder <elder@kernel.org> +Description: + The .../XXXXXXX.ipa/endpoint_id/modem_rx file contains + the ID of the AP endpoint on which packets originating + from the embedded modem are received. + +What: .../XXXXXXX.ipa/endpoint_id/modem_tx +Date: July 2022 +KernelVersion: v5.19 +Contact: Alex Elder <elder@kernel.org> +Description: + The .../XXXXXXX.ipa/endpoint_id/modem_tx file contains + the ID of the AP endpoint on which packets destined + for the embedded modem are sent. + +What: .../XXXXXXX.ipa/endpoint_id/monitor_rx +Date: July 2022 +KernelVersion: v5.19 +Contact: Alex Elder <elder@kernel.org> +Description: + The .../XXXXXXX.ipa/endpoint_id/monitor_rx file contains + the ID of the AP endpoint on which IPA "monitor" data is + received. The monitor endpoint supplies replicas of + packets that enter the IPA hardware for processing. + Each replicated packet is preceded by a fixed-size "ODL" + header (see .../XXXXXXX.ipa/feature/monitor, above). + Large packets are truncated, to reduce the bandwidth + required to provide the monitor function. + What: .../XXXXXXX.ipa/modem/ Date: June 2021 KernelVersion: v5.14 Contact: Alex Elder <elder@kernel.org> Description: - The .../XXXXXXX.ipa/modem/ directory contains a set of - attributes describing properties of the modem execution - environment reachable by the IPA hardware. + The .../XXXXXXX.ipa/modem/ directory contains attributes + describing properties of the modem embedded in the SoC. What: .../XXXXXXX.ipa/modem/rx_endpoint_id Date: June 2021 KernelVersion: v5.14 Contact: Alex Elder <elder@kernel.org> Description: - The .../XXXXXXX.ipa/feature/rx_endpoint_id file contains - the AP endpoint ID that receives packets originating from - the modem execution environment. The "rx" is from the - perspective of the AP; this endpoint is considered an "IPA - producer". An endpoint ID is a small unsigned integer. + The .../XXXXXXX.ipa/modem/rx_endpoint_id file duplicates + the value found in .../XXXXXXX.ipa/endpoint_id/modem_rx. What: .../XXXXXXX.ipa/modem/tx_endpoint_id Date: June 2021 KernelVersion: v5.14 Contact: Alex Elder <elder@kernel.org> Description: - The .../XXXXXXX.ipa/feature/tx_endpoint_id file contains - the AP endpoint ID used to transmit packets destined for - the modem execution environment. The "tx" is from the - perspective of the AP; this endpoint is considered an "IPA - consumer". An endpoint ID is a small unsigned integer. + The .../XXXXXXX.ipa/modem/tx_endpoint_id file duplicates + the value found in .../XXXXXXX.ipa/endpoint_id/modem_tx. diff --git a/Documentation/bpf/btf.rst b/Documentation/bpf/btf.rst index f49aeef62d0c..cf8722f96090 100644 --- a/Documentation/bpf/btf.rst +++ b/Documentation/bpf/btf.rst @@ -369,7 +369,8 @@ No additional type data follow ``btf_type``. * ``name_off``: offset to a valid C identifier * ``info.kind_flag``: 0 * ``info.kind``: BTF_KIND_FUNC - * ``info.vlen``: 0 + * ``info.vlen``: linkage information (BTF_FUNC_STATIC, BTF_FUNC_GLOBAL + or BTF_FUNC_EXTERN) * ``type``: a BTF_KIND_FUNC_PROTO type No additional type data follow ``btf_type``. @@ -380,6 +381,9 @@ type. The BTF_KIND_FUNC may in turn be referenced by a func_info in the :ref:`BTF_Ext_Section` (ELF) or in the arguments to :ref:`BPF_Prog_Load` (ABI). +Currently, only linkage values of BTF_FUNC_STATIC and BTF_FUNC_GLOBAL are +supported in the kernel. + 2.2.13 BTF_KIND_FUNC_PROTO ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Documentation/bpf/index.rst b/Documentation/bpf/index.rst index 96056a7447c7..1bc2c5c58bdb 100644 --- a/Documentation/bpf/index.rst +++ b/Documentation/bpf/index.rst @@ -19,6 +19,7 @@ that goes into great technical depth about the BPF Architecture. faq syscall_api helpers + kfuncs programs maps bpf_prog_run diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst new file mode 100644 index 000000000000..c0b7dae6dbf5 --- /dev/null +++ b/Documentation/bpf/kfuncs.rst @@ -0,0 +1,170 @@ +============================= +BPF Kernel Functions (kfuncs) +============================= + +1. Introduction +=============== + +BPF Kernel Functions or more commonly known as kfuncs are functions in the Linux +kernel which are exposed for use by BPF programs. Unlike normal BPF helpers, +kfuncs do not have a stable interface and can change from one kernel release to +another. Hence, BPF programs need to be updated in response to changes in the +kernel. + +2. Defining a kfunc +=================== + +There are two ways to expose a kernel function to BPF programs, either make an +existing function in the kernel visible, or add a new wrapper for BPF. In both +cases, care must be taken that BPF program can only call such function in a +valid context. To enforce this, visibility of a kfunc can be per program type. + +If you are not creating a BPF wrapper for existing kernel function, skip ahead +to :ref:`BPF_kfunc_nodef`. + +2.1 Creating a wrapper kfunc +---------------------------- + +When defining a wrapper kfunc, the wrapper function should have extern linkage. +This prevents the compiler from optimizing away dead code, as this wrapper kfunc +is not invoked anywhere in the kernel itself. It is not necessary to provide a +prototype in a header for the wrapper kfunc. + +An example is given below:: + + /* Disables missing prototype warnings */ + __diag_push(); + __diag_ignore_all("-Wmissing-prototypes", + "Global kfuncs as their definitions will be in BTF"); + + struct task_struct *bpf_find_get_task_by_vpid(pid_t nr) + { + return find_get_task_by_vpid(nr); + } + + __diag_pop(); + +A wrapper kfunc is often needed when we need to annotate parameters of the +kfunc. Otherwise one may directly make the kfunc visible to the BPF program by +registering it with the BPF subsystem. See :ref:`BPF_kfunc_nodef`. + +2.2 Annotating kfunc parameters +------------------------------- + +Similar to BPF helpers, there is sometime need for additional context required +by the verifier to make the usage of kernel functions safer and more useful. +Hence, we can annotate a parameter by suffixing the name of the argument of the +kfunc with a __tag, where tag may be one of the supported annotations. + +2.2.1 __sz Annotation +--------------------- + +This annotation is used to indicate a memory and size pair in the argument list. +An example is given below:: + + void bpf_memzero(void *mem, int mem__sz) + { + ... + } + +Here, the verifier will treat first argument as a PTR_TO_MEM, and second +argument as its size. By default, without __sz annotation, the size of the type +of the pointer is used. Without __sz annotation, a kfunc cannot accept a void +pointer. + +.. _BPF_kfunc_nodef: + +2.3 Using an existing kernel function +------------------------------------- + +When an existing function in the kernel is fit for consumption by BPF programs, +it can be directly registered with the BPF subsystem. However, care must still +be taken to review the context in which it will be invoked by the BPF program +and whether it is safe to do so. + +2.4 Annotating kfuncs +--------------------- + +In addition to kfuncs' arguments, verifier may need more information about the +type of kfunc(s) being registered with the BPF subsystem. To do so, we define +flags on a set of kfuncs as follows:: + + BTF_SET8_START(bpf_task_set) + BTF_ID_FLAGS(func, bpf_get_task_pid, KF_ACQUIRE | KF_RET_NULL) + BTF_ID_FLAGS(func, bpf_put_pid, KF_RELEASE) + BTF_SET8_END(bpf_task_set) + +This set encodes the BTF ID of each kfunc listed above, and encodes the flags +along with it. Ofcourse, it is also allowed to specify no flags. + +2.4.1 KF_ACQUIRE flag +--------------------- + +The KF_ACQUIRE flag is used to indicate that the kfunc returns a pointer to a +refcounted object. The verifier will then ensure that the pointer to the object +is eventually released using a release kfunc, or transferred to a map using a +referenced kptr (by invoking bpf_kptr_xchg). If not, the verifier fails the +loading of the BPF program until no lingering references remain in all possible +explored states of the program. + +2.4.2 KF_RET_NULL flag +---------------------- + +The KF_RET_NULL flag is used to indicate that the pointer returned by the kfunc +may be NULL. Hence, it forces the user to do a NULL check on the pointer +returned from the kfunc before making use of it (dereferencing or passing to +another helper). This flag is often used in pairing with KF_ACQUIRE flag, but +both are orthogonal to each other. + +2.4.3 KF_RELEASE flag +--------------------- + +The KF_RELEASE flag is used to indicate that the kfunc releases the pointer +passed in to it. There can be only one referenced pointer that can be passed in. +All copies of the pointer being released are invalidated as a result of invoking +kfunc with this flag. + +2.4.4 KF_KPTR_GET flag +---------------------- + +The KF_KPTR_GET flag is used to indicate that the kfunc takes the first argument +as a pointer to kptr, safely increments the refcount of the object it points to, +and returns a reference to the user. The rest of the arguments may be normal +arguments of a kfunc. The KF_KPTR_GET flag should be used in conjunction with +KF_ACQUIRE and KF_RET_NULL flags. + +2.4.5 KF_TRUSTED_ARGS flag +-------------------------- + +The KF_TRUSTED_ARGS flag is used for kfuncs taking pointer arguments. It +indicates that the all pointer arguments will always be refcounted, and have +their offset set to 0. It can be used to enforce that a pointer to a refcounted +object acquired from a kfunc or BPF helper is passed as an argument to this +kfunc without any modifications (e.g. pointer arithmetic) such that it is +trusted and points to the original object. This flag is often used for kfuncs +that operate (change some property, perform some operation) on an object that +was obtained using an acquire kfunc. Such kfuncs need an unchanged pointer to +ensure the integrity of the operation being performed on the expected object. + +2.5 Registering the kfuncs +-------------------------- + +Once the kfunc is prepared for use, the final step to making it visible is +registering it with the BPF subsystem. Registration is done per BPF program +type. An example is shown below:: + + BTF_SET8_START(bpf_task_set) + BTF_ID_FLAGS(func, bpf_get_task_pid, KF_ACQUIRE | KF_RET_NULL) + BTF_ID_FLAGS(func, bpf_put_pid, KF_RELEASE) + BTF_SET8_END(bpf_task_set) + + static const struct btf_kfunc_id_set bpf_task_kfunc_set = { + .owner = THIS_MODULE, + .set = &bpf_task_set, + }; + + static int init_subsystem(void) + { + return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_task_kfunc_set); + } + late_initcall(init_subsystem); diff --git a/Documentation/bpf/map_hash.rst b/Documentation/bpf/map_hash.rst new file mode 100644 index 000000000000..e85120878b27 --- /dev/null +++ b/Documentation/bpf/map_hash.rst @@ -0,0 +1,185 @@ +.. SPDX-License-Identifier: GPL-2.0-only +.. Copyright (C) 2022 Red Hat, Inc. + +=============================================== +BPF_MAP_TYPE_HASH, with PERCPU and LRU Variants +=============================================== + +.. note:: + - ``BPF_MAP_TYPE_HASH`` was introduced in kernel version 3.19 + - ``BPF_MAP_TYPE_PERCPU_HASH`` was introduced in version 4.6 + - Both ``BPF_MAP_TYPE_LRU_HASH`` and ``BPF_MAP_TYPE_LRU_PERCPU_HASH`` + were introduced in version 4.10 + +``BPF_MAP_TYPE_HASH`` and ``BPF_MAP_TYPE_PERCPU_HASH`` provide general +purpose hash map storage. Both the key and the value can be structs, +allowing for composite keys and values. + +The kernel is responsible for allocating and freeing key/value pairs, up +to the max_entries limit that you specify. Hash maps use pre-allocation +of hash table elements by default. The ``BPF_F_NO_PREALLOC`` flag can be +used to disable pre-allocation when it is too memory expensive. + +``BPF_MAP_TYPE_PERCPU_HASH`` provides a separate value slot per +CPU. The per-cpu values are stored internally in an array. + +The ``BPF_MAP_TYPE_LRU_HASH`` and ``BPF_MAP_TYPE_LRU_PERCPU_HASH`` +variants add LRU semantics to their respective hash tables. An LRU hash +will automatically evict the least recently used entries when the hash +table reaches capacity. An LRU hash maintains an internal LRU list that +is used to select elements for eviction. This internal LRU list is +shared across CPUs but it is possible to request a per CPU LRU list with +the ``BPF_F_NO_COMMON_LRU`` flag when calling ``bpf_map_create``. + +Usage +===== + +.. c:function:: + long bpf_map_update_elem(struct bpf_map *map, const void *key, const void *value, u64 flags) + +Hash entries can be added or updated using the ``bpf_map_update_elem()`` +helper. This helper replaces existing elements atomically. The ``flags`` +parameter can be used to control the update behaviour: + +- ``BPF_ANY`` will create a new element or update an existing element +- ``BPF_NOEXIST`` will create a new element only if one did not already + exist +- ``BPF_EXIST`` will update an existing element + +``bpf_map_update_elem()`` returns 0 on success, or negative error in +case of failure. + +.. c:function:: + void *bpf_map_lookup_elem(struct bpf_map *map, const void *key) + +Hash entries can be retrieved using the ``bpf_map_lookup_elem()`` +helper. This helper returns a pointer to the value associated with +``key``, or ``NULL`` if no entry was found. + +.. c:function:: + long bpf_map_delete_elem(struct bpf_map *map, const void *key) + +Hash entries can be deleted using the ``bpf_map_delete_elem()`` +helper. This helper will return 0 on success, or negative error in case +of failure. + +Per CPU Hashes +-------------- + +For ``BPF_MAP_TYPE_PERCPU_HASH`` and ``BPF_MAP_TYPE_LRU_PERCPU_HASH`` +the ``bpf_map_update_elem()`` and ``bpf_map_lookup_elem()`` helpers +automatically access the hash slot for the current CPU. + +.. c:function:: + void *bpf_map_lookup_percpu_elem(struct bpf_map *map, const void *key, u32 cpu) + +The ``bpf_map_lookup_percpu_elem()`` helper can be used to lookup the +value in the hash slot for a specific CPU. Returns value associated with +``key`` on ``cpu`` , or ``NULL`` if no entry was found or ``cpu`` is +invalid. + +Concurrency +----------- + +Values stored in ``BPF_MAP_TYPE_HASH`` can be accessed concurrently by +programs running on different CPUs. Since Kernel version 5.1, the BPF +infrastructure provides ``struct bpf_spin_lock`` to synchronise access. +See ``tools/testing/selftests/bpf/progs/test_spin_lock.c``. + +Userspace +--------- + +.. c:function:: + int bpf_map_get_next_key(int fd, const void *cur_key, void *next_key) + +In userspace, it is possible to iterate through the keys of a hash using +libbpf's ``bpf_map_get_next_key()`` function. The first key can be fetched by +calling ``bpf_map_get_next_key()`` with ``cur_key`` set to +``NULL``. Subsequent calls will fetch the next key that follows the +current key. ``bpf_map_get_next_key()`` returns 0 on success, -ENOENT if +cur_key is the last key in the hash, or negative error in case of +failure. + +Note that if ``cur_key`` gets deleted then ``bpf_map_get_next_key()`` +will instead return the *first* key in the hash table which is +undesirable. It is recommended to use batched lookup if there is going +to be key deletion intermixed with ``bpf_map_get_next_key()``. + +Examples +======== + +Please see the ``tools/testing/selftests/bpf`` directory for functional +examples. The code snippets below demonstrates API usage. + +This example shows how to declare an LRU Hash with a struct key and a +struct value. + +.. code-block:: c + + #include <linux/bpf.h> + #include <bpf/bpf_helpers.h> + + struct key { + __u32 srcip; + }; + + struct value { + __u64 packets; + __u64 bytes; + }; + + struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 32); + __type(key, struct key); + __type(value, struct value); + } packet_stats SEC(".maps"); + +This example shows how to create or update hash values using atomic +instructions: + +.. code-block:: c + + static void update_stats(__u32 srcip, int bytes) + { + struct key key = { + .srcip = srcip, + }; + struct value *value = bpf_map_lookup_elem(&packet_stats, &key); + + if (value) { + __sync_fetch_and_add(&value->packets, 1); + __sync_fetch_and_add(&value->bytes, bytes); + } else { + struct value newval = { 1, bytes }; + + bpf_map_update_elem(&packet_stats, &key, &newval, BPF_NOEXIST); + } + } + +Userspace walking the map elements from the map declared above: + +.. code-block:: c + + #include <bpf/libbpf.h> + #include <bpf/bpf.h> + + static void walk_hash_elements(int map_fd) + { + struct key *cur_key = NULL; + struct key next_key; + struct value value; + int err; + + for (;;) { + err = bpf_map_get_next_key(map_fd, cur_key, &next_key); + if (err) + break; + + bpf_map_lookup_elem(map_fd, &next_key, &value); + + // Use key and value here + + cur_key = &next_key; + } + } diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-engine.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-engine.yaml index c388ae5da1e4..c9c346e6228e 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-engine.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-engine.yaml @@ -94,6 +94,7 @@ if: - allwinner,sun8i-a83t-display-engine - allwinner,sun8i-r40-display-engine - allwinner,sun9i-a80-display-engine + - allwinner,sun20i-d1-display-engine - allwinner,sun50i-a64-display-engine then: diff --git a/Documentation/devicetree/bindings/net/broadcom-bluetooth.yaml b/Documentation/devicetree/bindings/net/broadcom-bluetooth.yaml index 5aac094fd217..445b2a553625 100644 --- a/Documentation/devicetree/bindings/net/broadcom-bluetooth.yaml +++ b/Documentation/devicetree/bindings/net/broadcom-bluetooth.yaml @@ -23,6 +23,8 @@ properties: - brcm,bcm4345c5 - brcm,bcm43540-bt - brcm,bcm4335a0 + - brcm,bcm4349-bt + - infineon,cyw55572-bt shutdown-gpios: maxItems: 1 @@ -92,6 +94,13 @@ properties: pcm-sync-mode: slave, master pcm-clock-mode: slave, master + brcm,requires-autobaud-mode: + type: boolean + description: + Set this property if autobaud mode is required. Autobaud mode is required + if the device's initial baud rate in normal mode is not supported by the + host or if the device requires autobaud mode startup before loading FW. + interrupts: items: - description: Handle to the line HOST_WAKE used to wake @@ -108,6 +117,22 @@ properties: required: - compatible +dependencies: + brcm,requires-autobaud-mode: [ 'shutdown-gpios' ] + +if: + not: + properties: + compatible: + contains: + enum: + - brcm,bcm20702a1 + - brcm,bcm4329-bt + - brcm,bcm4330-bt +then: + properties: + reset-gpios: false + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml b/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml new file mode 100644 index 000000000000..b1327c5b86cf --- /dev/null +++ b/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/can/nxp,sja1000.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Memory mapped SJA1000 CAN controller from NXP (formerly Philips) + +maintainers: + - Wolfgang Grandegger <wg@grandegger.com> + +properties: + compatible: + oneOf: + - enum: + - nxp,sja1000 + - technologic,sja1000 + - items: + - enum: + - renesas,r9a06g032-sja1000 # RZ/N1D + - renesas,r9a06g033-sja1000 # RZ/N1S + - const: renesas,rzn1-sja1000 # RZ/N1 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + reg-io-width: + $ref: /schemas/types.yaml#/definitions/uint32 + description: I/O register width (in bytes) implemented by this device + default: 1 + enum: [ 1, 2, 4 ] + + nxp,external-clock-frequency: + $ref: /schemas/types.yaml#/definitions/uint32 + default: 16000000 + description: | + Frequency of the external oscillator clock in Hz. + The internal clock frequency used by the SJA1000 is half of that value. + + nxp,tx-output-mode: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [ 0, 1, 2, 3 ] + default: 1 + description: | + operation mode of the TX output control logic. Valid values are: + <0> : bi-phase output mode + <1> : normal output mode (default) + <2> : test output mode + <3> : clock output mode + + nxp,tx-output-config: + $ref: /schemas/types.yaml#/definitions/uint32 + default: 0x02 + description: | + TX output pin configuration. Valid values are any one of the below + or combination of TX0 and TX1: + <0x01> : TX0 invert + <0x02> : TX0 pull-down (default) + <0x04> : TX0 pull-up + <0x06> : TX0 push-pull + <0x08> : TX1 invert + <0x10> : TX1 pull-down + <0x20> : TX1 pull-up + <0x30> : TX1 push-pull + + nxp,clock-out-frequency: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + clock frequency in Hz on the CLKOUT pin. + If not specified or if the specified value is 0, the CLKOUT pin + will be disabled. + + nxp,no-comparator-bypass: + type: boolean + description: Allows to disable the CAN input comparator. + +required: + - compatible + - reg + - interrupts + +allOf: + - $ref: can-controller.yaml# + - if: + properties: + compatible: + contains: + enum: + - technologic,sja1000 + - renesas,rzn1-sja1000 + then: + required: + - reg-io-width + - if: + properties: + compatible: + contains: + const: renesas,rzn1-sja1000 + then: + required: + - clocks + +unevaluatedProperties: false + +examples: + - | + can@1a000 { + compatible = "technologic,sja1000"; + reg = <0x1a000 0x100>; + interrupts = <1>; + reg-io-width = <2>; + nxp,tx-output-config = <0x06>; + nxp,external-clock-frequency = <24000000>; + }; + + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + #include <dt-bindings/clock/r9a06g032-sysctrl.h> + + can@52104000 { + compatible = "renesas,r9a06g032-sja1000", "renesas,rzn1-sja1000"; + reg = <0x52104000 0x800>; + reg-io-width = <4>; + interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&sysctrl R9A06G032_HCLK_CAN0>; + }; diff --git a/Documentation/devicetree/bindings/net/can/sja1000.txt b/Documentation/devicetree/bindings/net/can/sja1000.txt deleted file mode 100644 index ac3160eca96a..000000000000 --- a/Documentation/devicetree/bindings/net/can/sja1000.txt +++ /dev/null @@ -1,58 +0,0 @@ -Memory mapped SJA1000 CAN controller from NXP (formerly Philips) - -Required properties: - -- compatible : should be one of "nxp,sja1000", "technologic,sja1000". - -- reg : should specify the chip select, address offset and size required - to map the registers of the SJA1000. The size is usually 0x80. - -- interrupts: property with a value describing the interrupt source - (number and sensitivity) required for the SJA1000. - -Optional properties: - -- reg-io-width : Specify the size (in bytes) of the IO accesses that - should be performed on the device. Valid value is 1, 2 or 4. - This property is ignored for technologic version. - Default to 1 (8 bits). - -- nxp,external-clock-frequency : Frequency of the external oscillator - clock in Hz. Note that the internal clock frequency used by the - SJA1000 is half of that value. If not specified, a default value - of 16000000 (16 MHz) is used. - -- nxp,tx-output-mode : operation mode of the TX output control logic: - <0x0> : bi-phase output mode - <0x1> : normal output mode (default) - <0x2> : test output mode - <0x3> : clock output mode - -- nxp,tx-output-config : TX output pin configuration: - <0x01> : TX0 invert - <0x02> : TX0 pull-down (default) - <0x04> : TX0 pull-up - <0x06> : TX0 push-pull - <0x08> : TX1 invert - <0x10> : TX1 pull-down - <0x20> : TX1 pull-up - <0x30> : TX1 push-pull - -- nxp,clock-out-frequency : clock frequency in Hz on the CLKOUT pin. - If not specified or if the specified value is 0, the CLKOUT pin - will be disabled. - -- nxp,no-comparator-bypass : Allows to disable the CAN input comparator. - -For further information, please have a look to the SJA1000 data sheet. - -Examples: - -can@3,100 { - compatible = "nxp,sja1000"; - reg = <3 0x100 0x80>; - interrupts = <2 0>; - interrupt-parent = <&mpic>; - nxp,external-clock-frequency = <16000000>; -}; - diff --git a/Documentation/devicetree/bindings/net/cdns,macb.yaml b/Documentation/devicetree/bindings/net/cdns,macb.yaml index 9c92156869b2..762deccd3640 100644 --- a/Documentation/devicetree/bindings/net/cdns,macb.yaml +++ b/Documentation/devicetree/bindings/net/cdns,macb.yaml @@ -20,6 +20,7 @@ properties: - items: - enum: + - cdns,versal-gem # Xilinx Versal - cdns,zynq-gem # Xilinx Zynq-7xxx SoC - cdns,zynqmp-gem # Xilinx Zynq Ultrascale+ MPSoC - const: cdns,gem # Generic diff --git a/Documentation/devicetree/bindings/net/fsl,fec.yaml b/Documentation/devicetree/bindings/net/fsl,fec.yaml index daa2f79a294f..85a8d8fb6b8f 100644 --- a/Documentation/devicetree/bindings/net/fsl,fec.yaml +++ b/Documentation/devicetree/bindings/net/fsl,fec.yaml @@ -121,6 +121,10 @@ properties: mac-address: true + nvmem-cells: true + + nvmem-cell-names: true + tx-internal-delay-ps: enum: [0, 2000] @@ -213,7 +217,7 @@ required: # least undocumented properties. However, PHY may have a deprecated option to # place PHY OF properties in the MAC node, such as Micrel PHY, and we can find # these boards which is based on i.MX6QDL. -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/filesystems/netfs_library.rst b/Documentation/filesystems/netfs_library.rst index 4d19b19bcc08..73a4176144b3 100644 --- a/Documentation/filesystems/netfs_library.rst +++ b/Documentation/filesystems/netfs_library.rst @@ -301,7 +301,7 @@ through which it can issue requests and negotiate:: void (*issue_read)(struct netfs_io_subrequest *subreq); bool (*is_still_valid)(struct netfs_io_request *rreq); int (*check_write_begin)(struct file *file, loff_t pos, unsigned len, - struct folio *folio, void **_fsdata); + struct folio **foliop, void **_fsdata); void (*done)(struct netfs_io_request *rreq); }; @@ -381,8 +381,10 @@ The operations are as follows: allocated/grabbed the folio to be modified to allow the filesystem to flush conflicting state before allowing it to be modified. - It should return 0 if everything is now fine, -EAGAIN if the folio should be - regrabbed and any other error code to abort the operation. + It may unlock and discard the folio it was given and set the caller's folio + pointer to NULL. It should return 0 if everything is now fine (``*foliop`` + left set) or the op should be retried (``*foliop`` cleared) and any other + error code to abort the operation. * ``done`` diff --git a/Documentation/networking/device_drivers/ethernet/intel/ice.rst b/Documentation/networking/device_drivers/ethernet/intel/ice.rst index 67b7a701ce9e..dc2e60ced927 100644 --- a/Documentation/networking/device_drivers/ethernet/intel/ice.rst +++ b/Documentation/networking/device_drivers/ethernet/intel/ice.rst @@ -901,6 +901,15 @@ To enable/disable UDP Segmentation Offload, issue the following command:: # ethtool -K <ethX> tx-udp-segmentation [off|on] +GNSS module +----------- +Allows user to read messages from the GNSS module and write supported commands. +If the module is physically present, driver creates 2 TTYs for each supported +device in /dev, ttyGNSS_<device>:<function>_0 and _1. First one (_0) is RW and +the second one is RO. +The protocol of write commands is dependent on the GNSS module as the driver +writes raw bytes from the TTY to the GNSS i2c. Please refer to the module +documentation for details. Performance Optimization ======================== diff --git a/Documentation/networking/dsa/dsa.rst b/Documentation/networking/dsa/dsa.rst index ed7fa76e7a40..d742ba6bd211 100644 --- a/Documentation/networking/dsa/dsa.rst +++ b/Documentation/networking/dsa/dsa.rst @@ -503,26 +503,108 @@ per-port PHY specific details: interface connection, MDIO bus location, etc. Driver development ================== -DSA switch drivers need to implement a dsa_switch_ops structure which will +DSA switch drivers need to implement a ``dsa_switch_ops`` structure which will contain the various members described below. -``register_switch_driver()`` registers this dsa_switch_ops in its internal list -of drivers to probe for. ``unregister_switch_driver()`` does the exact opposite. +Probing, registration and device lifetime +----------------------------------------- -Unless requested differently by setting the priv_size member accordingly, DSA -does not allocate any driver private context space. +DSA switches are regular ``device`` structures on buses (be they platform, SPI, +I2C, MDIO or otherwise). The DSA framework is not involved in their probing +with the device core. + +Switch registration from the perspective of a driver means passing a valid +``struct dsa_switch`` pointer to ``dsa_register_switch()``, usually from the +switch driver's probing function. The following members must be valid in the +provided structure: + +- ``ds->dev``: will be used to parse the switch's OF node or platform data. + +- ``ds->num_ports``: will be used to create the port list for this switch, and + to validate the port indices provided in the OF node. + +- ``ds->ops``: a pointer to the ``dsa_switch_ops`` structure holding the DSA + method implementations. + +- ``ds->priv``: backpointer to a driver-private data structure which can be + retrieved in all further DSA method callbacks. + +In addition, the following flags in the ``dsa_switch`` structure may optionally +be configured to obtain driver-specific behavior from the DSA core. Their +behavior when set is documented through comments in ``include/net/dsa.h``. + +- ``ds->vlan_filtering_is_global`` + +- ``ds->needs_standalone_vlan_filtering`` + +- ``ds->configure_vlan_while_not_filtering`` + +- ``ds->untag_bridge_pvid`` + +- ``ds->assisted_learning_on_cpu_port`` + +- ``ds->mtu_enforcement_ingress`` + +- ``ds->fdb_isolation`` + +Internally, DSA keeps an array of switch trees (group of switches) global to +the kernel, and attaches a ``dsa_switch`` structure to a tree on registration. +The tree ID to which the switch is attached is determined by the first u32 +number of the ``dsa,member`` property of the switch's OF node (0 if missing). +The switch ID within the tree is determined by the second u32 number of the +same OF property (0 if missing). Registering multiple switches with the same +switch ID and tree ID is illegal and will cause an error. Using platform data, +a single switch and a single switch tree is permitted. + +In case of a tree with multiple switches, probing takes place asymmetrically. +The first N-1 callers of ``dsa_register_switch()`` only add their ports to the +port list of the tree (``dst->ports``), each port having a backpointer to its +associated switch (``dp->ds``). Then, these switches exit their +``dsa_register_switch()`` call early, because ``dsa_tree_setup_routing_table()`` +has determined that the tree is not yet complete (not all ports referenced by +DSA links are present in the tree's port list). The tree becomes complete when +the last switch calls ``dsa_register_switch()``, and this triggers the effective +continuation of initialization (including the call to ``ds->ops->setup()``) for +all switches within that tree, all as part of the calling context of the last +switch's probe function. + +The opposite of registration takes place when calling ``dsa_unregister_switch()``, +which removes a switch's ports from the port list of the tree. The entire tree +is torn down when the first switch unregisters. + +It is mandatory for DSA switch drivers to implement the ``shutdown()`` callback +of their respective bus, and call ``dsa_switch_shutdown()`` from it (a minimal +version of the full teardown performed by ``dsa_unregister_switch()``). +The reason is that DSA keeps a reference on the master net device, and if the +driver for the master device decides to unbind on shutdown, DSA's reference +will block that operation from finalizing. + +Either ``dsa_switch_shutdown()`` or ``dsa_unregister_switch()`` must be called, +but not both, and the device driver model permits the bus' ``remove()`` method +to be called even if ``shutdown()`` was already called. Therefore, drivers are +expected to implement a mutual exclusion method between ``remove()`` and +``shutdown()`` by setting their drvdata to NULL after any of these has run, and +checking whether the drvdata is NULL before proceeding to take any action. + +After ``dsa_switch_shutdown()`` or ``dsa_unregister_switch()`` was called, no +further callbacks via the provided ``dsa_switch_ops`` may take place, and the +driver may free the data structures associated with the ``dsa_switch``. Switch configuration -------------------- -- ``tag_protocol``: this is to indicate what kind of tagging protocol is supported, - should be a valid value from the ``dsa_tag_protocol`` enum +- ``get_tag_protocol``: this is to indicate what kind of tagging protocol is + supported, should be a valid value from the ``dsa_tag_protocol`` enum. + The returned information does not have to be static; the driver is passed the + CPU port number, as well as the tagging protocol of a possibly stacked + upstream switch, in case there are hardware limitations in terms of supported + tag formats. -- ``probe``: probe routine which will be invoked by the DSA platform device upon - registration to test for the presence/absence of a switch device. For MDIO - devices, it is recommended to issue a read towards internal registers using - the switch pseudo-PHY and return whether this is a supported device. For other - buses, return a non-NULL string +- ``change_tag_protocol``: when the default tagging protocol has compatibility + problems with the master or other issues, the driver may support changing it + at runtime, either through a device tree property or through sysfs. In that + case, further calls to ``get_tag_protocol`` should report the protocol in + current use. - ``setup``: setup function for the switch, this function is responsible for setting up the ``dsa_switch_ops`` private structure with all it needs: register maps, @@ -535,7 +617,17 @@ Switch configuration fully configured and ready to serve any kind of request. It is recommended to issue a software reset of the switch during this setup function in order to avoid relying on what a previous software agent such as a bootloader/firmware - may have previously configured. + may have previously configured. The method responsible for undoing any + applicable allocations or operations done here is ``teardown``. + +- ``port_setup`` and ``port_teardown``: methods for initialization and + destruction of per-port data structures. It is mandatory for some operations + such as registering and unregistering devlink port regions to be done from + these methods, otherwise they are optional. A port will be torn down only if + it has been previously set up. It is possible for a port to be set up during + probing only to be torn down immediately afterwards, for example in case its + PHY cannot be found. In this case, probing of the DSA switch continues + without that particular port. PHY devices and link management ------------------------------- @@ -635,26 +727,198 @@ Power management ``BR_STATE_DISABLED`` and propagating changes to the hardware if this port is disabled while being a bridge member +Address databases +----------------- + +Switching hardware is expected to have a table for FDB entries, however not all +of them are active at the same time. An address database is the subset (partition) +of FDB entries that is active (can be matched by address learning on RX, or FDB +lookup on TX) depending on the state of the port. An address database may +occasionally be called "FID" (Filtering ID) in this document, although the +underlying implementation may choose whatever is available to the hardware. + +For example, all ports that belong to a VLAN-unaware bridge (which is +*currently* VLAN-unaware) are expected to learn source addresses in the +database associated by the driver with that bridge (and not with other +VLAN-unaware bridges). During forwarding and FDB lookup, a packet received on a +VLAN-unaware bridge port should be able to find a VLAN-unaware FDB entry having +the same MAC DA as the packet, which is present on another port member of the +same bridge. At the same time, the FDB lookup process must be able to not find +an FDB entry having the same MAC DA as the packet, if that entry points towards +a port which is a member of a different VLAN-unaware bridge (and is therefore +associated with a different address database). + +Similarly, each VLAN of each offloaded VLAN-aware bridge should have an +associated address database, which is shared by all ports which are members of +that VLAN, but not shared by ports belonging to different bridges that are +members of the same VID. + +In this context, a VLAN-unaware database means that all packets are expected to +match on it irrespective of VLAN ID (only MAC address lookup), whereas a +VLAN-aware database means that packets are supposed to match based on the VLAN +ID from the classified 802.1Q header (or the pvid if untagged). + +At the bridge layer, VLAN-unaware FDB entries have the special VID value of 0, +whereas VLAN-aware FDB entries have non-zero VID values. Note that a +VLAN-unaware bridge may have VLAN-aware (non-zero VID) FDB entries, and a +VLAN-aware bridge may have VLAN-unaware FDB entries. As in hardware, the +software bridge keeps separate address databases, and offloads to hardware the +FDB entries belonging to these databases, through switchdev, asynchronously +relative to the moment when the databases become active or inactive. + +When a user port operates in standalone mode, its driver should configure it to +use a separate database called a port private database. This is different from +the databases described above, and should impede operation as standalone port +(packet in, packet out to the CPU port) as little as possible. For example, +on ingress, it should not attempt to learn the MAC SA of ingress traffic, since +learning is a bridging layer service and this is a standalone port, therefore +it would consume useless space. With no address learning, the port private +database should be empty in a naive implementation, and in this case, all +received packets should be trivially flooded to the CPU port. + +DSA (cascade) and CPU ports are also called "shared" ports because they service +multiple address databases, and the database that a packet should be associated +to is usually embedded in the DSA tag. This means that the CPU port may +simultaneously transport packets coming from a standalone port (which were +classified by hardware in one address database), and from a bridge port (which +were classified to a different address database). + +Switch drivers which satisfy certain criteria are able to optimize the naive +configuration by removing the CPU port from the flooding domain of the switch, +and just program the hardware with FDB entries pointing towards the CPU port +for which it is known that software is interested in those MAC addresses. +Packets which do not match a known FDB entry will not be delivered to the CPU, +which will save CPU cycles required for creating an skb just to drop it. + +DSA is able to perform host address filtering for the following kinds of +addresses: + +- Primary unicast MAC addresses of ports (``dev->dev_addr``). These are + associated with the port private database of the respective user port, + and the driver is notified to install them through ``port_fdb_add`` towards + the CPU port. + +- Secondary unicast and multicast MAC addresses of ports (addresses added + through ``dev_uc_add()`` and ``dev_mc_add()``). These are also associated + with the port private database of the respective user port. + +- Local/permanent bridge FDB entries (``BR_FDB_LOCAL``). These are the MAC + addresses of the bridge ports, for which packets must be terminated locally + and not forwarded. They are associated with the address database for that + bridge. + +- Static bridge FDB entries installed towards foreign (non-DSA) interfaces + present in the same bridge as some DSA switch ports. These are also + associated with the address database for that bridge. + +- Dynamically learned FDB entries on foreign interfaces present in the same + bridge as some DSA switch ports, only if ``ds->assisted_learning_on_cpu_port`` + is set to true by the driver. These are associated with the address database + for that bridge. + +For various operations detailed below, DSA provides a ``dsa_db`` structure +which can be of the following types: + +- ``DSA_DB_PORT``: the FDB (or MDB) entry to be installed or deleted belongs to + the port private database of user port ``db->dp``. +- ``DSA_DB_BRIDGE``: the entry belongs to one of the address databases of bridge + ``db->bridge``. Separation between the VLAN-unaware database and the per-VID + databases of this bridge is expected to be done by the driver. +- ``DSA_DB_LAG``: the entry belongs to the address database of LAG ``db->lag``. + Note: ``DSA_DB_LAG`` is currently unused and may be removed in the future. + +The drivers which act upon the ``dsa_db`` argument in ``port_fdb_add``, +``port_mdb_add`` etc should declare ``ds->fdb_isolation`` as true. + +DSA associates each offloaded bridge and each offloaded LAG with a one-based ID +(``struct dsa_bridge :: num``, ``struct dsa_lag :: id``) for the purposes of +refcounting addresses on shared ports. Drivers may piggyback on DSA's numbering +scheme (the ID is readable through ``db->bridge.num`` and ``db->lag.id`` or may +implement their own. + +Only the drivers which declare support for FDB isolation are notified of FDB +entries on the CPU port belonging to ``DSA_DB_PORT`` databases. +For compatibility/legacy reasons, ``DSA_DB_BRIDGE`` addresses are notified to +drivers even if they do not support FDB isolation. However, ``db->bridge.num`` +and ``db->lag.id`` are always set to 0 in that case (to denote the lack of +isolation, for refcounting purposes). + +Note that it is not mandatory for a switch driver to implement physically +separate address databases for each standalone user port. Since FDB entries in +the port private databases will always point to the CPU port, there is no risk +for incorrect forwarding decisions. In this case, all standalone ports may +share the same database, but the reference counting of host-filtered addresses +(not deleting the FDB entry for a port's MAC address if it's still in use by +another port) becomes the responsibility of the driver, because DSA is unaware +that the port databases are in fact shared. This can be achieved by calling +``dsa_fdb_present_in_other_db()`` and ``dsa_mdb_present_in_other_db()``. +The down side is that the RX filtering lists of each user port are in fact +shared, which means that user port A may accept a packet with a MAC DA it +shouldn't have, only because that MAC address was in the RX filtering list of +user port B. These packets will still be dropped in software, however. + Bridge layer ------------ +Offloading the bridge forwarding plane is optional and handled by the methods +below. They may be absent, return -EOPNOTSUPP, or ``ds->max_num_bridges`` may +be non-zero and exceeded, and in this case, joining a bridge port is still +possible, but the packet forwarding will take place in software, and the ports +under a software bridge must remain configured in the same way as for +standalone operation, i.e. have all bridging service functions (address +learning etc) disabled, and send all received packets to the CPU port only. + +Concretely, a port starts offloading the forwarding plane of a bridge once it +returns success to the ``port_bridge_join`` method, and stops doing so after +``port_bridge_leave`` has been called. Offloading the bridge means autonomously +learning FDB entries in accordance with the software bridge port's state, and +autonomously forwarding (or flooding) received packets without CPU intervention. +This is optional even when offloading a bridge port. Tagging protocol drivers +are expected to call ``dsa_default_offload_fwd_mark(skb)`` for packets which +have already been autonomously forwarded in the forwarding domain of the +ingress switch port. DSA, through ``dsa_port_devlink_setup()``, considers all +switch ports part of the same tree ID to be part of the same bridge forwarding +domain (capable of autonomous forwarding to each other). + +Offloading the TX forwarding process of a bridge is a distinct concept from +simply offloading its forwarding plane, and refers to the ability of certain +driver and tag protocol combinations to transmit a single skb coming from the +bridge device's transmit function to potentially multiple egress ports (and +thereby avoid its cloning in software). + +Packets for which the bridge requests this behavior are called data plane +packets and have ``skb->offload_fwd_mark`` set to true in the tag protocol +driver's ``xmit`` function. Data plane packets are subject to FDB lookup, +hardware learning on the CPU port, and do not override the port STP state. +Additionally, replication of data plane packets (multicast, flooding) is +handled in hardware and the bridge driver will transmit a single skb for each +packet that may or may not need replication. + +When the TX forwarding offload is enabled, the tag protocol driver is +responsible to inject packets into the data plane of the hardware towards the +correct bridging domain (FID) that the port is a part of. The port may be +VLAN-unaware, and in this case the FID must be equal to the FID used by the +driver for its VLAN-unaware address database associated with that bridge. +Alternatively, the bridge may be VLAN-aware, and in that case, it is guaranteed +that the packet is also VLAN-tagged with the VLAN ID that the bridge processed +this packet in. It is the responsibility of the hardware to untag the VID on +the egress-untagged ports, or keep the tag on the egress-tagged ones. + - ``port_bridge_join``: bridge layer function invoked when a given switch port is added to a bridge, this function should do what's necessary at the switch level to permit the joining port to be added to the relevant logical domain for it to ingress/egress traffic with other members of the bridge. + By setting the ``tx_fwd_offload`` argument to true, the TX forwarding process + of this bridge is also offloaded. - ``port_bridge_leave``: bridge layer function invoked when a given switch port is removed from a bridge, this function should do what's necessary at the switch level to deny the leaving port from ingress/egress traffic from the - remaining bridge members. When the port leaves the bridge, it should be aged - out at the switch hardware for the switch to (re) learn MAC addresses behind - this port. + remaining bridge members. - ``port_stp_state_set``: bridge layer function invoked when a given switch port STP state is computed by the bridge layer and should be propagated to switch - hardware to forward/block/learn traffic. The switch driver is responsible for - computing a STP state change based on current and asked parameters and perform - the relevant ageing based on the intersection results + hardware to forward/block/learn traffic. - ``port_bridge_flags``: bridge layer function invoked when a port must configure its settings for e.g. flooding of unknown traffic or source address @@ -667,21 +931,11 @@ Bridge layer CPU port, and flooding towards the CPU port should also be enabled, due to a lack of an explicit address filtering mechanism in the DSA core. -- ``port_bridge_tx_fwd_offload``: bridge layer function invoked after - ``port_bridge_join`` when a driver sets ``ds->num_fwd_offloading_bridges`` to - a non-zero value. Returning success in this function activates the TX - forwarding offload bridge feature for this port, which enables the tagging - protocol driver to inject data plane packets towards the bridging domain that - the port is a part of. Data plane packets are subject to FDB lookup, hardware - learning on the CPU port, and do not override the port STP state. - Additionally, replication of data plane packets (multicast, flooding) is - handled in hardware and the bridge driver will transmit a single skb for each - packet that needs replication. The method is provided as a configuration - point for drivers that need to configure the hardware for enabling this - feature. - -- ``port_bridge_tx_fwd_unoffload``: bridge layer function invoked when a driver - leaves a bridge port which had the TX forwarding offload feature enabled. +- ``port_fast_age``: bridge layer function invoked when flushing the + dynamically learned FDB entries on the port is necessary. This is called when + transitioning from an STP state where learning should take place to an STP + state where it shouldn't, or when leaving a bridge, or when address learning + is turned off via ``port_bridge_flags``. Bridge VLAN filtering --------------------- @@ -697,55 +951,44 @@ Bridge VLAN filtering allowed. - ``port_vlan_add``: bridge layer function invoked when a VLAN is configured - (tagged or untagged) for the given switch port. If the operation is not - supported by the hardware, this function should return ``-EOPNOTSUPP`` to - inform the bridge code to fallback to a software implementation. + (tagged or untagged) for the given switch port. The CPU port becomes a member + of a VLAN only if a foreign bridge port is also a member of it (and + forwarding needs to take place in software), or the VLAN is installed to the + VLAN group of the bridge device itself, for termination purposes + (``bridge vlan add dev br0 vid 100 self``). VLANs on shared ports are + reference counted and removed when there is no user left. Drivers do not need + to manually install a VLAN on the CPU port. - ``port_vlan_del``: bridge layer function invoked when a VLAN is removed from the given switch port -- ``port_vlan_dump``: bridge layer function invoked with a switchdev callback - function that the driver has to call for each VLAN the given port is a member - of. A switchdev object is used to carry the VID and bridge flags. - - ``port_fdb_add``: bridge layer function invoked when the bridge wants to install a Forwarding Database entry, the switch hardware should be programmed with the specified address in the specified VLAN Id in the forwarding database - associated with this VLAN ID. If the operation is not supported, this - function should return ``-EOPNOTSUPP`` to inform the bridge code to fallback to - a software implementation. - -.. note:: VLAN ID 0 corresponds to the port private database, which, in the context - of DSA, would be its port-based VLAN, used by the associated bridge device. + associated with this VLAN ID. - ``port_fdb_del``: bridge layer function invoked when the bridge wants to remove a Forwarding Database entry, the switch hardware should be programmed to delete the specified MAC address from the specified VLAN ID if it was mapped into this port forwarding database -- ``port_fdb_dump``: bridge layer function invoked with a switchdev callback - function that the driver has to call for each MAC address known to be behind - the given port. A switchdev object is used to carry the VID and FDB info. +- ``port_fdb_dump``: bridge bypass function invoked by ``ndo_fdb_dump`` on the + physical DSA port interfaces. Since DSA does not attempt to keep in sync its + hardware FDB entries with the software bridge, this method is implemented as + a means to view the entries visible on user ports in the hardware database. + The entries reported by this function have the ``self`` flag in the output of + the ``bridge fdb show`` command. - ``port_mdb_add``: bridge layer function invoked when the bridge wants to install - a multicast database entry. If the operation is not supported, this function - should return ``-EOPNOTSUPP`` to inform the bridge code to fallback to a - software implementation. The switch hardware should be programmed with the + a multicast database entry. The switch hardware should be programmed with the specified address in the specified VLAN ID in the forwarding database associated with this VLAN ID. -.. note:: VLAN ID 0 corresponds to the port private database, which, in the context - of DSA, would be its port-based VLAN, used by the associated bridge device. - - ``port_mdb_del``: bridge layer function invoked when the bridge wants to remove a multicast database entry, the switch hardware should be programmed to delete the specified MAC address from the specified VLAN ID if it was mapped into this port forwarding database. -- ``port_mdb_dump``: bridge layer function invoked with a switchdev callback - function that the driver has to call for each MAC address known to be behind - the given port. A switchdev object is used to carry the VID and MDB info. - Link aggregation ---------------- diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index 2b329042b38c..5879ef3bc2cb 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -1058,11 +1058,7 @@ udp_rmem_min - INTEGER Default: 4K udp_wmem_min - INTEGER - Minimal size of send buffer used by UDP sockets in moderation. - Each UDP socket is able to use the size for sending data, even if - total pages of UDP sockets exceed udp_mem pressure. The unit is byte. - - Default: 4K + UDP does not have tx memory accounting and this tunable has no effect. RAW variables ============= @@ -1633,12 +1629,15 @@ arp_notify - BOOLEAN or hardware address changes. == ========================================================== -arp_accept - BOOLEAN - Define behavior for gratuitous ARP frames who's IP is not - already present in the ARP table: +arp_accept - INTEGER + Define behavior for accepting gratuitous ARP (garp) frames from devices + that are not already present in the ARP table: - 0 - don't create new entries in the ARP table - 1 - create new entries in the ARP table + - 2 - create new entries only if the source IP address is in the same + subnet as an address configured on the interface that received the + garp message. Both replies and requests type gratuitous arp will trigger the ARP table to be updated, if this setting is on. @@ -2480,27 +2479,36 @@ drop_unsolicited_na - BOOLEAN By default this is turned off. -accept_untracked_na - BOOLEAN - Add a new neighbour cache entry in STALE state for routers on receiving a - neighbour advertisement (either solicited or unsolicited) with target - link-layer address option specified if no neighbour entry is already - present for the advertised IPv6 address. Without this knob, NAs received - for untracked addresses (absent in neighbour cache) are silently ignored. +accept_untracked_na - INTEGER + Define behavior for accepting neighbor advertisements from devices that + are absent in the neighbor cache: - This is as per router-side behaviour documented in RFC9131. + - 0 - (default) Do not accept unsolicited and untracked neighbor + advertisements. - This has lower precedence than drop_unsolicited_na. + - 1 - Add a new neighbor cache entry in STALE state for routers on + receiving a neighbor advertisement (either solicited or unsolicited) + with target link-layer address option specified if no neighbor entry + is already present for the advertised IPv6 address. Without this knob, + NAs received for untracked addresses (absent in neighbor cache) are + silently ignored. - This will optimize the return path for the initial off-link communication - that is initiated by a directly connected host, by ensuring that - the first-hop router which turns on this setting doesn't have to - buffer the initial return packets to do neighbour-solicitation. - The prerequisite is that the host is configured to send - unsolicited neighbour advertisements on interface bringup. - This setting should be used in conjunction with the ndisc_notify setting - on the host to satisfy this prerequisite. + This is as per router-side behavior documented in RFC9131. - By default this is turned off. + This has lower precedence than drop_unsolicited_na. + + This will optimize the return path for the initial off-link + communication that is initiated by a directly connected host, by + ensuring that the first-hop router which turns on this setting doesn't + have to buffer the initial return packets to do neighbor-solicitation. + The prerequisite is that the host is configured to send unsolicited + neighbor advertisements on interface bringup. This setting should be + used in conjunction with the ndisc_notify setting on the host to + satisfy this prerequisite. + + - 2 - Extend option (1) to add a new neighbor cache entry only if the + source IP address is in the same subnet as an address configured on + the interface that received the neighbor advertisement. enhanced_dad - BOOLEAN Include a nonce option in the IPv6 neighbor solicitation messages used for diff --git a/Documentation/networking/smc-sysctl.rst b/Documentation/networking/smc-sysctl.rst index 0987fd1bc220..742e90e6d822 100644 --- a/Documentation/networking/smc-sysctl.rst +++ b/Documentation/networking/smc-sysctl.rst @@ -21,3 +21,16 @@ autocorking_size - INTEGER know how/when to uncork their sockets. Default: 64K + +smcr_buf_type - INTEGER + Controls which type of sndbufs and RMBs to use in later newly created + SMC-R link group. Only for SMC-R. + + Default: 0 (physically contiguous sndbufs and RMBs) + + Possible values: + + - 0 - Use physically contiguous buffers + - 1 - Use virtually contiguous buffers + - 2 - Mixed use of the two types. Try physically contiguous buffers first. + If not available, use virtually contiguous buffers then. diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 11e00a46c610..6e090fb96a0e 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -5657,6 +5657,7 @@ by a string of size ``name_size``. #define KVM_STATS_UNIT_BYTES (0x1 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_SECONDS (0x2 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_CYCLES (0x3 << KVM_STATS_UNIT_SHIFT) + #define KVM_STATS_UNIT_BOOLEAN (0x4 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_MAX KVM_STATS_UNIT_CYCLES #define KVM_STATS_BASE_SHIFT 8 @@ -5702,14 +5703,13 @@ Bits 0-3 of ``flags`` encode the type: by the ``hist_param`` field. The range of the Nth bucket (1 <= N < ``size``) is [``hist_param``*(N-1), ``hist_param``*N), while the range of the last bucket is [``hist_param``*(``size``-1), +INF). (+INF means positive infinity - value.) The bucket value indicates how many samples fell in the bucket's range. + value.) * ``KVM_STATS_TYPE_LOG_HIST`` The statistic is reported as a logarithmic histogram. The number of buckets is specified by the ``size`` field. The range of the first bucket is [0, 1), while the range of the last bucket is [pow(2, ``size``-2), +INF). Otherwise, The Nth bucket (1 < N < ``size``) covers - [pow(2, N-2), pow(2, N-1)). The bucket value indicates how many samples fell - in the bucket's range. + [pow(2, N-2), pow(2, N-1)). Bits 4-7 of ``flags`` encode the unit: @@ -5724,6 +5724,15 @@ Bits 4-7 of ``flags`` encode the unit: It indicates that the statistics data is used to measure time or latency. * ``KVM_STATS_UNIT_CYCLES`` It indicates that the statistics data is used to measure CPU clock cycles. + * ``KVM_STATS_UNIT_BOOLEAN`` + It indicates that the statistic will always be either 0 or 1. Boolean + statistics of "peak" type will never go back from 1 to 0. Boolean + statistics can be linear histograms (with two buckets) but not logarithmic + histograms. + +Note that, in the case of histograms, the unit applies to the bucket +ranges, while the bucket value indicates how many samples fell in the +bucket's range. Bits 8-11 of ``flags``, together with ``exponent``, encode the scale of the unit: @@ -5746,7 +5755,7 @@ the corresponding statistics data. The ``bucket_size`` field is used as a parameter for histogram statistics data. It is only used by linear histogram statistics data, specifying the size of a -bucket. +bucket in the unit expressed by bits 4-11 of ``flags`` together with ``exponent``. The ``name`` field is the name string of the statistics data. The name string starts at the end of ``struct kvm_stats_desc``. The maximum length including diff --git a/MAINTAINERS b/MAINTAINERS index 66738c8330db..46b345ddc67c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2497,10 +2497,8 @@ F: drivers/power/reset/oxnas-restart.c N: oxnas ARM/PALM TREO SUPPORT -M: Tomas Cech <sleep_walker@suse.com> L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) -S: Maintained -W: http://hackndev.com +S: Orphan F: arch/arm/mach-pxa/palmtreo.* ARM/PALMTX,PALMT5,PALMLD,PALMTE2,PALMTC SUPPORT @@ -14377,7 +14375,8 @@ S: Maintained F: drivers/net/phy/nxp-c45-tja11xx.c NXP FSPI DRIVER -M: Ashish Kumar <ashish.kumar@nxp.com> +M: Han Xu <han.xu@nxp.com> +M: Haibo Chen <haibo.chen@nxp.com> R: Yogesh Gaur <yogeshgaur.83@gmail.com> L: linux-spi@vger.kernel.org S: Maintained @@ -17300,12 +17299,15 @@ N: riscv K: riscv RISC-V/MICROCHIP POLARFIRE SOC SUPPORT -M: Lewis Hanly <lewis.hanly@microchip.com> M: Conor Dooley <conor.dooley@microchip.com> +M: Daire McNamara <daire.mcnamara@microchip.com> L: linux-riscv@lists.infradead.org S: Supported F: arch/riscv/boot/dts/microchip/ +F: drivers/char/hw_random/mpfs-rng.c +F: drivers/clk/microchip/clk-mpfs.c F: drivers/mailbox/mailbox-mpfs.c +F: drivers/pci/controller/pcie-microchip-host.c F: drivers/soc/microchip/ F: include/soc/microchip/mpfs.h @@ -2,7 +2,7 @@ VERSION = 5 PATCHLEVEL = 19 SUBLEVEL = 0 -EXTRAVERSION = -rc6 +EXTRAVERSION = -rc7 NAME = Superb Owl # *DOCUMENTATION* diff --git a/arch/Kconfig b/arch/Kconfig index fcf9a41a4ef5..71b9272acb28 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -438,6 +438,13 @@ config MMU_GATHER_PAGE_SIZE config MMU_GATHER_NO_RANGE bool + select MMU_GATHER_MERGE_VMAS + +config MMU_GATHER_NO_FLUSH_CACHE + bool + +config MMU_GATHER_MERGE_VMAS + bool config MMU_GATHER_NO_GATHER bool diff --git a/arch/arm/boot/dts/imx6qdl-ts7970.dtsi b/arch/arm/boot/dts/imx6qdl-ts7970.dtsi index fded07f370b3..d6ba4b2a60f6 100644 --- a/arch/arm/boot/dts/imx6qdl-ts7970.dtsi +++ b/arch/arm/boot/dts/imx6qdl-ts7970.dtsi @@ -226,7 +226,7 @@ reg = <0x28>; #gpio-cells = <2>; gpio-controller; - ngpio = <32>; + ngpios = <62>; }; sgtl5000: codec@a { diff --git a/arch/arm/boot/dts/imx6ull-colibri.dtsi b/arch/arm/boot/dts/imx6ull-colibri.dtsi index 15621e03fa4d..2c3ae715c683 100644 --- a/arch/arm/boot/dts/imx6ull-colibri.dtsi +++ b/arch/arm/boot/dts/imx6ull-colibri.dtsi @@ -166,7 +166,7 @@ atmel_mxt_ts: touchscreen@4a { compatible = "atmel,maxtouch"; pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_atmel_conn>; + pinctrl-0 = <&pinctrl_atmel_conn &pinctrl_atmel_snvs_conn>; reg = <0x4a>; interrupt-parent = <&gpio5>; interrupts = <4 IRQ_TYPE_EDGE_FALLING>; /* SODIMM 107 / INT */ @@ -331,7 +331,6 @@ pinctrl_atmel_conn: atmelconngrp { fsl,pins = < MX6UL_PAD_JTAG_MOD__GPIO1_IO10 0xb0a0 /* SODIMM 106 */ - MX6ULL_PAD_SNVS_TAMPER4__GPIO5_IO04 0xb0a0 /* SODIMM 107 */ >; }; @@ -684,6 +683,12 @@ }; &iomuxc_snvs { + pinctrl_atmel_snvs_conn: atmelsnvsconngrp { + fsl,pins = < + MX6ULL_PAD_SNVS_TAMPER4__GPIO5_IO04 0xb0a0 /* SODIMM 107 */ + >; + }; + pinctrl_snvs_gpio1: snvsgpio1grp { fsl,pins = < MX6ULL_PAD_SNVS_TAMPER6__GPIO5_IO06 0x110a0 /* SODIMM 93 */ diff --git a/arch/arm/boot/dts/lan966x-kontron-kswitch-d10-mmt.dtsi b/arch/arm/boot/dts/lan966x-kontron-kswitch-d10-mmt.dtsi index 4cab1b3b3b29..725dcf707b31 100644 --- a/arch/arm/boot/dts/lan966x-kontron-kswitch-d10-mmt.dtsi +++ b/arch/arm/boot/dts/lan966x-kontron-kswitch-d10-mmt.dtsi @@ -87,22 +87,22 @@ phy4: ethernet-phy@5 { reg = <5>; - coma-mode-gpios = <&gpio 37 GPIO_ACTIVE_HIGH>; + coma-mode-gpios = <&gpio 37 GPIO_OPEN_DRAIN>; }; phy5: ethernet-phy@6 { reg = <6>; - coma-mode-gpios = <&gpio 37 GPIO_ACTIVE_HIGH>; + coma-mode-gpios = <&gpio 37 GPIO_OPEN_DRAIN>; }; phy6: ethernet-phy@7 { reg = <7>; - coma-mode-gpios = <&gpio 37 GPIO_ACTIVE_HIGH>; + coma-mode-gpios = <&gpio 37 GPIO_OPEN_DRAIN>; }; phy7: ethernet-phy@8 { reg = <8>; - coma-mode-gpios = <&gpio 37 GPIO_ACTIVE_HIGH>; + coma-mode-gpios = <&gpio 37 GPIO_OPEN_DRAIN>; }; }; diff --git a/arch/arm/boot/dts/qcom-msm8974.dtsi b/arch/arm/boot/dts/qcom-msm8974.dtsi index 814ad0b46232..c3b8a6d63027 100644 --- a/arch/arm/boot/dts/qcom-msm8974.dtsi +++ b/arch/arm/boot/dts/qcom-msm8974.dtsi @@ -506,6 +506,8 @@ interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_BLSP1_UART2_APPS_CLK>, <&gcc GCC_BLSP1_AHB_CLK>; clock-names = "core", "iface"; + pinctrl-names = "default"; + pinctrl-0 = <&blsp1_uart2_default>; status = "disabled"; }; @@ -581,6 +583,9 @@ interrupts = <GIC_SPI 113 IRQ_TYPE_NONE>; clocks = <&gcc GCC_BLSP2_UART1_APPS_CLK>, <&gcc GCC_BLSP2_AHB_CLK>; clock-names = "core", "iface"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&blsp2_uart1_default>; + pinctrl-1 = <&blsp2_uart1_sleep>; status = "disabled"; }; @@ -599,6 +604,8 @@ interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_BLSP2_UART4_APPS_CLK>, <&gcc GCC_BLSP2_AHB_CLK>; clock-names = "core", "iface"; + pinctrl-names = "default"; + pinctrl-0 = <&blsp2_uart4_default>; status = "disabled"; }; @@ -639,6 +646,9 @@ interrupts = <0 106 IRQ_TYPE_LEVEL_HIGH>; clocks = <&gcc GCC_BLSP2_QUP6_I2C_APPS_CLK>, <&gcc GCC_BLSP2_AHB_CLK>; clock-names = "core", "iface"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&blsp2_i2c6_default>; + pinctrl-1 = <&blsp2_i2c6_sleep>; #address-cells = <1>; #size-cells = <0>; }; @@ -1256,7 +1266,7 @@ }; }; - blsp1_uart2_active: blsp1-uart2-active { + blsp1_uart2_default: blsp1-uart2-default { rx { pins = "gpio5"; function = "blsp_uart2"; @@ -1272,7 +1282,7 @@ }; }; - blsp2_uart1_active: blsp2-uart1-active { + blsp2_uart1_default: blsp2-uart1-default { tx-rts { pins = "gpio41", "gpio44"; function = "blsp_uart7"; @@ -1295,7 +1305,7 @@ bias-pull-down; }; - blsp2_uart4_active: blsp2-uart4-active { + blsp2_uart4_default: blsp2-uart4-default { tx-rts { pins = "gpio53", "gpio56"; function = "blsp_uart10"; @@ -1406,7 +1416,19 @@ bias-pull-up; }; - /* BLSP2_I2C6 info is missing - nobody uses it though? */ + blsp2_i2c6_default: blsp2-i2c6-default { + pins = "gpio87", "gpio88"; + function = "blsp_i2c12"; + drive-strength = <2>; + bias-disable; + }; + + blsp2_i2c6_sleep: blsp2-i2c6-sleep { + pins = "gpio87", "gpio88"; + function = "blsp_i2c12"; + drive-strength = <2>; + bias-pull-up; + }; spi8_default: spi8_default { mosi { diff --git a/arch/arm/boot/dts/sama5d2.dtsi b/arch/arm/boot/dts/sama5d2.dtsi index 89c71d419f82..659a17fc755c 100644 --- a/arch/arm/boot/dts/sama5d2.dtsi +++ b/arch/arm/boot/dts/sama5d2.dtsi @@ -1124,7 +1124,7 @@ clocks = <&pmc PMC_TYPE_PERIPHERAL 55>, <&pmc PMC_TYPE_GCK 55>; clock-names = "pclk", "gclk"; assigned-clocks = <&pmc PMC_TYPE_CORE PMC_I2S1_MUX>; - assigned-parrents = <&pmc PMC_TYPE_GCK 55>; + assigned-clock-parents = <&pmc PMC_TYPE_GCK 55>; status = "disabled"; }; diff --git a/arch/arm/boot/dts/sun8i-h2-plus-orangepi-zero.dts b/arch/arm/boot/dts/sun8i-h2-plus-orangepi-zero.dts index f19ed981da9d..3706216ffb40 100644 --- a/arch/arm/boot/dts/sun8i-h2-plus-orangepi-zero.dts +++ b/arch/arm/boot/dts/sun8i-h2-plus-orangepi-zero.dts @@ -169,7 +169,7 @@ flash@0 { #address-cells = <1>; #size-cells = <1>; - compatible = "mxicy,mx25l1606e", "winbond,w25q128"; + compatible = "mxicy,mx25l1606e", "jedec,spi-nor"; reg = <0>; spi-max-frequency = <40000000>; }; diff --git a/arch/arm/mach-rockchip/pm.c b/arch/arm/mach-rockchip/pm.c index 87389d9456b9..30d781d80fe0 100644 --- a/arch/arm/mach-rockchip/pm.c +++ b/arch/arm/mach-rockchip/pm.c @@ -311,7 +311,7 @@ void __init rockchip_suspend_init(void) &match); if (!match) { pr_err("Failed to find PMU node\n"); - return; + goto out_put; } pm_data = (struct rockchip_pm_data *) match->data; @@ -320,9 +320,12 @@ void __init rockchip_suspend_init(void) if (ret) { pr_err("%s: matches init error %d\n", __func__, ret); - return; + goto out_put; } } suspend_set_ops(pm_data->ops); + +out_put: + of_node_put(np); } diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906.dtsi b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906.dtsi index 66023d553524..d084c33d5ca8 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906.dtsi @@ -9,6 +9,14 @@ /delete-node/ cpu@3; }; + timer { + compatible = "arm,armv8-timer"; + interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>; + }; + pmu { compatible = "arm,cortex-a53-pmu"; interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>, diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi index a4be040a00c0..967d2cd3c3ce 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi @@ -29,6 +29,8 @@ device_type = "cpu"; compatible = "brcm,brahma-b53"; reg = <0x0>; + enable-method = "spin-table"; + cpu-release-addr = <0x0 0xfff8>; next-level-cache = <&l2>; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi index 92465f777603..d5cdd77e5a95 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi @@ -224,9 +224,12 @@ little-endian; }; - efuse@1e80000 { + sfp: efuse@1e80000 { compatible = "fsl,ls1028a-sfp"; reg = <0x0 0x1e80000 0x0 0x10000>; + clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL + QORIQ_CLK_PLL_DIV(4)>; + clock-names = "sfp"; #address-cells = <1>; #size-cells = <1>; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi index 913d845eb51a..1977103a5ef4 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi @@ -376,7 +376,8 @@ camera: &i2c7 { <&cru ACLK_VIO>, <&cru ACLK_GIC_PRE>, <&cru PCLK_DDR>, - <&cru ACLK_HDCP>; + <&cru ACLK_HDCP>, + <&cru ACLK_VDU>; assigned-clock-rates = <600000000>, <1600000000>, <1000000000>, @@ -388,6 +389,7 @@ camera: &i2c7 { <400000000>, <200000000>, <200000000>, + <400000000>, <400000000>; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi index fbd0346624e6..9d5b0e8c9cca 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi @@ -1462,7 +1462,8 @@ <&cru HCLK_PERILP1>, <&cru PCLK_PERILP1>, <&cru ACLK_VIO>, <&cru ACLK_HDCP>, <&cru ACLK_GIC_PRE>, - <&cru PCLK_DDR>; + <&cru PCLK_DDR>, + <&cru ACLK_VDU>; assigned-clock-rates = <594000000>, <800000000>, <1000000000>, @@ -1473,7 +1474,8 @@ <100000000>, <50000000>, <400000000>, <400000000>, <200000000>, - <200000000>; + <200000000>, + <400000000>; }; grf: syscon@ff770000 { diff --git a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts index 1534e11a9ad1..fa953b736642 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts +++ b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts @@ -687,6 +687,7 @@ }; &usb_host0_xhci { + dr_mode = "host"; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts index 7bdcecc0dfe4..02d5f5a8ca03 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts +++ b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts @@ -133,7 +133,7 @@ assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru SCLK_GMAC1>; assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru SCLK_GMAC1>, <&gmac1_clkin>; clock_in_out = "input"; - phy-mode = "rgmii-id"; + phy-mode = "rgmii"; phy-supply = <&vcc_3v3>; pinctrl-names = "default"; pinctrl-0 = <&gmac1m1_miim diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 6aa2dc836db1..834bff720582 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -510,6 +510,9 @@ u32 aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg, unsigned int imm, enum aarch64_insn_size_type size, enum aarch64_insn_ldst_type type); +u32 aarch64_insn_gen_load_literal(unsigned long pc, unsigned long addr, + enum aarch64_insn_register reg, + bool is64bit); u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1, enum aarch64_insn_register reg2, enum aarch64_insn_register base, diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c index 695d7368fadc..49e972beeac7 100644 --- a/arch/arm64/lib/insn.c +++ b/arch/arm64/lib/insn.c @@ -323,7 +323,7 @@ static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type, return insn; } -static inline long branch_imm_common(unsigned long pc, unsigned long addr, +static inline long label_imm_common(unsigned long pc, unsigned long addr, long range) { long offset; @@ -354,7 +354,7 @@ u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, * ARM64 virtual address arrangement guarantees all kernel and module * texts are within +/-128M. */ - offset = branch_imm_common(pc, addr, SZ_128M); + offset = label_imm_common(pc, addr, SZ_128M); if (offset >= SZ_128M) return AARCH64_BREAK_FAULT; @@ -382,7 +382,7 @@ u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr, u32 insn; long offset; - offset = branch_imm_common(pc, addr, SZ_1M); + offset = label_imm_common(pc, addr, SZ_1M); if (offset >= SZ_1M) return AARCH64_BREAK_FAULT; @@ -421,7 +421,7 @@ u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, u32 insn; long offset; - offset = branch_imm_common(pc, addr, SZ_1M); + offset = label_imm_common(pc, addr, SZ_1M); insn = aarch64_insn_get_bcond_value(); @@ -543,6 +543,28 @@ u32 aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg, return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm); } +u32 aarch64_insn_gen_load_literal(unsigned long pc, unsigned long addr, + enum aarch64_insn_register reg, + bool is64bit) +{ + u32 insn; + long offset; + + offset = label_imm_common(pc, addr, SZ_1M); + if (offset >= SZ_1M) + return AARCH64_BREAK_FAULT; + + insn = aarch64_insn_get_ldr_lit_value(); + + if (is64bit) + insn |= BIT(30); + + insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); + + return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, + offset >> 2); +} + u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1, enum aarch64_insn_register reg2, enum aarch64_insn_register base, diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h index 194c95ccc1cf..a6acb94ea3d6 100644 --- a/arch/arm64/net/bpf_jit.h +++ b/arch/arm64/net/bpf_jit.h @@ -80,6 +80,12 @@ #define A64_STR64I(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 64, STORE) #define A64_LDR64I(Xt, Xn, imm) A64_LS_IMM(Xt, Xn, imm, 64, LOAD) +/* LDR (literal) */ +#define A64_LDR32LIT(Wt, offset) \ + aarch64_insn_gen_load_literal(0, offset, Wt, false) +#define A64_LDR64LIT(Xt, offset) \ + aarch64_insn_gen_load_literal(0, offset, Xt, true) + /* Load/store register pair */ #define A64_LS_PAIR(Rt, Rt2, Rn, offset, ls, type) \ aarch64_insn_gen_load_store_pair(Rt, Rt2, Rn, offset, \ @@ -270,6 +276,7 @@ #define A64_BTI_C A64_HINT(AARCH64_INSN_HINT_BTIC) #define A64_BTI_J A64_HINT(AARCH64_INSN_HINT_BTIJ) #define A64_BTI_JC A64_HINT(AARCH64_INSN_HINT_BTIJC) +#define A64_NOP A64_HINT(AARCH64_INSN_HINT_NOP) /* DMB */ #define A64_DMB_ISH aarch64_insn_gen_dmb(AARCH64_INSN_MB_ISH) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index f08a4447d363..7ca8779ae34f 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -10,6 +10,7 @@ #include <linux/bitfield.h> #include <linux/bpf.h> #include <linux/filter.h> +#include <linux/memory.h> #include <linux/printk.h> #include <linux/slab.h> @@ -18,6 +19,7 @@ #include <asm/cacheflush.h> #include <asm/debug-monitors.h> #include <asm/insn.h> +#include <asm/patching.h> #include <asm/set_memory.h> #include "bpf_jit.h" @@ -78,6 +80,15 @@ struct jit_ctx { int fpb_offset; }; +struct bpf_plt { + u32 insn_ldr; /* load target */ + u32 insn_br; /* branch to target */ + u64 target; /* target value */ +}; + +#define PLT_TARGET_SIZE sizeof_field(struct bpf_plt, target) +#define PLT_TARGET_OFFSET offsetof(struct bpf_plt, target) + static inline void emit(const u32 insn, struct jit_ctx *ctx) { if (ctx->image != NULL) @@ -140,6 +151,12 @@ static inline void emit_a64_mov_i64(const int reg, const u64 val, } } +static inline void emit_bti(u32 insn, struct jit_ctx *ctx) +{ + if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) + emit(insn, ctx); +} + /* * Kernel addresses in the vmalloc space use at most 48 bits, and the * remaining bits are guaranteed to be 0x1. So we can compose the address @@ -159,6 +176,14 @@ static inline void emit_addr_mov_i64(const int reg, const u64 val, } } +static inline void emit_call(u64 target, struct jit_ctx *ctx) +{ + u8 tmp = bpf2a64[TMP_REG_1]; + + emit_addr_mov_i64(tmp, target, ctx); + emit(A64_BLR(tmp), ctx); +} + static inline int bpf2a64_offset(int bpf_insn, int off, const struct jit_ctx *ctx) { @@ -235,13 +260,30 @@ static bool is_lsi_offset(int offset, int scale) return true; } +/* generated prologue: + * bti c // if CONFIG_ARM64_BTI_KERNEL + * mov x9, lr + * nop // POKE_OFFSET + * paciasp // if CONFIG_ARM64_PTR_AUTH_KERNEL + * stp x29, lr, [sp, #-16]! + * mov x29, sp + * stp x19, x20, [sp, #-16]! + * stp x21, x22, [sp, #-16]! + * stp x25, x26, [sp, #-16]! + * stp x27, x28, [sp, #-16]! + * mov x25, sp + * mov tcc, #0 + * // PROLOGUE_OFFSET + */ + +#define BTI_INSNS (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) ? 1 : 0) +#define PAC_INSNS (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) ? 1 : 0) + +/* Offset of nop instruction in bpf prog entry to be poked */ +#define POKE_OFFSET (BTI_INSNS + 1) + /* Tail call offset to jump into */ -#if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) || \ - IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) -#define PROLOGUE_OFFSET 9 -#else -#define PROLOGUE_OFFSET 8 -#endif +#define PROLOGUE_OFFSET (BTI_INSNS + 2 + PAC_INSNS + 8) static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) { @@ -280,12 +322,14 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) * */ + emit_bti(A64_BTI_C, ctx); + + emit(A64_MOV(1, A64_R(9), A64_LR), ctx); + emit(A64_NOP, ctx); + /* Sign lr */ if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) emit(A64_PACIASP, ctx); - /* BTI landing pad */ - else if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) - emit(A64_BTI_C, ctx); /* Save FP and LR registers to stay align with ARM64 AAPCS */ emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); @@ -312,8 +356,7 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) } /* BTI landing pad for the tail call, done with a BR */ - if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) - emit(A64_BTI_J, ctx); + emit_bti(A64_BTI_J, ctx); } emit(A64_SUB_I(1, fpb, fp, ctx->fpb_offset), ctx); @@ -557,6 +600,53 @@ static int emit_ll_sc_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) return 0; } +void dummy_tramp(void); + +asm ( +" .pushsection .text, \"ax\", @progbits\n" +" .global dummy_tramp\n" +" .type dummy_tramp, %function\n" +"dummy_tramp:" +#if IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) +" bti j\n" /* dummy_tramp is called via "br x10" */ +#endif +" mov x10, x30\n" +" mov x30, x9\n" +" ret x10\n" +" .size dummy_tramp, .-dummy_tramp\n" +" .popsection\n" +); + +/* build a plt initialized like this: + * + * plt: + * ldr tmp, target + * br tmp + * target: + * .quad dummy_tramp + * + * when a long jump trampoline is attached, target is filled with the + * trampoline address, and when the trampoline is removed, target is + * restored to dummy_tramp address. + */ +static void build_plt(struct jit_ctx *ctx) +{ + const u8 tmp = bpf2a64[TMP_REG_1]; + struct bpf_plt *plt = NULL; + + /* make sure target is 64-bit aligned */ + if ((ctx->idx + PLT_TARGET_OFFSET / AARCH64_INSN_SIZE) % 2) + emit(A64_NOP, ctx); + + plt = (struct bpf_plt *)(ctx->image + ctx->idx); + /* plt is called via bl, no BTI needed here */ + emit(A64_LDR64LIT(tmp, 2 * AARCH64_INSN_SIZE), ctx); + emit(A64_BR(tmp), ctx); + + if (ctx->image) + plt->target = (u64)&dummy_tramp; +} + static void build_epilogue(struct jit_ctx *ctx) { const u8 r0 = bpf2a64[BPF_REG_0]; @@ -991,8 +1081,7 @@ emit_cond_jmp: &func_addr, &func_addr_fixed); if (ret < 0) return ret; - emit_addr_mov_i64(tmp, func_addr, ctx); - emit(A64_BLR(tmp), ctx); + emit_call(func_addr, ctx); emit(A64_MOV(1, r0, A64_R(0)), ctx); break; } @@ -1336,6 +1425,13 @@ static int validate_code(struct jit_ctx *ctx) if (a64_insn == AARCH64_BREAK_FAULT) return -1; } + return 0; +} + +static int validate_ctx(struct jit_ctx *ctx) +{ + if (validate_code(ctx)) + return -1; if (WARN_ON_ONCE(ctx->exentry_idx != ctx->prog->aux->num_exentries)) return -1; @@ -1356,7 +1452,7 @@ struct arm64_jit_data { struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { - int image_size, prog_size, extable_size; + int image_size, prog_size, extable_size, extable_align, extable_offset; struct bpf_prog *tmp, *orig_prog = prog; struct bpf_binary_header *header; struct arm64_jit_data *jit_data; @@ -1426,13 +1522,17 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx.epilogue_offset = ctx.idx; build_epilogue(&ctx); + build_plt(&ctx); + extable_align = __alignof__(struct exception_table_entry); extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry); /* Now we know the actual image size. */ prog_size = sizeof(u32) * ctx.idx; - image_size = prog_size + extable_size; + /* also allocate space for plt target */ + extable_offset = round_up(prog_size + PLT_TARGET_SIZE, extable_align); + image_size = extable_offset + extable_size; header = bpf_jit_binary_alloc(image_size, &image_ptr, sizeof(u32), jit_fill_hole); if (header == NULL) { @@ -1444,7 +1544,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx.image = (__le32 *)image_ptr; if (extable_size) - prog->aux->extable = (void *)image_ptr + prog_size; + prog->aux->extable = (void *)image_ptr + extable_offset; skip_init_ctx: ctx.idx = 0; ctx.exentry_idx = 0; @@ -1458,9 +1558,10 @@ skip_init_ctx: } build_epilogue(&ctx); + build_plt(&ctx); /* 3. Extra pass to validate JITed code. */ - if (validate_code(&ctx)) { + if (validate_ctx(&ctx)) { bpf_jit_binary_free(header); prog = orig_prog; goto out_off; @@ -1537,3 +1638,583 @@ bool bpf_jit_supports_subprog_tailcalls(void) { return true; } + +static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, + int args_off, int retval_off, int run_ctx_off, + bool save_ret) +{ + u32 *branch; + u64 enter_prog; + u64 exit_prog; + struct bpf_prog *p = l->link.prog; + int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); + + if (p->aux->sleepable) { + enter_prog = (u64)__bpf_prog_enter_sleepable; + exit_prog = (u64)__bpf_prog_exit_sleepable; + } else { + enter_prog = (u64)__bpf_prog_enter; + exit_prog = (u64)__bpf_prog_exit; + } + + if (l->cookie == 0) { + /* if cookie is zero, one instruction is enough to store it */ + emit(A64_STR64I(A64_ZR, A64_SP, run_ctx_off + cookie_off), ctx); + } else { + emit_a64_mov_i64(A64_R(10), l->cookie, ctx); + emit(A64_STR64I(A64_R(10), A64_SP, run_ctx_off + cookie_off), + ctx); + } + + /* save p to callee saved register x19 to avoid loading p with mov_i64 + * each time. + */ + emit_addr_mov_i64(A64_R(19), (const u64)p, ctx); + + /* arg1: prog */ + emit(A64_MOV(1, A64_R(0), A64_R(19)), ctx); + /* arg2: &run_ctx */ + emit(A64_ADD_I(1, A64_R(1), A64_SP, run_ctx_off), ctx); + + emit_call(enter_prog, ctx); + + /* if (__bpf_prog_enter(prog) == 0) + * goto skip_exec_of_prog; + */ + branch = ctx->image + ctx->idx; + emit(A64_NOP, ctx); + + /* save return value to callee saved register x20 */ + emit(A64_MOV(1, A64_R(20), A64_R(0)), ctx); + + emit(A64_ADD_I(1, A64_R(0), A64_SP, args_off), ctx); + if (!p->jited) + emit_addr_mov_i64(A64_R(1), (const u64)p->insnsi, ctx); + + emit_call((const u64)p->bpf_func, ctx); + + if (save_ret) + emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx); + + if (ctx->image) { + int offset = &ctx->image[ctx->idx] - branch; + *branch = A64_CBZ(1, A64_R(0), offset); + } + + /* arg1: prog */ + emit(A64_MOV(1, A64_R(0), A64_R(19)), ctx); + /* arg2: start time */ + emit(A64_MOV(1, A64_R(1), A64_R(20)), ctx); + /* arg3: &run_ctx */ + emit(A64_ADD_I(1, A64_R(2), A64_SP, run_ctx_off), ctx); + + emit_call(exit_prog, ctx); +} + +static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl, + int args_off, int retval_off, int run_ctx_off, + u32 **branches) +{ + int i; + + /* The first fmod_ret program will receive a garbage return value. + * Set this to 0 to avoid confusing the program. + */ + emit(A64_STR64I(A64_ZR, A64_SP, retval_off), ctx); + for (i = 0; i < tl->nr_links; i++) { + invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, + run_ctx_off, true); + /* if (*(u64 *)(sp + retval_off) != 0) + * goto do_fexit; + */ + emit(A64_LDR64I(A64_R(10), A64_SP, retval_off), ctx); + /* Save the location of branch, and generate a nop. + * This nop will be replaced with a cbnz later. + */ + branches[i] = ctx->image + ctx->idx; + emit(A64_NOP, ctx); + } +} + +static void save_args(struct jit_ctx *ctx, int args_off, int nargs) +{ + int i; + + for (i = 0; i < nargs; i++) { + emit(A64_STR64I(i, A64_SP, args_off), ctx); + args_off += 8; + } +} + +static void restore_args(struct jit_ctx *ctx, int args_off, int nargs) +{ + int i; + + for (i = 0; i < nargs; i++) { + emit(A64_LDR64I(i, A64_SP, args_off), ctx); + args_off += 8; + } +} + +/* Based on the x86's implementation of arch_prepare_bpf_trampoline(). + * + * bpf prog and function entry before bpf trampoline hooked: + * mov x9, lr + * nop + * + * bpf prog and function entry after bpf trampoline hooked: + * mov x9, lr + * bl <bpf_trampoline or plt> + * + */ +static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_links *tlinks, void *orig_call, + int nargs, u32 flags) +{ + int i; + int stack_size; + int retaddr_off; + int regs_off; + int retval_off; + int args_off; + int nargs_off; + int ip_off; + int run_ctx_off; + struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; + struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; + struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; + bool save_ret; + u32 **branches = NULL; + + /* trampoline stack layout: + * [ parent ip ] + * [ FP ] + * SP + retaddr_off [ self ip ] + * [ FP ] + * + * [ padding ] align SP to multiples of 16 + * + * [ x20 ] callee saved reg x20 + * SP + regs_off [ x19 ] callee saved reg x19 + * + * SP + retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or + * BPF_TRAMP_F_RET_FENTRY_RET + * + * [ argN ] + * [ ... ] + * SP + args_off [ arg1 ] + * + * SP + nargs_off [ args count ] + * + * SP + ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag + * + * SP + run_ctx_off [ bpf_tramp_run_ctx ] + */ + + stack_size = 0; + run_ctx_off = stack_size; + /* room for bpf_tramp_run_ctx */ + stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); + + ip_off = stack_size; + /* room for IP address argument */ + if (flags & BPF_TRAMP_F_IP_ARG) + stack_size += 8; + + nargs_off = stack_size; + /* room for args count */ + stack_size += 8; + + args_off = stack_size; + /* room for args */ + stack_size += nargs * 8; + + /* room for return value */ + retval_off = stack_size; + save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); + if (save_ret) + stack_size += 8; + + /* room for callee saved registers, currently x19 and x20 are used */ + regs_off = stack_size; + stack_size += 16; + + /* round up to multiples of 16 to avoid SPAlignmentFault */ + stack_size = round_up(stack_size, 16); + + /* return address locates above FP */ + retaddr_off = stack_size + 8; + + /* bpf trampoline may be invoked by 3 instruction types: + * 1. bl, attached to bpf prog or kernel function via short jump + * 2. br, attached to bpf prog or kernel function via long jump + * 3. blr, working as a function pointer, used by struct_ops. + * So BTI_JC should used here to support both br and blr. + */ + emit_bti(A64_BTI_JC, ctx); + + /* frame for parent function */ + emit(A64_PUSH(A64_FP, A64_R(9), A64_SP), ctx); + emit(A64_MOV(1, A64_FP, A64_SP), ctx); + + /* frame for patched function */ + emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); + emit(A64_MOV(1, A64_FP, A64_SP), ctx); + + /* allocate stack space */ + emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx); + + if (flags & BPF_TRAMP_F_IP_ARG) { + /* save ip address of the traced function */ + emit_addr_mov_i64(A64_R(10), (const u64)orig_call, ctx); + emit(A64_STR64I(A64_R(10), A64_SP, ip_off), ctx); + } + + /* save args count*/ + emit(A64_MOVZ(1, A64_R(10), nargs, 0), ctx); + emit(A64_STR64I(A64_R(10), A64_SP, nargs_off), ctx); + + /* save args */ + save_args(ctx, args_off, nargs); + + /* save callee saved registers */ + emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx); + emit(A64_STR64I(A64_R(20), A64_SP, regs_off + 8), ctx); + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + emit_addr_mov_i64(A64_R(0), (const u64)im, ctx); + emit_call((const u64)__bpf_tramp_enter, ctx); + } + + for (i = 0; i < fentry->nr_links; i++) + invoke_bpf_prog(ctx, fentry->links[i], args_off, + retval_off, run_ctx_off, + flags & BPF_TRAMP_F_RET_FENTRY_RET); + + if (fmod_ret->nr_links) { + branches = kcalloc(fmod_ret->nr_links, sizeof(u32 *), + GFP_KERNEL); + if (!branches) + return -ENOMEM; + + invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off, + run_ctx_off, branches); + } + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + restore_args(ctx, args_off, nargs); + /* call original func */ + emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx); + emit(A64_BLR(A64_R(10)), ctx); + /* store return value */ + emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx); + /* reserve a nop for bpf_tramp_image_put */ + im->ip_after_call = ctx->image + ctx->idx; + emit(A64_NOP, ctx); + } + + /* update the branches saved in invoke_bpf_mod_ret with cbnz */ + for (i = 0; i < fmod_ret->nr_links && ctx->image != NULL; i++) { + int offset = &ctx->image[ctx->idx] - branches[i]; + *branches[i] = A64_CBNZ(1, A64_R(10), offset); + } + + for (i = 0; i < fexit->nr_links; i++) + invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off, + run_ctx_off, false); + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + im->ip_epilogue = ctx->image + ctx->idx; + emit_addr_mov_i64(A64_R(0), (const u64)im, ctx); + emit_call((const u64)__bpf_tramp_exit, ctx); + } + + if (flags & BPF_TRAMP_F_RESTORE_REGS) + restore_args(ctx, args_off, nargs); + + /* restore callee saved register x19 and x20 */ + emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx); + emit(A64_LDR64I(A64_R(20), A64_SP, regs_off + 8), ctx); + + if (save_ret) + emit(A64_LDR64I(A64_R(0), A64_SP, retval_off), ctx); + + /* reset SP */ + emit(A64_MOV(1, A64_SP, A64_FP), ctx); + + /* pop frames */ + emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); + emit(A64_POP(A64_FP, A64_R(9), A64_SP), ctx); + + if (flags & BPF_TRAMP_F_SKIP_FRAME) { + /* skip patched function, return to parent */ + emit(A64_MOV(1, A64_LR, A64_R(9)), ctx); + emit(A64_RET(A64_R(9)), ctx); + } else { + /* return to patched function */ + emit(A64_MOV(1, A64_R(10), A64_LR), ctx); + emit(A64_MOV(1, A64_LR, A64_R(9)), ctx); + emit(A64_RET(A64_R(10)), ctx); + } + + if (ctx->image) + bpf_flush_icache(ctx->image, ctx->image + ctx->idx); + + kfree(branches); + + return ctx->idx; +} + +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, + void *image_end, const struct btf_func_model *m, + u32 flags, struct bpf_tramp_links *tlinks, + void *orig_call) +{ + int ret; + int nargs = m->nr_args; + int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE; + struct jit_ctx ctx = { + .image = NULL, + .idx = 0, + }; + + /* the first 8 arguments are passed by registers */ + if (nargs > 8) + return -ENOTSUPP; + + ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nargs, flags); + if (ret < 0) + return ret; + + if (ret > max_insns) + return -EFBIG; + + ctx.image = image; + ctx.idx = 0; + + jit_fill_hole(image, (unsigned int)(image_end - image)); + ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nargs, flags); + + if (ret > 0 && validate_code(&ctx) < 0) + ret = -EINVAL; + + if (ret > 0) + ret *= AARCH64_INSN_SIZE; + + return ret; +} + +static bool is_long_jump(void *ip, void *target) +{ + long offset; + + /* NULL target means this is a NOP */ + if (!target) + return false; + + offset = (long)target - (long)ip; + return offset < -SZ_128M || offset >= SZ_128M; +} + +static int gen_branch_or_nop(enum aarch64_insn_branch_type type, void *ip, + void *addr, void *plt, u32 *insn) +{ + void *target; + + if (!addr) { + *insn = aarch64_insn_gen_nop(); + return 0; + } + + if (is_long_jump(ip, addr)) + target = plt; + else + target = addr; + + *insn = aarch64_insn_gen_branch_imm((unsigned long)ip, + (unsigned long)target, + type); + + return *insn != AARCH64_BREAK_FAULT ? 0 : -EFAULT; +} + +/* Replace the branch instruction from @ip to @old_addr in a bpf prog or a bpf + * trampoline with the branch instruction from @ip to @new_addr. If @old_addr + * or @new_addr is NULL, the old or new instruction is NOP. + * + * When @ip is the bpf prog entry, a bpf trampoline is being attached or + * detached. Since bpf trampoline and bpf prog are allocated separately with + * vmalloc, the address distance may exceed 128MB, the maximum branch range. + * So long jump should be handled. + * + * When a bpf prog is constructed, a plt pointing to empty trampoline + * dummy_tramp is placed at the end: + * + * bpf_prog: + * mov x9, lr + * nop // patchsite + * ... + * ret + * + * plt: + * ldr x10, target + * br x10 + * target: + * .quad dummy_tramp // plt target + * + * This is also the state when no trampoline is attached. + * + * When a short-jump bpf trampoline is attached, the patchsite is patched + * to a bl instruction to the trampoline directly: + * + * bpf_prog: + * mov x9, lr + * bl <short-jump bpf trampoline address> // patchsite + * ... + * ret + * + * plt: + * ldr x10, target + * br x10 + * target: + * .quad dummy_tramp // plt target + * + * When a long-jump bpf trampoline is attached, the plt target is filled with + * the trampoline address and the patchsite is patched to a bl instruction to + * the plt: + * + * bpf_prog: + * mov x9, lr + * bl plt // patchsite + * ... + * ret + * + * plt: + * ldr x10, target + * br x10 + * target: + * .quad <long-jump bpf trampoline address> // plt target + * + * The dummy_tramp is used to prevent another CPU from jumping to unknown + * locations during the patching process, making the patching process easier. + */ +int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, + void *old_addr, void *new_addr) +{ + int ret; + u32 old_insn; + u32 new_insn; + u32 replaced; + struct bpf_plt *plt = NULL; + unsigned long size = 0UL; + unsigned long offset = ~0UL; + enum aarch64_insn_branch_type branch_type; + char namebuf[KSYM_NAME_LEN]; + void *image = NULL; + u64 plt_target = 0ULL; + bool poking_bpf_entry; + + if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) + /* Only poking bpf text is supported. Since kernel function + * entry is set up by ftrace, we reply on ftrace to poke kernel + * functions. + */ + return -ENOTSUPP; + + image = ip - offset; + /* zero offset means we're poking bpf prog entry */ + poking_bpf_entry = (offset == 0UL); + + /* bpf prog entry, find plt and the real patchsite */ + if (poking_bpf_entry) { + /* plt locates at the end of bpf prog */ + plt = image + size - PLT_TARGET_OFFSET; + + /* skip to the nop instruction in bpf prog entry: + * bti c // if BTI enabled + * mov x9, x30 + * nop + */ + ip = image + POKE_OFFSET * AARCH64_INSN_SIZE; + } + + /* long jump is only possible at bpf prog entry */ + if (WARN_ON((is_long_jump(ip, new_addr) || is_long_jump(ip, old_addr)) && + !poking_bpf_entry)) + return -EINVAL; + + if (poke_type == BPF_MOD_CALL) + branch_type = AARCH64_INSN_BRANCH_LINK; + else + branch_type = AARCH64_INSN_BRANCH_NOLINK; + + if (gen_branch_or_nop(branch_type, ip, old_addr, plt, &old_insn) < 0) + return -EFAULT; + + if (gen_branch_or_nop(branch_type, ip, new_addr, plt, &new_insn) < 0) + return -EFAULT; + + if (is_long_jump(ip, new_addr)) + plt_target = (u64)new_addr; + else if (is_long_jump(ip, old_addr)) + /* if the old target is a long jump and the new target is not, + * restore the plt target to dummy_tramp, so there is always a + * legal and harmless address stored in plt target, and we'll + * never jump from plt to an unknown place. + */ + plt_target = (u64)&dummy_tramp; + + if (plt_target) { + /* non-zero plt_target indicates we're patching a bpf prog, + * which is read only. + */ + if (set_memory_rw(PAGE_MASK & ((uintptr_t)&plt->target), 1)) + return -EFAULT; + WRITE_ONCE(plt->target, plt_target); + set_memory_ro(PAGE_MASK & ((uintptr_t)&plt->target), 1); + /* since plt target points to either the new trampoline + * or dummy_tramp, even if another CPU reads the old plt + * target value before fetching the bl instruction to plt, + * it will be brought back by dummy_tramp, so no barrier is + * required here. + */ + } + + /* if the old target and the new target are both long jumps, no + * patching is required + */ + if (old_insn == new_insn) + return 0; + + mutex_lock(&text_mutex); + if (aarch64_insn_read(ip, &replaced)) { + ret = -EFAULT; + goto out; + } + + if (replaced != old_insn) { + ret = -EFAULT; + goto out; + } + + /* We call aarch64_insn_patch_text_nosync() to replace instruction + * atomically, so no other CPUs will fetch a half-new and half-old + * instruction. But there is chance that another CPU executes the + * old instruction after the patching operation finishes (e.g., + * pipeline not flushed, or icache not synchronized yet). + * + * 1. when a new trampoline is attached, it is not a problem for + * different CPUs to jump to different trampolines temporarily. + * + * 2. when an old trampoline is freed, we should wait for all other + * CPUs to exit the trampoline and make sure the trampoline is no + * longer reachable, since bpf_tramp_image_put() function already + * uses percpu_ref and task-based rcu to do the sync, no need to call + * the sync version here, see bpf_tramp_image_put() for details. + */ + ret = aarch64_insn_patch_text_nosync(ip, new_insn); +out: + mutex_unlock(&text_mutex); + + return ret; +} diff --git a/arch/csky/include/asm/tlb.h b/arch/csky/include/asm/tlb.h index 3498e65f59f8..702861c68874 100644 --- a/arch/csky/include/asm/tlb.h +++ b/arch/csky/include/asm/tlb.h @@ -4,21 +4,6 @@ #define __ASM_CSKY_TLB_H #include <asm/cacheflush.h> - -#define tlb_start_vma(tlb, vma) \ - do { \ - if (!(tlb)->fullmm) \ - flush_cache_range(vma, (vma)->vm_start, (vma)->vm_end); \ - } while (0) - -#define tlb_end_vma(tlb, vma) \ - do { \ - if (!(tlb)->fullmm) \ - flush_tlb_range(vma, (vma)->vm_start, (vma)->vm_end); \ - } while (0) - -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm) - #include <asm-generic/tlb.h> #endif /* __ASM_CSKY_TLB_H */ diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 53a912befb62..b57daee98b89 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -108,6 +108,7 @@ config LOONGARCH select TRACE_IRQFLAGS_SUPPORT select USE_PERCPU_NUMA_NODE_ID select ZONE_DMA32 + select MMU_GATHER_MERGE_VMAS if MMU config 32BIT bool diff --git a/arch/loongarch/include/asm/tlb.h b/arch/loongarch/include/asm/tlb.h index 4f629ae9d5a9..dd24f5898f65 100644 --- a/arch/loongarch/include/asm/tlb.h +++ b/arch/loongarch/include/asm/tlb.h @@ -137,16 +137,6 @@ static inline void invtlb_all(u32 op, u32 info, u64 addr) ); } -/* - * LoongArch doesn't need any special per-pte or per-vma handling, except - * we need to flush cache for area to be unmapped. - */ -#define tlb_start_vma(tlb, vma) \ - do { \ - if (!(tlb)->fullmm) \ - flush_cache_range(vma, vma->vm_start, vma->vm_end); \ - } while (0) -#define tlb_end_vma(tlb, vma) do { } while (0) #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) static void tlb_flush(struct mmu_gather *tlb); diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 7aa12e88c580..c235648fae23 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -256,6 +256,7 @@ config PPC select IRQ_FORCED_THREADING select MMU_GATHER_PAGE_SIZE select MMU_GATHER_RCU_TABLE_FREE + select MMU_GATHER_MERGE_VMAS select MODULES_USE_ELF_RELA select NEED_DMA_MAP_STATE if PPC64 || NOT_COHERENT_CACHE select NEED_PER_CPU_EMBED_FIRST_CHUNK if PPC64 diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h index 09a9ae5f3656..b3de6102a907 100644 --- a/arch/powerpc/include/asm/tlb.h +++ b/arch/powerpc/include/asm/tlb.h @@ -19,8 +19,6 @@ #include <linux/pagemap.h> -#define tlb_start_vma(tlb, vma) do { } while (0) -#define tlb_end_vma(tlb, vma) do { } while (0) #define __tlb_remove_tlb_entry __tlb_remove_tlb_entry #define tlb_flush tlb_flush diff --git a/arch/riscv/boot/dts/microchip/mpfs.dtsi b/arch/riscv/boot/dts/microchip/mpfs.dtsi index 0ec9a143dfd4..3c83e98e82e4 100644 --- a/arch/riscv/boot/dts/microchip/mpfs.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs.dtsi @@ -50,6 +50,7 @@ riscv,isa = "rv64imafdc"; clocks = <&clkcfg CLK_CPU>; tlb-split; + next-level-cache = <&cctrllr>; status = "okay"; cpu1_intc: interrupt-controller { @@ -77,6 +78,7 @@ riscv,isa = "rv64imafdc"; clocks = <&clkcfg CLK_CPU>; tlb-split; + next-level-cache = <&cctrllr>; status = "okay"; cpu2_intc: interrupt-controller { @@ -104,6 +106,7 @@ riscv,isa = "rv64imafdc"; clocks = <&clkcfg CLK_CPU>; tlb-split; + next-level-cache = <&cctrllr>; status = "okay"; cpu3_intc: interrupt-controller { @@ -131,6 +134,7 @@ riscv,isa = "rv64imafdc"; clocks = <&clkcfg CLK_CPU>; tlb-split; + next-level-cache = <&cctrllr>; status = "okay"; cpu4_intc: interrupt-controller { #interrupt-cells = <1>; diff --git a/arch/riscv/errata/sifive/errata.c b/arch/riscv/errata/sifive/errata.c index 672f02b21ce0..1031038423e7 100644 --- a/arch/riscv/errata/sifive/errata.c +++ b/arch/riscv/errata/sifive/errata.c @@ -111,6 +111,7 @@ void __init_or_module sifive_errata_patch_func(struct alt_entry *begin, cpu_apply_errata |= tmp; } } - if (cpu_apply_errata != cpu_req_errata) + if (stage != RISCV_ALTERNATIVES_MODULE && + cpu_apply_errata != cpu_req_errata) warn_miss_errata(cpu_req_errata - cpu_apply_errata); } diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 5c2aba5efbd0..dc42375c2357 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -175,7 +175,7 @@ static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot) static inline unsigned long _pud_pfn(pud_t pud) { - return pud_val(pud) >> _PAGE_PFN_SHIFT; + return __page_val_to_pfn(pud_val(pud)); } static inline pmd_t *pud_pgtable(pud_t pud) @@ -278,13 +278,13 @@ static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot) static inline unsigned long _p4d_pfn(p4d_t p4d) { - return p4d_val(p4d) >> _PAGE_PFN_SHIFT; + return __page_val_to_pfn(p4d_val(p4d)); } static inline pud_t *p4d_pgtable(p4d_t p4d) { if (pgtable_l4_enabled) - return (pud_t *)pfn_to_virt(p4d_val(p4d) >> _PAGE_PFN_SHIFT); + return (pud_t *)pfn_to_virt(__page_val_to_pfn(p4d_val(p4d))); return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) }); } @@ -292,7 +292,7 @@ static inline pud_t *p4d_pgtable(p4d_t p4d) static inline struct page *p4d_page(p4d_t p4d) { - return pfn_to_page(p4d_val(p4d) >> _PAGE_PFN_SHIFT); + return pfn_to_page(__page_val_to_pfn(p4d_val(p4d))); } #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) @@ -347,7 +347,7 @@ static inline void pgd_clear(pgd_t *pgd) static inline p4d_t *pgd_pgtable(pgd_t pgd) { if (pgtable_l5_enabled) - return (p4d_t *)pfn_to_virt(pgd_val(pgd) >> _PAGE_PFN_SHIFT); + return (p4d_t *)pfn_to_virt(__page_val_to_pfn(pgd_val(pgd))); return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) }); } @@ -355,7 +355,7 @@ static inline p4d_t *pgd_pgtable(pgd_t pgd) static inline struct page *pgd_page(pgd_t pgd) { - return pfn_to_page(pgd_val(pgd) >> _PAGE_PFN_SHIFT); + return pfn_to_page(__page_val_to_pfn(pgd_val(pgd))); } #define pgd_page(pgd) pgd_page(pgd) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 1d1be9d9419c..5dbd6610729b 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -261,7 +261,7 @@ static inline pgd_t pfn_pgd(unsigned long pfn, pgprot_t prot) static inline unsigned long _pgd_pfn(pgd_t pgd) { - return pgd_val(pgd) >> _PAGE_PFN_SHIFT; + return __page_val_to_pfn(pgd_val(pgd)); } static inline struct page *pmd_page(pmd_t pmd) @@ -590,14 +590,14 @@ static inline pmd_t pmd_mkinvalid(pmd_t pmd) return __pmd(pmd_val(pmd) & ~(_PAGE_PRESENT|_PAGE_PROT_NONE)); } -#define __pmd_to_phys(pmd) (pmd_val(pmd) >> _PAGE_PFN_SHIFT << PAGE_SHIFT) +#define __pmd_to_phys(pmd) (__page_val_to_pfn(pmd_val(pmd)) << PAGE_SHIFT) static inline unsigned long pmd_pfn(pmd_t pmd) { return ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT); } -#define __pud_to_phys(pud) (pud_val(pud) >> _PAGE_PFN_SHIFT << PAGE_SHIFT) +#define __pud_to_phys(pud) (__page_val_to_pfn(pud_val(pud)) << PAGE_SHIFT) static inline unsigned long pud_pfn(pud_t pud) { diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 1c00695ebee7..9826073fbc67 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -54,7 +54,7 @@ static inline unsigned long gstage_pte_index(gpa_t addr, u32 level) static inline unsigned long gstage_pte_page_vaddr(pte_t pte) { - return (unsigned long)pfn_to_virt(pte_val(pte) >> _PAGE_PFN_SHIFT); + return (unsigned long)pfn_to_virt(__page_val_to_pfn(pte_val(pte))); } static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level) diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 7f4ad5e4373a..f3455dc013fa 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -781,9 +781,11 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) if (kvm_request_pending(vcpu)) { if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) { + kvm_vcpu_srcu_read_unlock(vcpu); rcuwait_wait_event(wait, (!vcpu->arch.power_off) && (!vcpu->arch.pause), TASK_INTERRUPTIBLE); + kvm_vcpu_srcu_read_lock(vcpu); if (vcpu->arch.power_off || vcpu->arch.pause) { /* diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 8cd9e56c629b..5a1a8dfda6f8 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -204,6 +204,7 @@ config S390 select IOMMU_SUPPORT if PCI select MMU_GATHER_NO_GATHER select MMU_GATHER_RCU_TABLE_FREE + select MMU_GATHER_MERGE_VMAS select MODULES_USE_ELF_RELA select NEED_DMA_MAP_STATE if PCI select NEED_SG_DMA_LENGTH if PCI diff --git a/arch/s390/Makefile b/arch/s390/Makefile index 495c68a4df1e..4cb5d17e7ead 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -82,7 +82,7 @@ endif ifdef CONFIG_EXPOLINE ifdef CONFIG_EXPOLINE_EXTERN - KBUILD_LDFLAGS_MODULE += arch/s390/lib/expoline.o + KBUILD_LDFLAGS_MODULE += arch/s390/lib/expoline/expoline.o CC_FLAGS_EXPOLINE := -mindirect-branch=thunk-extern CC_FLAGS_EXPOLINE += -mfunction-return=thunk-extern else @@ -163,6 +163,12 @@ vdso_prepare: prepare0 $(Q)$(MAKE) $(build)=arch/s390/kernel/vdso64 include/generated/vdso64-offsets.h $(if $(CONFIG_COMPAT),$(Q)$(MAKE) \ $(build)=arch/s390/kernel/vdso32 include/generated/vdso32-offsets.h) + +ifdef CONFIG_EXPOLINE_EXTERN +modules_prepare: expoline_prepare +expoline_prepare: + $(Q)$(MAKE) $(build)=arch/s390/lib/expoline arch/s390/lib/expoline/expoline.o +endif endif # Don't use tabs in echo arguments diff --git a/arch/s390/include/asm/nospec-insn.h b/arch/s390/include/asm/nospec-insn.h index d910d71b5bb5..7e9e99523e95 100644 --- a/arch/s390/include/asm/nospec-insn.h +++ b/arch/s390/include/asm/nospec-insn.h @@ -2,8 +2,6 @@ #ifndef _ASM_S390_NOSPEC_ASM_H #define _ASM_S390_NOSPEC_ASM_H -#include <asm/alternative-asm.h> -#include <asm/asm-offsets.h> #include <asm/dwarf.h> #ifdef __ASSEMBLY__ diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h index fe6407f0eb1b..3a5c8fb590e5 100644 --- a/arch/s390/include/asm/tlb.h +++ b/arch/s390/include/asm/tlb.h @@ -27,9 +27,6 @@ static inline void tlb_flush(struct mmu_gather *tlb); static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size); -#define tlb_start_vma(tlb, vma) do { } while (0) -#define tlb_end_vma(tlb, vma) do { } while (0) - #define tlb_flush tlb_flush #define pte_free_tlb pte_free_tlb #define pmd_free_tlb pmd_free_tlb diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index 5d415b3db6d1..580d2e3265cb 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -7,7 +7,6 @@ lib-y += delay.o string.o uaccess.o find.o spinlock.o obj-y += mem.o xor.o lib-$(CONFIG_KPROBES) += probes.o lib-$(CONFIG_UPROBES) += probes.o -obj-$(CONFIG_EXPOLINE_EXTERN) += expoline.o obj-$(CONFIG_S390_KPROBES_SANITY_TEST) += test_kprobes_s390.o test_kprobes_s390-objs += test_kprobes_asm.o test_kprobes.o @@ -22,3 +21,5 @@ obj-$(CONFIG_S390_MODULES_SANITY_TEST) += test_modules.o obj-$(CONFIG_S390_MODULES_SANITY_TEST_HELPERS) += test_modules_helpers.o lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o + +obj-$(CONFIG_EXPOLINE_EXTERN) += expoline/ diff --git a/arch/s390/lib/expoline/Makefile b/arch/s390/lib/expoline/Makefile new file mode 100644 index 000000000000..854631d9cb03 --- /dev/null +++ b/arch/s390/lib/expoline/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-y += expoline.o diff --git a/arch/s390/lib/expoline.S b/arch/s390/lib/expoline/expoline.S index 92ed8409a7a4..92ed8409a7a4 100644 --- a/arch/s390/lib/expoline.S +++ b/arch/s390/lib/expoline/expoline.S diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index ba449c47effd..4f7d1dfbc608 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -67,6 +67,8 @@ config SPARC64 select HAVE_KRETPROBES select HAVE_KPROBES select MMU_GATHER_RCU_TABLE_FREE if SMP + select MMU_GATHER_MERGE_VMAS + select MMU_GATHER_NO_FLUSH_CACHE select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h index 779a5a0f0608..3037187482db 100644 --- a/arch/sparc/include/asm/tlb_64.h +++ b/arch/sparc/include/asm/tlb_64.h @@ -22,8 +22,6 @@ void smp_flush_tlb_mm(struct mm_struct *mm); void __flush_tlb_pending(unsigned long, unsigned long, unsigned long *); void flush_tlb_pending(void); -#define tlb_start_vma(tlb, vma) do { } while (0) -#define tlb_end_vma(tlb, vma) do { } while (0) #define tlb_flush(tlb) flush_tlb_pending() /* diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c index 0760e24f2eba..9838967d0b2f 100644 --- a/arch/um/kernel/um_arch.c +++ b/arch/um/kernel/um_arch.c @@ -432,6 +432,10 @@ void apply_retpolines(s32 *start, s32 *end) { } +void apply_returns(s32 *start, s32 *end) +{ +} + void apply_alternatives(struct alt_instr *start, struct alt_instr *end) { } diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index e58798f636d4..7fff10e15969 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -245,6 +245,7 @@ config X86 select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select MMU_GATHER_RCU_TABLE_FREE if PARAVIRT + select MMU_GATHER_MERGE_VMAS select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if UNWINDER_ORC || STACK_VALIDATION diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 285e043a3e40..9953d966d124 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -727,7 +727,6 @@ native_irq_return_ldt: pushq %rdi /* Stash user RDI */ swapgs /* to kernel GS */ SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */ - UNTRAIN_RET movq PER_CPU_VAR(espfix_waddr), %rdi movq %rax, (0*8)(%rdi) /* user RAX */ diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h index 1bfe979bb9bc..580636cdc257 100644 --- a/arch/x86/include/asm/tlb.h +++ b/arch/x86/include/asm/tlb.h @@ -2,9 +2,6 @@ #ifndef _ASM_X86_TLB_H #define _ASM_X86_TLB_H -#define tlb_start_vma(tlb, vma) do { } while (0) -#define tlb_end_vma(tlb, vma) do { } while (0) - #define tlb_flush tlb_flush static inline void tlb_flush(struct mmu_gather *tlb); diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index 734b96454896..8d8752b44f11 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -16,6 +16,12 @@ bool cpc_supported_by_cpu(void) switch (boot_cpu_data.x86_vendor) { case X86_VENDOR_AMD: case X86_VENDOR_HYGON: + if (boot_cpu_data.x86 == 0x19 && ((boot_cpu_data.x86_model <= 0x0f) || + (boot_cpu_data.x86_model >= 0x20 && boot_cpu_data.x86_model <= 0x2f))) + return true; + else if (boot_cpu_data.x86 == 0x17 && + boot_cpu_data.x86_model >= 0x70 && boot_cpu_data.x86_model <= 0x7f) + return true; return boot_cpu_has(X86_FEATURE_CPPC); } return false; diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 0dd04713434b..aa34f908c39f 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -793,7 +793,7 @@ enum retbleed_mitigation_cmd { RETBLEED_CMD_IBPB, }; -const char * const retbleed_strings[] = { +static const char * const retbleed_strings[] = { [RETBLEED_MITIGATION_NONE] = "Vulnerable", [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", @@ -1181,7 +1181,7 @@ spectre_v2_user_select_mitigation(void) if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET) { if (mode != SPECTRE_V2_USER_STRICT && mode != SPECTRE_V2_USER_STRICT_PREFERRED) - pr_info("Selecting STIBP always-on mode to complement retbleed mitigation'\n"); + pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); mode = SPECTRE_V2_USER_STRICT_PREFERRED; } diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index eb8656bac99b..9b7acc9c7874 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -23,6 +23,7 @@ #include <asm/cpufeatures.h> #include <asm/percpu.h> #include <asm/nops.h> +#include <asm/nospec-branch.h> #include <asm/bootparam.h> #include <asm/export.h> #include <asm/pgtable_32.h> diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index db96bf7d1122..f8382abe22ff 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -189,9 +189,6 @@ #define X8(x...) X4(x), X4(x) #define X16(x...) X8(x), X8(x) -#define NR_FASTOP (ilog2(sizeof(ulong)) + 1) -#define FASTOP_SIZE (8 * (1 + HAS_KERNEL_IBT)) - struct opcode { u64 flags; u8 intercept; @@ -306,9 +303,15 @@ static void invalidate_registers(struct x86_emulate_ctxt *ctxt) * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for * different operand sizes can be reached by calculation, rather than a jump * table (which would be bigger than the code). + * + * The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR + * and 1 for the straight line speculation INT3, leaves 7 bytes for the + * body of the function. Currently none is larger than 4. */ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); +#define FASTOP_SIZE 16 + #define __FOP_FUNC(name) \ ".align " __stringify(FASTOP_SIZE) " \n\t" \ ".type " name ", @function \n\t" \ @@ -442,11 +445,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); * RET | JMP __x86_return_thunk [1,5 bytes; CONFIG_RETHUNK] * INT3 [1 byte; CONFIG_SLS] */ -#define RET_LENGTH (1 + (4 * IS_ENABLED(CONFIG_RETHUNK)) + \ - IS_ENABLED(CONFIG_SLS)) -#define SETCC_LENGTH (ENDBR_INSN_SIZE + 3 + RET_LENGTH) -#define SETCC_ALIGN (4 << ((SETCC_LENGTH > 4) & 1) << ((SETCC_LENGTH > 8) & 1)) -static_assert(SETCC_LENGTH <= SETCC_ALIGN); +#define SETCC_ALIGN 16 #define FOP_SETCC(op) \ ".align " __stringify(SETCC_ALIGN) " \n\t" \ diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 3a4e895269d7..ab135f9ef52f 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2278,7 +2278,6 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs0 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | SECONDARY_EXEC_APIC_REGISTER_VIRT | SECONDARY_EXEC_ENABLE_VMFUNC | - SECONDARY_EXEC_TSC_SCALING | SECONDARY_EXEC_DESC); if (nested_cpu_has(vmcs12, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 26d0cac32f73..143e37298d8a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -298,7 +298,7 @@ const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { STATS_DESC_COUNTER(VCPU, directed_yield_successful), STATS_DESC_COUNTER(VCPU, preemption_reported), STATS_DESC_COUNTER(VCPU, preemption_other), - STATS_DESC_ICOUNTER(VCPU, guest_mode) + STATS_DESC_IBOOLEAN(VCPU, guest_mode) }; const struct kvm_stats_header kvm_vcpu_stats_header = { @@ -9143,15 +9143,17 @@ static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr, */ static void kvm_pv_kick_cpu_op(struct kvm *kvm, int apicid) { - struct kvm_lapic_irq lapic_irq; - - lapic_irq.shorthand = APIC_DEST_NOSHORT; - lapic_irq.dest_mode = APIC_DEST_PHYSICAL; - lapic_irq.level = 0; - lapic_irq.dest_id = apicid; - lapic_irq.msi_redir_hint = false; + /* + * All other fields are unused for APIC_DM_REMRD, but may be consumed by + * common code, e.g. for tracing. Defer initialization to the compiler. + */ + struct kvm_lapic_irq lapic_irq = { + .delivery_mode = APIC_DM_REMRD, + .dest_mode = APIC_DEST_PHYSICAL, + .shorthand = APIC_DEST_NOSHORT, + .dest_id = apicid, + }; - lapic_irq.delivery_mode = APIC_DM_REMRD; kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL); } diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index d8cfce221275..57ba5502aecf 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -77,10 +77,20 @@ static uint8_t __pte2cachemode_tbl[8] = { [__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC, }; -/* Check that the write-protect PAT entry is set for write-protect */ +/* + * Check that the write-protect PAT entry is set for write-protect. + * To do this without making assumptions how PAT has been set up (Xen has + * another layout than the kernel), translate the _PAGE_CACHE_MODE_WP cache + * mode via the __cachemode2pte_tbl[] into protection bits (those protection + * bits will select a cache mode of WP or better), and then translate the + * protection bits back into the cache mode using __pte2cm_idx() and the + * __pte2cachemode_tbl[] array. This will return the really used cache mode. + */ bool x86_has_pat_wp(void) { - return __pte2cachemode_tbl[_PAGE_CACHE_MODE_WP] == _PAGE_CACHE_MODE_WP; + uint16_t prot = __cachemode2pte_tbl[_PAGE_CACHE_MODE_WP]; + + return __pte2cachemode_tbl[__pte2cm_idx(prot)] == _PAGE_CACHE_MODE_WP; } enum page_cache_mode pgprot2cachemode(pgprot_t pgprot) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 7e95697a6459..c1f6c1c51d99 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1950,23 +1950,6 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, return 0; } -static bool is_valid_bpf_tramp_flags(unsigned int flags) -{ - if ((flags & BPF_TRAMP_F_RESTORE_REGS) && - (flags & BPF_TRAMP_F_SKIP_FRAME)) - return false; - - /* - * BPF_TRAMP_F_RET_FENTRY_RET is only used by bpf_struct_ops, - * and it must be used alone. - */ - if ((flags & BPF_TRAMP_F_RET_FENTRY_RET) && - (flags & ~BPF_TRAMP_F_RET_FENTRY_RET)) - return false; - - return true; -} - /* Example: * __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev); * its 'struct btf_func_model' will be nr_args=2 @@ -2045,9 +2028,6 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (nr_args > 6) return -ENOTSUPP; - if (!is_valid_bpf_tramp_flags(flags)) - return -EINVAL; - /* Generated trampoline stack layout: * * RBP + 8 [ return address ] @@ -2153,10 +2133,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (flags & BPF_TRAMP_F_CALL_ORIG) { restore_regs(m, &prog, nr_args, regs_off); - /* call original function */ - if (emit_call(&prog, orig_call, prog)) { - ret = -EINVAL; - goto cleanup; + if (flags & BPF_TRAMP_F_ORIG_STACK) { + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8); + EMIT2(0xff, 0xd0); /* call *rax */ + } else { + /* call original function */ + if (emit_call(&prog, orig_call, prog)) { + ret = -EINVAL; + goto cleanup; + } } /* remember return value in a stack for bpf prog to access */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); @@ -2520,3 +2505,28 @@ bool bpf_jit_supports_subprog_tailcalls(void) { return true; } + +void bpf_jit_free(struct bpf_prog *prog) +{ + if (prog->jited) { + struct x64_jit_data *jit_data = prog->aux->jit_data; + struct bpf_binary_header *hdr; + + /* + * If we fail the final pass of JIT (from jit_subprogs), + * the program may not be finalized yet. Call finalize here + * before freeing it. + */ + if (jit_data) { + bpf_jit_binary_pack_finalize(prog, jit_data->header, + jit_data->rw_header); + kvfree(jit_data->addrs); + kfree(jit_data); + } + hdr = bpf_jit_binary_pack_hdr(prog); + bpf_jit_binary_pack_free(hdr, NULL); + WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); + } + + bpf_prog_unlock_free(prog); +} diff --git a/arch/x86/platform/efi/efi_thunk_64.S b/arch/x86/platform/efi/efi_thunk_64.S index 9ffe2bad27d5..4e5257a4811b 100644 --- a/arch/x86/platform/efi/efi_thunk_64.S +++ b/arch/x86/platform/efi/efi_thunk_64.S @@ -23,6 +23,7 @@ #include <linux/objtool.h> #include <asm/page_types.h> #include <asm/segment.h> +#include <asm/nospec-branch.h> .text .code64 @@ -75,7 +76,9 @@ STACK_FRAME_NON_STANDARD __efi64_thunk 1: movq 0x20(%rsp), %rsp pop %rbx pop %rbp - RET + ANNOTATE_UNRET_SAFE + ret + int3 .code32 2: pushl $__KERNEL_CS diff --git a/block/blk-merge.c b/block/blk-merge.c index 7771dacc99cb..f5e6527ebc9c 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -345,6 +345,7 @@ void __blk_queue_split(struct request_queue *q, struct bio **bio, /* there isn't chance to merge the splitted bio */ split->bi_opf |= REQ_NOMERGE; + blkcg_bio_issue_init(split); bio_chain(split, *bio); trace_block_split(split, (*bio)->bi_iter.bi_sector); submit_bio_noacct(*bio); diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index 43177c20ce4f..eaea733b368a 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -73,7 +73,7 @@ module_param(device_id_scheme, bool, 0444); static int only_lcd = -1; module_param(only_lcd, int, 0444); -static bool has_backlight; +static bool may_report_brightness_keys; static int register_count; static DEFINE_MUTEX(register_count_mutex); static DEFINE_MUTEX(video_list_lock); @@ -1224,7 +1224,7 @@ acpi_video_bus_get_one_device(struct acpi_device *device, acpi_video_device_find_cap(data); if (data->cap._BCM && data->cap._BCL) - has_backlight = true; + may_report_brightness_keys = true; mutex_lock(&video->device_list_lock); list_add_tail(&data->entry, &video->video_device_list); @@ -1693,6 +1693,9 @@ static void acpi_video_device_notify(acpi_handle handle, u32 event, void *data) break; } + if (keycode) + may_report_brightness_keys = true; + acpi_notifier_call_chain(device, event, 0); if (keycode && (report_key_events & REPORT_BRIGHTNESS_KEY_EVENTS)) { @@ -2253,7 +2256,7 @@ void acpi_video_unregister(void) if (register_count) { acpi_bus_unregister_driver(&acpi_video_bus); register_count = 0; - has_backlight = false; + may_report_brightness_keys = false; } mutex_unlock(®ister_count_mutex); } @@ -2275,7 +2278,7 @@ void acpi_video_unregister_backlight(void) bool acpi_video_handles_brightness_key_presses(void) { - return has_backlight && + return may_report_brightness_keys && (report_key_events & REPORT_BRIGHTNESS_KEY_EVENTS); } EXPORT_SYMBOL(acpi_video_handles_brightness_key_presses); diff --git a/drivers/bluetooth/btbcm.c b/drivers/bluetooth/btbcm.c index 76fbb046bdbe..3006e2a0f37e 100644 --- a/drivers/bluetooth/btbcm.c +++ b/drivers/bluetooth/btbcm.c @@ -403,6 +403,13 @@ static int btbcm_read_info(struct hci_dev *hdev) bt_dev_info(hdev, "BCM: chip id %u", skb->data[1]); kfree_skb(skb); + return 0; +} + +static int btbcm_print_controller_features(struct hci_dev *hdev) +{ + struct sk_buff *skb; + /* Read Controller Features */ skb = btbcm_read_controller_features(hdev); if (IS_ERR(skb)) @@ -454,6 +461,8 @@ static const struct bcm_subver_table bcm_uart_subver_table[] = { { 0x6606, "BCM4345C5" }, /* 003.006.006 */ { 0x230f, "BCM4356A2" }, /* 001.003.015 */ { 0x220e, "BCM20702A1" }, /* 001.002.014 */ + { 0x420d, "BCM4349B1" }, /* 002.002.013 */ + { 0x420e, "BCM4349B1" }, /* 002.002.014 */ { 0x4217, "BCM4329B1" }, /* 002.002.023 */ { 0x6106, "BCM4359C0" }, /* 003.001.006 */ { 0x4106, "BCM4335A0" }, /* 002.001.006 */ @@ -514,7 +523,7 @@ static const char *btbcm_get_board_name(struct device *dev) #endif } -int btbcm_initialize(struct hci_dev *hdev, bool *fw_load_done) +int btbcm_initialize(struct hci_dev *hdev, bool *fw_load_done, bool use_autobaud_mode) { u16 subver, rev, pid, vid; struct sk_buff *skb; @@ -551,9 +560,16 @@ int btbcm_initialize(struct hci_dev *hdev, bool *fw_load_done) if (err) return err; } - err = btbcm_print_local_name(hdev); - if (err) - return err; + + if (!use_autobaud_mode) { + err = btbcm_print_controller_features(hdev); + if (err) + return err; + + err = btbcm_print_local_name(hdev); + if (err) + return err; + } bcm_subver_table = (hdev->bus == HCI_USB) ? bcm_usb_subver_table : bcm_uart_subver_table; @@ -636,13 +652,13 @@ int btbcm_initialize(struct hci_dev *hdev, bool *fw_load_done) } EXPORT_SYMBOL_GPL(btbcm_initialize); -int btbcm_finalize(struct hci_dev *hdev, bool *fw_load_done) +int btbcm_finalize(struct hci_dev *hdev, bool *fw_load_done, bool use_autobaud_mode) { int err; /* Re-initialize if necessary */ if (*fw_load_done) { - err = btbcm_initialize(hdev, fw_load_done); + err = btbcm_initialize(hdev, fw_load_done, use_autobaud_mode); if (err) return err; } @@ -658,15 +674,16 @@ EXPORT_SYMBOL_GPL(btbcm_finalize); int btbcm_setup_patchram(struct hci_dev *hdev) { bool fw_load_done = false; + bool use_autobaud_mode = false; int err; /* Initialize */ - err = btbcm_initialize(hdev, &fw_load_done); + err = btbcm_initialize(hdev, &fw_load_done, use_autobaud_mode); if (err) return err; /* Re-initialize after loading Patch */ - return btbcm_finalize(hdev, &fw_load_done); + return btbcm_finalize(hdev, &fw_load_done, use_autobaud_mode); } EXPORT_SYMBOL_GPL(btbcm_setup_patchram); diff --git a/drivers/bluetooth/btbcm.h b/drivers/bluetooth/btbcm.h index 8bf01565fdfc..b4cb24231a20 100644 --- a/drivers/bluetooth/btbcm.h +++ b/drivers/bluetooth/btbcm.h @@ -62,8 +62,8 @@ int btbcm_write_pcm_int_params(struct hci_dev *hdev, int btbcm_setup_patchram(struct hci_dev *hdev); int btbcm_setup_apple(struct hci_dev *hdev); -int btbcm_initialize(struct hci_dev *hdev, bool *fw_load_done); -int btbcm_finalize(struct hci_dev *hdev, bool *fw_load_done); +int btbcm_initialize(struct hci_dev *hdev, bool *fw_load_done, bool use_autobaud_mode); +int btbcm_finalize(struct hci_dev *hdev, bool *fw_load_done, bool use_autobaud_mode); #else @@ -104,12 +104,12 @@ static inline int btbcm_setup_apple(struct hci_dev *hdev) return 0; } -static inline int btbcm_initialize(struct hci_dev *hdev, bool *fw_load_done) +static inline int btbcm_initialize(struct hci_dev *hdev, bool *fw_load_done, bool use_autobaud_mode) { return 0; } -static inline int btbcm_finalize(struct hci_dev *hdev, bool *fw_load_done) +static inline int btbcm_finalize(struct hci_dev *hdev, bool *fw_load_done, bool use_autobaud_mode) { return 0; } diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c index d6700efcfe8c..f9a3444753c2 100644 --- a/drivers/bluetooth/btmtksdio.c +++ b/drivers/bluetooth/btmtksdio.c @@ -1282,6 +1282,13 @@ err: hci_reset_dev(hdev); } +static bool btmtksdio_sdio_inband_wakeup(struct hci_dev *hdev) +{ + struct btmtksdio_dev *bdev = hci_get_drvdata(hdev); + + return device_may_wakeup(bdev->dev); +} + static bool btmtksdio_sdio_wakeup(struct hci_dev *hdev) { struct btmtksdio_dev *bdev = hci_get_drvdata(hdev); @@ -1349,6 +1356,14 @@ static int btmtksdio_probe(struct sdio_func *func, hdev->shutdown = btmtksdio_shutdown; hdev->send = btmtksdio_send_frame; hdev->wakeup = btmtksdio_sdio_wakeup; + /* + * If SDIO controller supports wake on Bluetooth, sending a wakeon + * command is not necessary. + */ + if (device_can_wakeup(func->card->host->parent)) + hdev->wakeup = btmtksdio_sdio_inband_wakeup; + else + hdev->wakeup = btmtksdio_sdio_wakeup; hdev->set_bdaddr = btmtk_set_bdaddr; SET_HCIDEV_DEV(hdev, &func->dev); diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c index 47c28fd8f006..fb52313a1d45 100644 --- a/drivers/bluetooth/btrtl.c +++ b/drivers/bluetooth/btrtl.c @@ -330,7 +330,7 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev, /* Loop from the end of the firmware parsing instructions, until * we find an instruction that identifies the "project ID" for the * hardware supported by this firwmare file. - * Once we have that, we double-check that that project_id is suitable + * Once we have that, we double-check that project_id is suitable * for the hardware we are working with. */ while (fwptr >= btrtl_dev->fw_data + (sizeof(*epatch_info) + 3)) { diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index e25fcd49db70..15caa6469538 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -427,6 +427,18 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x04ca, 0x4006), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + /* Realtek 8852CE Bluetooth devices */ + { USB_DEVICE(0x04ca, 0x4007), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x04c5, 0x1675), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0cb8, 0xc558), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3587), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3586), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + /* Realtek Bluetooth devices */ { USB_VENDOR_AND_INTERFACE_INFO(0x0bda, 0xe0, 0x01, 0x01), .driver_info = BTUSB_REALTEK }, @@ -477,6 +489,12 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x0489, 0xe0d9), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH | BTUSB_VALID_LE_STATES }, + { USB_DEVICE(0x13d3, 0x3568), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH | + BTUSB_VALID_LE_STATES }, + { USB_DEVICE(0x0489, 0xe0e2), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH | + BTUSB_VALID_LE_STATES }, /* Additional Realtek 8723AE Bluetooth devices */ { USB_DEVICE(0x0930, 0x021d), .driver_info = BTUSB_REALTEK }, @@ -893,11 +911,21 @@ static int btusb_recv_bulk(struct btusb_data *data, void *buffer, int count) hci_skb_expect(skb) -= len; if (skb->len == HCI_ACL_HDR_SIZE) { + __u16 handle = __le16_to_cpu(hci_acl_hdr(skb)->handle); __le16 dlen = hci_acl_hdr(skb)->dlen; + __u8 type; /* Complete ACL header */ hci_skb_expect(skb) = __le16_to_cpu(dlen); + /* Detect if ISO packet has been sent over bulk */ + if (hci_conn_num(data->hdev, ISO_LINK)) { + type = hci_conn_lookup_type(data->hdev, + hci_handle(handle)); + if (type == ISO_LINK) + hci_skb_pkt_type(skb) = HCI_ISODATA_PKT; + } + if (skb_tailroom(skb) < hci_skb_expect(skb)) { kfree_skb(skb); skb = NULL; @@ -1762,6 +1790,13 @@ static int btusb_send_frame(struct hci_dev *hdev, struct sk_buff *skb) hdev->stat.sco_tx++; return submit_tx_urb(hdev, urb); + + case HCI_ISODATA_PKT: + urb = alloc_bulk_urb(hdev, skb); + if (IS_ERR(urb)) + return PTR_ERR(urb); + + return submit_or_queue_tx_urb(hdev, urb); } return -EILSEQ; @@ -2069,7 +2104,6 @@ static int btusb_setup_csr(struct hci_dev *hdev) * without these the controller will lock up. */ set_bit(HCI_QUIRK_BROKEN_STORED_LINK_KEY, &hdev->quirks); - set_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks); set_bit(HCI_QUIRK_BROKEN_FILTER_CLEAR_ALL, &hdev->quirks); set_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks); @@ -2255,6 +2289,13 @@ static int btusb_send_frame_intel(struct hci_dev *hdev, struct sk_buff *skb) hdev->stat.sco_tx++; return submit_tx_urb(hdev, urb); + + case HCI_ISODATA_PKT: + urb = alloc_bulk_urb(hdev, skb); + if (IS_ERR(urb)) + return PTR_ERR(urb); + + return submit_or_queue_tx_urb(hdev, urb); } return -EILSEQ; @@ -3352,7 +3393,6 @@ static int btusb_setup_qca(struct hci_dev *hdev) * work with the likes of HSP/HFP mSBC. */ set_bit(HCI_QUIRK_BROKEN_ENHANCED_SETUP_SYNC_CONN, &hdev->quirks); - set_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks); return 0; } @@ -3795,6 +3835,7 @@ static int btusb_probe(struct usb_interface *intf, hdev->manufacturer = 70; hdev->cmd_timeout = btusb_mtk_cmd_timeout; hdev->set_bdaddr = btmtk_set_bdaddr; + set_bit(HCI_QUIRK_BROKEN_ENHANCED_SETUP_SYNC_CONN, &hdev->quirks); set_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks); data->recv_acl = btusb_recv_acl_mtk; } diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c index 785f445dd60d..d7e0b75db8a6 100644 --- a/drivers/bluetooth/hci_bcm.c +++ b/drivers/bluetooth/hci_bcm.c @@ -53,10 +53,12 @@ * struct bcm_device_data - device specific data * @no_early_set_baudrate: Disallow set baudrate before driver setup() * @drive_rts_on_open: drive RTS signal on ->open() when platform requires it + * @max_autobaud_speed: max baudrate supported by device in autobaud mode */ struct bcm_device_data { bool no_early_set_baudrate; bool drive_rts_on_open; + u32 max_autobaud_speed; }; /** @@ -99,6 +101,8 @@ struct bcm_device_data { * @no_early_set_baudrate: don't set_baudrate before setup() * @drive_rts_on_open: drive RTS signal on ->open() when platform requires it * @pcm_int_params: keep the initial PCM configuration + * @use_autobaud_mode: start Bluetooth device in autobaud mode + * @max_autobaud_speed: max baudrate supported by device in autobaud mode */ struct bcm_device { /* Must be the first member, hci_serdev.c expects this. */ @@ -136,7 +140,9 @@ struct bcm_device { #endif bool no_early_set_baudrate; bool drive_rts_on_open; + bool use_autobaud_mode; u8 pcm_int_params[5]; + u32 max_autobaud_speed; }; /* generic bcm uart resources */ @@ -472,15 +478,20 @@ static int bcm_open(struct hci_uart *hu) out: if (bcm->dev) { - if (bcm->dev->drive_rts_on_open) + if (bcm->dev->use_autobaud_mode) + hci_uart_set_flow_control(hu, false); /* Assert BT_UART_CTS_N */ + else if (bcm->dev->drive_rts_on_open) hci_uart_set_flow_control(hu, true); - hu->init_speed = bcm->dev->init_speed; + if (bcm->dev->use_autobaud_mode && bcm->dev->max_autobaud_speed) + hu->init_speed = min(bcm->dev->oper_speed, bcm->dev->max_autobaud_speed); + else + hu->init_speed = bcm->dev->init_speed; /* If oper_speed is set, ldisc/serdev will set the baudrate * before calling setup() */ - if (!bcm->dev->no_early_set_baudrate) + if (!bcm->dev->no_early_set_baudrate && !bcm->dev->use_autobaud_mode) hu->oper_speed = bcm->dev->oper_speed; err = bcm_gpio_set_power(bcm->dev, true); @@ -564,6 +575,7 @@ static int bcm_setup(struct hci_uart *hu) { struct bcm_data *bcm = hu->priv; bool fw_load_done = false; + bool use_autobaud_mode = (bcm->dev ? bcm->dev->use_autobaud_mode : 0); unsigned int speed; int err; @@ -572,7 +584,7 @@ static int bcm_setup(struct hci_uart *hu) hu->hdev->set_diag = bcm_set_diag; hu->hdev->set_bdaddr = btbcm_set_bdaddr; - err = btbcm_initialize(hu->hdev, &fw_load_done); + err = btbcm_initialize(hu->hdev, &fw_load_done, use_autobaud_mode); if (err) return err; @@ -580,8 +592,8 @@ static int bcm_setup(struct hci_uart *hu) return 0; /* Init speed if any */ - if (hu->init_speed) - speed = hu->init_speed; + if (bcm->dev && bcm->dev->init_speed) + speed = bcm->dev->init_speed; else if (hu->proto->init_speed) speed = hu->proto->init_speed; else @@ -616,7 +628,7 @@ static int bcm_setup(struct hci_uart *hu) btbcm_write_pcm_int_params(hu->hdev, ¶ms); } - err = btbcm_finalize(hu->hdev, &fw_load_done); + err = btbcm_finalize(hu->hdev, &fw_load_done, use_autobaud_mode); if (err) return err; @@ -1197,6 +1209,8 @@ static int bcm_acpi_probe(struct bcm_device *dev) static int bcm_of_probe(struct bcm_device *bdev) { + bdev->use_autobaud_mode = device_property_read_bool(bdev->dev, + "brcm,requires-autobaud-mode"); device_property_read_u32(bdev->dev, "max-speed", &bdev->oper_speed); device_property_read_u8_array(bdev->dev, "brcm,bt-pcm-int-params", bdev->pcm_int_params, 5); @@ -1512,6 +1526,7 @@ static int bcm_serdev_probe(struct serdev_device *serdev) data = device_get_match_data(bcmdev->dev); if (data) { + bcmdev->max_autobaud_speed = data->max_autobaud_speed; bcmdev->no_early_set_baudrate = data->no_early_set_baudrate; bcmdev->drive_rts_on_open = data->drive_rts_on_open; } @@ -1535,6 +1550,10 @@ static struct bcm_device_data bcm43438_device_data = { .drive_rts_on_open = true, }; +static struct bcm_device_data cyw55572_device_data = { + .max_autobaud_speed = 921600, +}; + static const struct of_device_id bcm_bluetooth_of_match[] = { { .compatible = "brcm,bcm20702a1" }, { .compatible = "brcm,bcm4329-bt" }, @@ -1544,8 +1563,10 @@ static const struct of_device_id bcm_bluetooth_of_match[] = { { .compatible = "brcm,bcm43430a0-bt" }, { .compatible = "brcm,bcm43430a1-bt" }, { .compatible = "brcm,bcm43438-bt", .data = &bcm43438_device_data }, + { .compatible = "brcm,bcm4349-bt", .data = &bcm43438_device_data }, { .compatible = "brcm,bcm43540-bt", .data = &bcm4354_device_data }, { .compatible = "brcm,bcm4335a0" }, + { .compatible = "infineon,cyw55572-bt", .data = &cyw55572_device_data }, { }, }; MODULE_DEVICE_TABLE(of, bcm_bluetooth_of_match); diff --git a/drivers/bluetooth/hci_intel.c b/drivers/bluetooth/hci_intel.c index 7249b91d9b91..78afb9a348e7 100644 --- a/drivers/bluetooth/hci_intel.c +++ b/drivers/bluetooth/hci_intel.c @@ -1217,7 +1217,11 @@ static struct platform_driver intel_driver = { int __init intel_init(void) { - platform_driver_register(&intel_driver); + int err; + + err = platform_driver_register(&intel_driver); + if (err) + return err; return hci_uart_register_proto(&intel_proto); } diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index eab34e24d944..8df11016fd51 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -1588,7 +1588,7 @@ static bool qca_wakeup(struct hci_dev *hdev) wakeup = device_may_wakeup(hu->serdev->ctrl->dev.parent); bt_dev_dbg(hu->hdev, "wakeup status : %d", wakeup); - return !wakeup; + return wakeup; } static int qca_regulator_init(struct hci_uart *hu) diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c index 4cda890ce647..c0e5f42ec6b7 100644 --- a/drivers/bluetooth/hci_serdev.c +++ b/drivers/bluetooth/hci_serdev.c @@ -231,6 +231,15 @@ static int hci_uart_setup(struct hci_dev *hdev) return 0; } +/* Check if the device is wakeable */ +static bool hci_uart_wakeup(struct hci_dev *hdev) +{ + /* HCI UART devices are assumed to be wakeable by default. + * Implement wakeup callback to override this behavior. + */ + return true; +} + /** hci_uart_write_wakeup - transmit buffer wakeup * @serdev: serial device * @@ -342,6 +351,8 @@ int hci_uart_register_device(struct hci_uart *hu, hdev->flush = hci_uart_flush; hdev->send = hci_uart_send_frame; hdev->setup = hci_uart_setup; + if (!hdev->wakeup) + hdev->wakeup = hci_uart_wakeup; SET_HCIDEV_DEV(hdev, &hu->serdev->dev); if (test_bit(HCI_UART_NO_SUSPEND_NOTIFIER, &hu->flags)) diff --git a/drivers/char/random.c b/drivers/char/random.c index e3dd1dd3dd22..a1af90bacc9f 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1174,7 +1174,7 @@ static void __cold entropy_timer(struct timer_list *timer) */ static void __cold try_to_generate_entropy(void) { - enum { NUM_TRIAL_SAMPLES = 8192, MAX_SAMPLES_PER_BIT = 32 }; + enum { NUM_TRIAL_SAMPLES = 8192, MAX_SAMPLES_PER_BIT = HZ / 30 }; struct entropy_timer_state stack; unsigned int i, num_different = 0; unsigned long last = random_get_entropy(); diff --git a/drivers/cpufreq/mediatek-cpufreq.c b/drivers/cpufreq/mediatek-cpufreq.c index 37a1eb20f5ba..76f6b3884e6b 100644 --- a/drivers/cpufreq/mediatek-cpufreq.c +++ b/drivers/cpufreq/mediatek-cpufreq.c @@ -439,9 +439,13 @@ static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu) /* Both presence and absence of sram regulator are valid cases. */ info->sram_reg = regulator_get_optional(cpu_dev, "sram"); - if (IS_ERR(info->sram_reg)) + if (IS_ERR(info->sram_reg)) { + ret = PTR_ERR(info->sram_reg); + if (ret == -EPROBE_DEFER) + goto out_free_resources; + info->sram_reg = NULL; - else { + } else { ret = regulator_enable(info->sram_reg); if (ret) { dev_warn(cpu_dev, "cpu%d: failed to enable vsram\n", cpu); diff --git a/drivers/firmware/efi/reboot.c b/drivers/firmware/efi/reboot.c index 73089a24f04b..ceae84c19d22 100644 --- a/drivers/firmware/efi/reboot.c +++ b/drivers/firmware/efi/reboot.c @@ -6,7 +6,7 @@ #include <linux/efi.h> #include <linux/reboot.h> -static void (*orig_pm_power_off)(void); +static struct sys_off_handler *efi_sys_off_handler; int efi_reboot_quirk_mode = -1; @@ -51,15 +51,11 @@ bool __weak efi_poweroff_required(void) return false; } -static void efi_power_off(void) +static int efi_power_off(struct sys_off_data *data) { efi.reset_system(EFI_RESET_SHUTDOWN, EFI_SUCCESS, 0, NULL); - /* - * The above call should not return, if it does fall back to - * the original power off method (typically ACPI poweroff). - */ - if (orig_pm_power_off) - orig_pm_power_off(); + + return NOTIFY_DONE; } static int __init efi_shutdown_init(void) @@ -68,8 +64,13 @@ static int __init efi_shutdown_init(void) return -ENODEV; if (efi_poweroff_required()) { - orig_pm_power_off = pm_power_off; - pm_power_off = efi_power_off; + /* SYS_OFF_PRIO_FIRMWARE + 1 so that it runs before acpi_power_off */ + efi_sys_off_handler = + register_sys_off_handler(SYS_OFF_MODE_POWER_OFF, + SYS_OFF_PRIO_FIRMWARE + 1, + efi_power_off, NULL); + if (IS_ERR(efi_sys_off_handler)) + return PTR_ERR(efi_sys_off_handler); } return 0; diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c index 98109839102f..1020c2feb249 100644 --- a/drivers/gpio/gpio-sim.c +++ b/drivers/gpio/gpio-sim.c @@ -991,28 +991,22 @@ static struct configfs_attribute *gpio_sim_device_config_attrs[] = { }; struct gpio_sim_chip_name_ctx { - struct gpio_sim_device *dev; + struct fwnode_handle *swnode; char *page; }; static int gpio_sim_emit_chip_name(struct device *dev, void *data) { struct gpio_sim_chip_name_ctx *ctx = data; - struct fwnode_handle *swnode; - struct gpio_sim_bank *bank; /* This would be the sysfs device exported in /sys/class/gpio. */ if (dev->class) return 0; - swnode = dev_fwnode(dev); + if (device_match_fwnode(dev, ctx->swnode)) + return sprintf(ctx->page, "%s\n", dev_name(dev)); - list_for_each_entry(bank, &ctx->dev->bank_list, siblings) { - if (bank->swnode == swnode) - return sprintf(ctx->page, "%s\n", dev_name(dev)); - } - - return -ENODATA; + return 0; } static ssize_t gpio_sim_bank_config_chip_name_show(struct config_item *item, @@ -1020,7 +1014,7 @@ static ssize_t gpio_sim_bank_config_chip_name_show(struct config_item *item, { struct gpio_sim_bank *bank = to_gpio_sim_bank(item); struct gpio_sim_device *dev = gpio_sim_bank_get_device(bank); - struct gpio_sim_chip_name_ctx ctx = { dev, page }; + struct gpio_sim_chip_name_ctx ctx = { bank->swnode, page }; int ret; mutex_lock(&dev->lock); diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 0c9a63becfef..b26e64338376 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -421,6 +421,10 @@ out_free_lh: * @work: the worker that implements software debouncing * @sw_debounced: flag indicating if the software debouncer is active * @level: the current debounced physical level of the line + * @hdesc: the Hardware Timestamp Engine (HTE) descriptor + * @raw_level: the line level at the time of event + * @total_discard_seq: the running counter of the discarded events + * @last_seqno: the last sequence number before debounce period expires */ struct line { struct gpio_desc *desc; diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index e88c497fa010..f65656df3619 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -256,7 +256,6 @@ config DRM_AMDGPU select HWMON select BACKLIGHT_CLASS_DEVICE select INTERVAL_TREE - select DRM_BUDDY help Choose this option if you have a recent AMD Radeon graphics card. diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h index 6546552e596c..acfa207cf970 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h @@ -30,15 +30,12 @@ #include <drm/ttm/ttm_resource.h> #include <drm/ttm/ttm_range_manager.h> -#include "amdgpu_vram_mgr.h" - /* state back for walking over vram_mgr and gtt_mgr allocations */ struct amdgpu_res_cursor { uint64_t start; uint64_t size; uint64_t remaining; - void *node; - uint32_t mem_type; + struct drm_mm_node *node; }; /** @@ -55,63 +52,27 @@ static inline void amdgpu_res_first(struct ttm_resource *res, uint64_t start, uint64_t size, struct amdgpu_res_cursor *cur) { - struct drm_buddy_block *block; - struct list_head *head, *next; struct drm_mm_node *node; - if (!res) - goto fallback; - - BUG_ON(start + size > res->num_pages << PAGE_SHIFT); - - cur->mem_type = res->mem_type; - - switch (cur->mem_type) { - case TTM_PL_VRAM: - head = &to_amdgpu_vram_mgr_resource(res)->blocks; - - block = list_first_entry_or_null(head, - struct drm_buddy_block, - link); - if (!block) - goto fallback; - - while (start >= amdgpu_vram_mgr_block_size(block)) { - start -= amdgpu_vram_mgr_block_size(block); - - next = block->link.next; - if (next != head) - block = list_entry(next, struct drm_buddy_block, link); - } - - cur->start = amdgpu_vram_mgr_block_start(block) + start; - cur->size = min(amdgpu_vram_mgr_block_size(block) - start, size); - cur->remaining = size; - cur->node = block; - break; - case TTM_PL_TT: - node = to_ttm_range_mgr_node(res)->mm_nodes; - while (start >= node->size << PAGE_SHIFT) - start -= node++->size << PAGE_SHIFT; - - cur->start = (node->start << PAGE_SHIFT) + start; - cur->size = min((node->size << PAGE_SHIFT) - start, size); + if (!res || res->mem_type == TTM_PL_SYSTEM) { + cur->start = start; + cur->size = size; cur->remaining = size; - cur->node = node; - break; - default: - goto fallback; + cur->node = NULL; + WARN_ON(res && start + size > res->num_pages << PAGE_SHIFT); + return; } - return; + BUG_ON(start + size > res->num_pages << PAGE_SHIFT); -fallback: - cur->start = start; - cur->size = size; + node = to_ttm_range_mgr_node(res)->mm_nodes; + while (start >= node->size << PAGE_SHIFT) + start -= node++->size << PAGE_SHIFT; + + cur->start = (node->start << PAGE_SHIFT) + start; + cur->size = min((node->size << PAGE_SHIFT) - start, size); cur->remaining = size; - cur->node = NULL; - WARN_ON(res && start + size > res->num_pages << PAGE_SHIFT); - return; + cur->node = node; } /** @@ -124,9 +85,7 @@ fallback: */ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size) { - struct drm_buddy_block *block; - struct drm_mm_node *node; - struct list_head *next; + struct drm_mm_node *node = cur->node; BUG_ON(size > cur->remaining); @@ -140,27 +99,9 @@ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size) return; } - switch (cur->mem_type) { - case TTM_PL_VRAM: - block = cur->node; - - next = block->link.next; - block = list_entry(next, struct drm_buddy_block, link); - - cur->node = block; - cur->start = amdgpu_vram_mgr_block_start(block); - cur->size = min(amdgpu_vram_mgr_block_size(block), cur->remaining); - break; - case TTM_PL_TT: - node = cur->node; - - cur->node = ++node; - cur->start = node->start << PAGE_SHIFT; - cur->size = min(node->size << PAGE_SHIFT, cur->remaining); - break; - default: - return; - } + cur->node = ++node; + cur->start = node->start << PAGE_SHIFT; + cur->size = min(node->size << PAGE_SHIFT, cur->remaining); } #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h index 6a70818039dd..9120ae80ef52 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h @@ -26,7 +26,6 @@ #include <linux/dma-direction.h> #include <drm/gpu_scheduler.h> -#include "amdgpu_vram_mgr.h" #include "amdgpu.h" #define AMDGPU_PL_GDS (TTM_PL_PRIV + 0) @@ -39,6 +38,15 @@ #define AMDGPU_POISON 0xd0bed0be +struct amdgpu_vram_mgr { + struct ttm_resource_manager manager; + struct drm_mm mm; + spinlock_t lock; + struct list_head reservations_pending; + struct list_head reserved_pages; + atomic64_t vis_usage; +}; + struct amdgpu_gtt_mgr { struct ttm_resource_manager manager; struct drm_mm mm; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index 49e4092f447f..0a7611648573 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -32,10 +32,8 @@ #include "atom.h" struct amdgpu_vram_reservation { - u64 start; - u64 size; - struct list_head allocated; - struct list_head blocks; + struct list_head node; + struct drm_mm_node mm_node; }; static inline struct amdgpu_vram_mgr * @@ -188,18 +186,18 @@ const struct attribute_group amdgpu_vram_mgr_attr_group = { }; /** - * amdgpu_vram_mgr_vis_size - Calculate visible block size + * amdgpu_vram_mgr_vis_size - Calculate visible node size * * @adev: amdgpu_device pointer - * @block: DRM BUDDY block structure + * @node: MM node structure * - * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM + * Calculate how many bytes of the MM node are inside visible VRAM */ static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, - struct drm_buddy_block *block) + struct drm_mm_node *node) { - u64 start = amdgpu_vram_mgr_block_start(block); - u64 end = start + amdgpu_vram_mgr_block_size(block); + uint64_t start = node->start << PAGE_SHIFT; + uint64_t end = (node->size + node->start) << PAGE_SHIFT; if (start >= adev->gmc.visible_vram_size) return 0; @@ -220,9 +218,9 @@ u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); struct ttm_resource *res = bo->tbo.resource; - struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); - struct drm_buddy_block *block; - u64 usage = 0; + unsigned pages = res->num_pages; + struct drm_mm_node *mm; + u64 usage; if (amdgpu_gmc_vram_full_visible(&adev->gmc)) return amdgpu_bo_size(bo); @@ -230,8 +228,9 @@ u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) return 0; - list_for_each_entry(block, &vres->blocks, link) - usage += amdgpu_vram_mgr_vis_size(adev, block); + mm = &container_of(res, struct ttm_range_mgr_node, base)->mm_nodes[0]; + for (usage = 0; pages; pages -= mm->size, mm++) + usage += amdgpu_vram_mgr_vis_size(adev, mm); return usage; } @@ -241,30 +240,23 @@ static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man) { struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); struct amdgpu_device *adev = to_amdgpu_device(mgr); - struct drm_buddy *mm = &mgr->mm; + struct drm_mm *mm = &mgr->mm; struct amdgpu_vram_reservation *rsv, *temp; - struct drm_buddy_block *block; uint64_t vis_usage; - list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) { - if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size, - rsv->size, mm->chunk_size, &rsv->allocated, - DRM_BUDDY_RANGE_ALLOCATION)) - continue; - - block = amdgpu_vram_mgr_first_block(&rsv->allocated); - if (!block) + list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node) { + if (drm_mm_reserve_node(mm, &rsv->mm_node)) continue; dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n", - rsv->start, rsv->size); + rsv->mm_node.start, rsv->mm_node.size); - vis_usage = amdgpu_vram_mgr_vis_size(adev, block); + vis_usage = amdgpu_vram_mgr_vis_size(adev, &rsv->mm_node); atomic64_add(vis_usage, &mgr->vis_usage); spin_lock(&man->bdev->lru_lock); - man->usage += rsv->size; + man->usage += rsv->mm_node.size << PAGE_SHIFT; spin_unlock(&man->bdev->lru_lock); - list_move(&rsv->blocks, &mgr->reserved_pages); + list_move(&rsv->node, &mgr->reserved_pages); } } @@ -286,16 +278,14 @@ int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr, if (!rsv) return -ENOMEM; - INIT_LIST_HEAD(&rsv->allocated); - INIT_LIST_HEAD(&rsv->blocks); + INIT_LIST_HEAD(&rsv->node); + rsv->mm_node.start = start >> PAGE_SHIFT; + rsv->mm_node.size = size >> PAGE_SHIFT; - rsv->start = start; - rsv->size = size; - - mutex_lock(&mgr->lock); - list_add_tail(&rsv->blocks, &mgr->reservations_pending); + spin_lock(&mgr->lock); + list_add_tail(&rsv->node, &mgr->reservations_pending); amdgpu_vram_mgr_do_reserve(&mgr->manager); - mutex_unlock(&mgr->lock); + spin_unlock(&mgr->lock); return 0; } @@ -317,19 +307,19 @@ int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr, struct amdgpu_vram_reservation *rsv; int ret; - mutex_lock(&mgr->lock); + spin_lock(&mgr->lock); - list_for_each_entry(rsv, &mgr->reservations_pending, blocks) { - if (rsv->start <= start && - (start < (rsv->start + rsv->size))) { + list_for_each_entry(rsv, &mgr->reservations_pending, node) { + if ((rsv->mm_node.start <= start) && + (start < (rsv->mm_node.start + rsv->mm_node.size))) { ret = -EBUSY; goto out; } } - list_for_each_entry(rsv, &mgr->reserved_pages, blocks) { - if (rsv->start <= start && - (start < (rsv->start + rsv->size))) { + list_for_each_entry(rsv, &mgr->reserved_pages, node) { + if ((rsv->mm_node.start <= start) && + (start < (rsv->mm_node.start + rsv->mm_node.size))) { ret = 0; goto out; } @@ -337,11 +327,33 @@ int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr, ret = -ENOENT; out: - mutex_unlock(&mgr->lock); + spin_unlock(&mgr->lock); return ret; } /** + * amdgpu_vram_mgr_virt_start - update virtual start address + * + * @mem: ttm_resource to update + * @node: just allocated node + * + * Calculate a virtual BO start address to easily check if everything is CPU + * accessible. + */ +static void amdgpu_vram_mgr_virt_start(struct ttm_resource *mem, + struct drm_mm_node *node) +{ + unsigned long start; + + start = node->start + node->size; + if (start > mem->num_pages) + start -= mem->num_pages; + else + start = 0; + mem->start = max(mem->start, start); +} + +/** * amdgpu_vram_mgr_new - allocate new ranges * * @man: TTM memory type manager @@ -356,44 +368,46 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, const struct ttm_place *place, struct ttm_resource **res) { - u64 vis_usage = 0, max_bytes, cur_size, min_block_size; + unsigned long lpfn, num_nodes, pages_per_node, pages_left, pages; struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); struct amdgpu_device *adev = to_amdgpu_device(mgr); - struct amdgpu_vram_mgr_resource *vres; - u64 size, remaining_size, lpfn, fpfn; - struct drm_buddy *mm = &mgr->mm; - struct drm_buddy_block *block; - unsigned long pages_per_block; + uint64_t vis_usage = 0, mem_bytes, max_bytes; + struct ttm_range_mgr_node *node; + struct drm_mm *mm = &mgr->mm; + enum drm_mm_insert_mode mode; + unsigned i; int r; - lpfn = place->lpfn << PAGE_SHIFT; + lpfn = place->lpfn; if (!lpfn) - lpfn = man->size; - - fpfn = place->fpfn << PAGE_SHIFT; + lpfn = man->size >> PAGE_SHIFT; max_bytes = adev->gmc.mc_vram_size; if (tbo->type != ttm_bo_type_kernel) max_bytes -= AMDGPU_VM_RESERVED_VRAM; + mem_bytes = tbo->base.size; if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { - pages_per_block = ~0ul; + pages_per_node = ~0ul; + num_nodes = 1; } else { #ifdef CONFIG_TRANSPARENT_HUGEPAGE - pages_per_block = HPAGE_PMD_NR; + pages_per_node = HPAGE_PMD_NR; #else /* default to 2MB */ - pages_per_block = 2UL << (20UL - PAGE_SHIFT); + pages_per_node = 2UL << (20UL - PAGE_SHIFT); #endif - pages_per_block = max_t(uint32_t, pages_per_block, - tbo->page_alignment); + pages_per_node = max_t(uint32_t, pages_per_node, + tbo->page_alignment); + num_nodes = DIV_ROUND_UP_ULL(PFN_UP(mem_bytes), pages_per_node); } - vres = kzalloc(sizeof(*vres), GFP_KERNEL); - if (!vres) + node = kvmalloc(struct_size(node, mm_nodes, num_nodes), + GFP_KERNEL | __GFP_ZERO); + if (!node) return -ENOMEM; - ttm_resource_init(tbo, place, &vres->base); + ttm_resource_init(tbo, place, &node->base); /* bail out quickly if there's likely not enough VRAM for this BO */ if (ttm_resource_manager_usage(man) > max_bytes) { @@ -401,130 +415,66 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, goto error_fini; } - INIT_LIST_HEAD(&vres->blocks); - + mode = DRM_MM_INSERT_BEST; if (place->flags & TTM_PL_FLAG_TOPDOWN) - vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; - - if (fpfn || lpfn != man->size) - /* Allocate blocks in desired range */ - vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; - - remaining_size = vres->base.num_pages << PAGE_SHIFT; - - mutex_lock(&mgr->lock); - while (remaining_size) { - if (tbo->page_alignment) - min_block_size = tbo->page_alignment << PAGE_SHIFT; - else - min_block_size = mgr->default_page_size; - - BUG_ON(min_block_size < mm->chunk_size); - - /* Limit maximum size to 2GiB due to SG table limitations */ - size = min(remaining_size, 2ULL << 30); - - if (size >= pages_per_block << PAGE_SHIFT) - min_block_size = pages_per_block << PAGE_SHIFT; - - cur_size = size; - - if (fpfn + size != place->lpfn << PAGE_SHIFT) { - /* - * Except for actual range allocation, modify the size and - * min_block_size conforming to continuous flag enablement - */ - if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { - size = roundup_pow_of_two(size); - min_block_size = size; - /* - * Modify the size value if size is not - * aligned with min_block_size - */ - } else if (!IS_ALIGNED(size, min_block_size)) { - size = round_up(size, min_block_size); + mode = DRM_MM_INSERT_HIGH; + + pages_left = node->base.num_pages; + + /* Limit maximum size to 2GB due to SG table limitations */ + pages = min(pages_left, 2UL << (30 - PAGE_SHIFT)); + + i = 0; + spin_lock(&mgr->lock); + while (pages_left) { + uint32_t alignment = tbo->page_alignment; + + if (pages >= pages_per_node) + alignment = pages_per_node; + + r = drm_mm_insert_node_in_range(mm, &node->mm_nodes[i], pages, + alignment, 0, place->fpfn, + lpfn, mode); + if (unlikely(r)) { + if (pages > pages_per_node) { + if (is_power_of_2(pages)) + pages = pages / 2; + else + pages = rounddown_pow_of_two(pages); + continue; } + goto error_free; } - r = drm_buddy_alloc_blocks(mm, fpfn, - lpfn, - size, - min_block_size, - &vres->blocks, - vres->flags); - if (unlikely(r)) - goto error_free_blocks; - - if (size > remaining_size) - remaining_size = 0; - else - remaining_size -= size; - } - mutex_unlock(&mgr->lock); - - if (cur_size != size) { - struct drm_buddy_block *block; - struct list_head *trim_list; - u64 original_size; - LIST_HEAD(temp); - - trim_list = &vres->blocks; - original_size = vres->base.num_pages << PAGE_SHIFT; - - /* - * If size value is rounded up to min_block_size, trim the last - * block to the required size - */ - if (!list_is_singular(&vres->blocks)) { - block = list_last_entry(&vres->blocks, typeof(*block), link); - list_move_tail(&block->link, &temp); - trim_list = &temp; - /* - * Compute the original_size value by subtracting the - * last block size with (aligned size - original size) - */ - original_size = amdgpu_vram_mgr_block_size(block) - (size - cur_size); - } + vis_usage += amdgpu_vram_mgr_vis_size(adev, &node->mm_nodes[i]); + amdgpu_vram_mgr_virt_start(&node->base, &node->mm_nodes[i]); + pages_left -= pages; + ++i; - mutex_lock(&mgr->lock); - drm_buddy_block_trim(mm, - original_size, - trim_list); - mutex_unlock(&mgr->lock); - - if (!list_empty(&temp)) - list_splice_tail(trim_list, &vres->blocks); - } - - list_for_each_entry(block, &vres->blocks, link) - vis_usage += amdgpu_vram_mgr_vis_size(adev, block); - - block = amdgpu_vram_mgr_first_block(&vres->blocks); - if (!block) { - r = -EINVAL; - goto error_fini; + if (pages > pages_left) + pages = pages_left; } + spin_unlock(&mgr->lock); - vres->base.start = amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; - - if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks)) - vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; + if (i == 1) + node->base.placement |= TTM_PL_FLAG_CONTIGUOUS; if (adev->gmc.xgmi.connected_to_cpu) - vres->base.bus.caching = ttm_cached; + node->base.bus.caching = ttm_cached; else - vres->base.bus.caching = ttm_write_combined; + node->base.bus.caching = ttm_write_combined; atomic64_add(vis_usage, &mgr->vis_usage); - *res = &vres->base; + *res = &node->base; return 0; -error_free_blocks: - drm_buddy_free_list(mm, &vres->blocks); - mutex_unlock(&mgr->lock); +error_free: + while (i--) + drm_mm_remove_node(&node->mm_nodes[i]); + spin_unlock(&mgr->lock); error_fini: - ttm_resource_fini(man, &vres->base); - kfree(vres); + ttm_resource_fini(man, &node->base); + kvfree(node); return r; } @@ -540,26 +490,27 @@ error_fini: static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, struct ttm_resource *res) { - struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); + struct ttm_range_mgr_node *node = to_ttm_range_mgr_node(res); struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); struct amdgpu_device *adev = to_amdgpu_device(mgr); - struct drm_buddy *mm = &mgr->mm; - struct drm_buddy_block *block; uint64_t vis_usage = 0; + unsigned i, pages; - mutex_lock(&mgr->lock); - list_for_each_entry(block, &vres->blocks, link) - vis_usage += amdgpu_vram_mgr_vis_size(adev, block); + spin_lock(&mgr->lock); + for (i = 0, pages = res->num_pages; pages; + pages -= node->mm_nodes[i].size, ++i) { + struct drm_mm_node *mm = &node->mm_nodes[i]; + drm_mm_remove_node(mm); + vis_usage += amdgpu_vram_mgr_vis_size(adev, mm); + } amdgpu_vram_mgr_do_reserve(man); - - drm_buddy_free_list(mm, &vres->blocks); - mutex_unlock(&mgr->lock); + spin_unlock(&mgr->lock); atomic64_sub(vis_usage, &mgr->vis_usage); ttm_resource_fini(man, res); - kfree(vres); + kvfree(node); } /** @@ -591,7 +542,7 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, if (!*sgt) return -ENOMEM; - /* Determine the number of DRM_BUDDY blocks to export */ + /* Determine the number of DRM_MM nodes to export */ amdgpu_res_first(res, offset, length, &cursor); while (cursor.remaining) { num_entries++; @@ -607,10 +558,10 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, sg->length = 0; /* - * Walk down DRM_BUDDY blocks to populate scatterlist nodes - * @note: Use iterator api to get first the DRM_BUDDY block + * Walk down DRM_MM nodes to populate scatterlist nodes + * @note: Use iterator api to get first the DRM_MM node * and the number of bytes from it. Access the following - * DRM_BUDDY block(s) if more buffer needs to exported + * DRM_MM node(s) if more buffer needs to exported */ amdgpu_res_first(res, offset, length, &cursor); for_each_sgtable_sg((*sgt), sg, i) { @@ -697,22 +648,13 @@ static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, struct drm_printer *printer) { struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); - struct drm_buddy *mm = &mgr->mm; - struct drm_buddy_block *block; drm_printf(printer, " vis usage:%llu\n", amdgpu_vram_mgr_vis_usage(mgr)); - mutex_lock(&mgr->lock); - drm_printf(printer, "default_page_size: %lluKiB\n", - mgr->default_page_size >> 10); - - drm_buddy_print(mm, printer); - - drm_printf(printer, "reserved:\n"); - list_for_each_entry(block, &mgr->reserved_pages, link) - drm_buddy_block_print(mm, block, printer); - mutex_unlock(&mgr->lock); + spin_lock(&mgr->lock); + drm_mm_print(&mgr->mm, printer); + spin_unlock(&mgr->lock); } static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { @@ -732,21 +674,16 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev) { struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; struct ttm_resource_manager *man = &mgr->manager; - int err; ttm_resource_manager_init(man, &adev->mman.bdev, adev->gmc.real_vram_size); man->func = &amdgpu_vram_mgr_func; - err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE); - if (err) - return err; - - mutex_init(&mgr->lock); + drm_mm_init(&mgr->mm, 0, man->size >> PAGE_SHIFT); + spin_lock_init(&mgr->lock); INIT_LIST_HEAD(&mgr->reservations_pending); INIT_LIST_HEAD(&mgr->reserved_pages); - mgr->default_page_size = PAGE_SIZE; ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); ttm_resource_manager_set_used(man, true); @@ -774,16 +711,16 @@ void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) if (ret) return; - mutex_lock(&mgr->lock); - list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) + spin_lock(&mgr->lock); + list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node) kfree(rsv); - list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) { - drm_buddy_free_list(&mgr->mm, &rsv->blocks); + list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) { + drm_mm_remove_node(&rsv->mm_node); kfree(rsv); } - drm_buddy_fini(&mgr->mm); - mutex_unlock(&mgr->lock); + drm_mm_takedown(&mgr->mm); + spin_unlock(&mgr->lock); ttm_resource_manager_cleanup(man); ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h deleted file mode 100644 index 9a2db87186c7..000000000000 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h +++ /dev/null @@ -1,89 +0,0 @@ -/* SPDX-License-Identifier: MIT - * Copyright 2021 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef __AMDGPU_VRAM_MGR_H__ -#define __AMDGPU_VRAM_MGR_H__ - -#include <drm/drm_buddy.h> - -struct amdgpu_vram_mgr { - struct ttm_resource_manager manager; - struct drm_buddy mm; - /* protects access to buffer objects */ - struct mutex lock; - struct list_head reservations_pending; - struct list_head reserved_pages; - atomic64_t vis_usage; - u64 default_page_size; -}; - -struct amdgpu_vram_mgr_resource { - struct ttm_resource base; - struct list_head blocks; - unsigned long flags; -}; - -static inline u64 amdgpu_vram_mgr_block_start(struct drm_buddy_block *block) -{ - return drm_buddy_block_offset(block); -} - -static inline u64 amdgpu_vram_mgr_block_size(struct drm_buddy_block *block) -{ - return PAGE_SIZE << drm_buddy_block_order(block); -} - -static inline struct drm_buddy_block * -amdgpu_vram_mgr_first_block(struct list_head *list) -{ - return list_first_entry_or_null(list, struct drm_buddy_block, link); -} - -static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head) -{ - struct drm_buddy_block *block; - u64 start, size; - - block = amdgpu_vram_mgr_first_block(head); - if (!block) - return false; - - while (head != block->link.next) { - start = amdgpu_vram_mgr_block_start(block); - size = amdgpu_vram_mgr_block_size(block); - - block = list_entry(block->link.next, struct drm_buddy_block, link); - if (start + size != amdgpu_vram_mgr_block_start(block)) - return false; - } - - return true; -} - -static inline struct amdgpu_vram_mgr_resource * -to_amdgpu_vram_mgr_resource(struct ttm_resource *res) -{ - return container_of(res, struct amdgpu_vram_mgr_resource, base); -} - -#endif diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index bf4200457772..a08769c5e94b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -184,6 +184,8 @@ static void kfd_device_info_init(struct kfd_dev *kfd, /* Navi2x+, Navi1x+ */ if (gc_version == IP_VERSION(10, 3, 6)) kfd->device_info.no_atomic_fw_version = 14; + else if (gc_version == IP_VERSION(10, 3, 7)) + kfd->device_info.no_atomic_fw_version = 3; else if (gc_version >= IP_VERSION(10, 3, 0)) kfd->device_info.no_atomic_fw_version = 92; else if (gc_version >= IP_VERSION(10, 1, 1)) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 1c2984bbda51..93ac33a8de9a 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -72,6 +72,7 @@ #include <linux/pci.h> #include <linux/firmware.h> #include <linux/component.h> +#include <linux/dmi.h> #include <drm/display/drm_dp_mst_helper.h> #include <drm/display/drm_hdmi_helper.h> @@ -462,6 +463,26 @@ static void dm_pflip_high_irq(void *interrupt_params) vrr_active, (int) !e); } +static void dm_crtc_handle_vblank(struct amdgpu_crtc *acrtc) +{ + struct drm_crtc *crtc = &acrtc->base; + struct drm_device *dev = crtc->dev; + unsigned long flags; + + drm_crtc_handle_vblank(crtc); + + spin_lock_irqsave(&dev->event_lock, flags); + + /* Send completion event for cursor-only commits */ + if (acrtc->event && acrtc->pflip_status != AMDGPU_FLIP_SUBMITTED) { + drm_crtc_send_vblank_event(crtc, acrtc->event); + drm_crtc_vblank_put(crtc); + acrtc->event = NULL; + } + + spin_unlock_irqrestore(&dev->event_lock, flags); +} + static void dm_vupdate_high_irq(void *interrupt_params) { struct common_irq_params *irq_params = interrupt_params; @@ -500,7 +521,7 @@ static void dm_vupdate_high_irq(void *interrupt_params) * if a pageflip happened inside front-porch. */ if (vrr_active) { - drm_crtc_handle_vblank(&acrtc->base); + dm_crtc_handle_vblank(acrtc); /* BTR processing for pre-DCE12 ASICs */ if (acrtc->dm_irq_params.stream && @@ -552,7 +573,7 @@ static void dm_crtc_high_irq(void *interrupt_params) * to dm_vupdate_high_irq after end of front-porch. */ if (!vrr_active) - drm_crtc_handle_vblank(&acrtc->base); + dm_crtc_handle_vblank(acrtc); /** * Following stuff must happen at start of vblank, for crc @@ -1382,6 +1403,41 @@ static bool dm_should_disable_stutter(struct pci_dev *pdev) return false; } +static const struct dmi_system_id hpd_disconnect_quirk_table[] = { + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Precision 3660"), + }, + }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Precision 3260"), + }, + }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Precision 3460"), + }, + }, + {} +}; + +static void retrieve_dmi_info(struct amdgpu_display_manager *dm) +{ + const struct dmi_system_id *dmi_id; + + dm->aux_hpd_discon_quirk = false; + + dmi_id = dmi_first_match(hpd_disconnect_quirk_table); + if (dmi_id) { + dm->aux_hpd_discon_quirk = true; + DRM_INFO("aux_hpd_discon_quirk attached\n"); + } +} + static int amdgpu_dm_init(struct amdgpu_device *adev) { struct dc_init_data init_data; @@ -1508,6 +1564,9 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) } INIT_LIST_HEAD(&adev->dm.da_list); + + retrieve_dmi_info(&adev->dm); + /* Display Core create. */ adev->dm.dc = dc_create(&init_data); @@ -5407,7 +5466,7 @@ fill_blending_from_plane_state(const struct drm_plane_state *plane_state, } } - if (per_pixel_alpha && plane_state->pixel_blend_mode == DRM_MODE_BLEND_COVERAGE) + if (*per_pixel_alpha && plane_state->pixel_blend_mode == DRM_MODE_BLEND_COVERAGE) *pre_multiplied_alpha = false; } @@ -9135,6 +9194,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, struct amdgpu_bo *abo; uint32_t target_vblank, last_flip_vblank; bool vrr_active = amdgpu_dm_vrr_active(acrtc_state); + bool cursor_update = false; bool pflip_present = false; struct { struct dc_surface_update surface_updates[MAX_SURFACES]; @@ -9170,8 +9230,13 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, struct dm_plane_state *dm_new_plane_state = to_dm_plane_state(new_plane_state); /* Cursor plane is handled after stream updates */ - if (plane->type == DRM_PLANE_TYPE_CURSOR) + if (plane->type == DRM_PLANE_TYPE_CURSOR) { + if ((fb && crtc == pcrtc) || + (old_plane_state->fb && old_plane_state->crtc == pcrtc)) + cursor_update = true; + continue; + } if (!fb || !crtc || pcrtc != crtc) continue; @@ -9334,6 +9399,17 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, bundle->stream_update.vrr_infopacket = &acrtc_state->stream->vrr_infopacket; } + } else if (cursor_update && acrtc_state->active_planes > 0 && + !acrtc_state->force_dpms_off && + acrtc_attach->base.state->event) { + drm_crtc_vblank_get(pcrtc); + + spin_lock_irqsave(&pcrtc->dev->event_lock, flags); + + acrtc_attach->event = acrtc_attach->base.state->event; + acrtc_attach->base.state->event = NULL; + + spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); } /* Update the planes if changed or disable if we don't have any. */ diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index aa34c0068f41..e80ef93f6550 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -540,6 +540,14 @@ struct amdgpu_display_manager { * last successfully applied backlight values. */ u32 actual_brightness[AMDGPU_DM_MAX_NUM_EDP]; + + /** + * @aux_hpd_discon_quirk: + * + * quirk for hpd discon while aux is on-going. + * occurred on certain intel platform + */ + bool aux_hpd_discon_quirk; }; enum dsc_clock_force_state { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index 9221b6690a4a..2b9b095e5f03 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -56,6 +56,8 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux, ssize_t result = 0; struct aux_payload payload; enum aux_return_code_type operation_result; + struct amdgpu_device *adev; + struct ddc_service *ddc; if (WARN_ON(msg->size > 16)) return -E2BIG; @@ -74,6 +76,21 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux, result = dc_link_aux_transfer_raw(TO_DM_AUX(aux)->ddc_service, &payload, &operation_result); + /* + * w/a on certain intel platform where hpd is unexpected to pull low during + * 1st sideband message transaction by return AUX_RET_ERROR_HPD_DISCON + * aux transaction is succuess in such case, therefore bypass the error + */ + ddc = TO_DM_AUX(aux)->ddc_service; + adev = ddc->ctx->driver_context; + if (adev->dm.aux_hpd_discon_quirk) { + if (msg->address == DP_SIDEBAND_MSG_DOWN_REQ_BASE && + operation_result == AUX_RET_ERROR_HPD_DISCON) { + result = 0; + operation_result = AUX_RET_SUCCESS; + } + } + if (payload.write && result >= 0) result = msg->size; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 6774dd8bb53e..3fe3fbac1e63 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -1117,12 +1117,13 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx) * on certain displays, such as the Sharp 4k. 36bpp is needed * to support SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 and * SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616 with actual > 10 bpc - * precision on at least DCN display engines. However, at least - * Carrizo with DCE_VERSION_11_0 does not like 36 bpp lb depth, - * so use only 30 bpp on DCE_VERSION_11_0. Testing with DCE 11.2 and 8.3 - * did not show such problems, so this seems to be the exception. + * precision on DCN display engines, but apparently not for DCE, as + * far as testing on DCE-11.2 and DCE-8 showed. Various DCE parts have + * problems: Carrizo with DCE_VERSION_11_0 does not like 36 bpp lb depth, + * neither do DCE-8 at 4k resolution, or DCE-11.2 (broken identify pixel + * passthrough). Therefore only use 36 bpp on DCN where it is actually needed. */ - if (plane_state->ctx->dce_version > DCE_VERSION_11_0) + if (plane_state->ctx->dce_version > DCE_VERSION_MAX) pipe_ctx->plane_res.scl_data.lb_params.depth = LB_PIXEL_DEPTH_36BPP; else pipe_ctx->plane_res.scl_data.lb_params.depth = LB_PIXEL_DEPTH_30BPP; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c index 5f8809f6990d..2fbd2926a531 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c @@ -1228,6 +1228,8 @@ int smu_v11_0_set_fan_speed_rpm(struct smu_context *smu, uint32_t crystal_clock_freq = 2500; uint32_t tach_period; + if (speed == 0) + return -EINVAL; /* * To prevent from possible overheat, some ASICs may have requirement * for minimum fan speed: diff --git a/drivers/gpu/drm/i915/gem/i915_gem_region.c b/drivers/gpu/drm/i915/gem/i915_gem_region.c index f46ee16a323a..a4fb577eceb4 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_region.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_region.c @@ -60,6 +60,8 @@ __i915_gem_object_create_region(struct intel_memory_region *mem, if (page_size) default_page_size = page_size; + /* We should be able to fit a page within an sg entry */ + GEM_BUG_ON(overflows_type(default_page_size, u32)); GEM_BUG_ON(!is_power_of_2_u64(default_page_size)); GEM_BUG_ON(default_page_size < PAGE_SIZE); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index 4c25d9b2f138..8f1bb6a4b7d1 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -620,10 +620,15 @@ i915_ttm_resource_get_st(struct drm_i915_gem_object *obj, struct ttm_resource *res) { struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); + u32 page_alignment; if (!i915_ttm_gtt_binds_lmem(res)) return i915_ttm_tt_get_st(bo->ttm); + page_alignment = bo->page_alignment << PAGE_SHIFT; + if (!page_alignment) + page_alignment = obj->mm.region->min_page_size; + /* * If CPU mapping differs, we need to add the ttm_tt pages to * the resulting st. Might make sense for GGTT. @@ -634,7 +639,8 @@ i915_ttm_resource_get_st(struct drm_i915_gem_object *obj, struct i915_refct_sgt *rsgt; rsgt = intel_region_ttm_resource_to_rsgt(obj->mm.region, - res); + res, + page_alignment); if (IS_ERR(rsgt)) return rsgt; @@ -643,7 +649,8 @@ i915_ttm_resource_get_st(struct drm_i915_gem_object *obj, return i915_refct_sgt_get(obj->ttm.cached_io_rsgt); } - return intel_region_ttm_resource_to_rsgt(obj->mm.region, res); + return intel_region_ttm_resource_to_rsgt(obj->mm.region, res, + page_alignment); } static int i915_ttm_truncate(struct drm_i915_gem_object *obj) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_wait.c b/drivers/gpu/drm/i915/gem/i915_gem_wait.c index 319936f91ac5..e6e01c2a74a6 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_wait.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_wait.c @@ -9,6 +9,7 @@ #include <linux/jiffies.h> #include "gt/intel_engine.h" +#include "gt/intel_rps.h" #include "i915_gem_ioctls.h" #include "i915_gem_object.h" @@ -31,6 +32,37 @@ i915_gem_object_wait_fence(struct dma_fence *fence, timeout); } +static void +i915_gem_object_boost(struct dma_resv *resv, unsigned int flags) +{ + struct dma_resv_iter cursor; + struct dma_fence *fence; + + /* + * Prescan all fences for potential boosting before we begin waiting. + * + * When we wait, we wait on outstanding fences serially. If the + * dma-resv contains a sequence such as 1:1, 1:2 instead of a reduced + * form 1:2, then as we look at each wait in turn we see that each + * request is currently executing and not worthy of boosting. But if + * we only happen to look at the final fence in the sequence (because + * of request coalescing or splitting between read/write arrays by + * the iterator), then we would boost. As such our decision to boost + * or not is delicately balanced on the order we wait on fences. + * + * So instead of looking for boosts sequentially, look for all boosts + * upfront and then wait on the outstanding fences. + */ + + dma_resv_iter_begin(&cursor, resv, + dma_resv_usage_rw(flags & I915_WAIT_ALL)); + dma_resv_for_each_fence_unlocked(&cursor, fence) + if (dma_fence_is_i915(fence) && + !i915_request_started(to_request(fence))) + intel_rps_boost(to_request(fence)); + dma_resv_iter_end(&cursor); +} + static long i915_gem_object_wait_reservation(struct dma_resv *resv, unsigned int flags, @@ -40,6 +72,8 @@ i915_gem_object_wait_reservation(struct dma_resv *resv, struct dma_fence *fence; long ret = timeout ?: 1; + i915_gem_object_boost(resv, flags); + dma_resv_iter_begin(&cursor, resv, dma_resv_usage_rw(flags & I915_WAIT_ALL)); dma_resv_for_each_fence_unlocked(&cursor, fence) { diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index 51a0fe60c050..531af6ad7007 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -1209,6 +1209,20 @@ void intel_gt_invalidate_tlbs(struct intel_gt *gt) mutex_lock(>->tlb_invalidate_lock); intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); + spin_lock_irq(&uncore->lock); /* serialise invalidate with GT reset */ + + for_each_engine(engine, gt, id) { + struct reg_and_bit rb; + + rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num); + if (!i915_mmio_reg_offset(rb.reg)) + continue; + + intel_uncore_write_fw(uncore, rb.reg, rb.bit); + } + + spin_unlock_irq(&uncore->lock); + for_each_engine(engine, gt, id) { /* * HW architecture suggest typical invalidation time at 40us, @@ -1223,7 +1237,6 @@ void intel_gt_invalidate_tlbs(struct intel_gt *gt) if (!i915_mmio_reg_offset(rb.reg)) continue; - intel_uncore_write_fw(uncore, rb.reg, rb.bit); if (__intel_wait_for_register_fw(uncore, rb.reg, rb.bit, 0, timeout_us, timeout_ms, diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index a5338c3fde7a..c68d36fb5bbd 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -300,9 +300,9 @@ static int gen6_hw_domain_reset(struct intel_gt *gt, u32 hw_domain_mask) return err; } -static int gen6_reset_engines(struct intel_gt *gt, - intel_engine_mask_t engine_mask, - unsigned int retry) +static int __gen6_reset_engines(struct intel_gt *gt, + intel_engine_mask_t engine_mask, + unsigned int retry) { struct intel_engine_cs *engine; u32 hw_mask; @@ -321,6 +321,20 @@ static int gen6_reset_engines(struct intel_gt *gt, return gen6_hw_domain_reset(gt, hw_mask); } +static int gen6_reset_engines(struct intel_gt *gt, + intel_engine_mask_t engine_mask, + unsigned int retry) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(>->uncore->lock, flags); + ret = __gen6_reset_engines(gt, engine_mask, retry); + spin_unlock_irqrestore(>->uncore->lock, flags); + + return ret; +} + static struct intel_engine_cs *find_sfc_paired_vecs_engine(struct intel_engine_cs *engine) { int vecs_id; @@ -487,9 +501,9 @@ static void gen11_unlock_sfc(struct intel_engine_cs *engine) rmw_clear_fw(uncore, sfc_lock.lock_reg, sfc_lock.lock_bit); } -static int gen11_reset_engines(struct intel_gt *gt, - intel_engine_mask_t engine_mask, - unsigned int retry) +static int __gen11_reset_engines(struct intel_gt *gt, + intel_engine_mask_t engine_mask, + unsigned int retry) { struct intel_engine_cs *engine; intel_engine_mask_t tmp; @@ -583,8 +597,11 @@ static int gen8_reset_engines(struct intel_gt *gt, struct intel_engine_cs *engine; const bool reset_non_ready = retry >= 1; intel_engine_mask_t tmp; + unsigned long flags; int ret; + spin_lock_irqsave(>->uncore->lock, flags); + for_each_engine_masked(engine, gt, engine_mask, tmp) { ret = gen8_engine_reset_prepare(engine); if (ret && !reset_non_ready) @@ -612,17 +629,19 @@ static int gen8_reset_engines(struct intel_gt *gt, * This is best effort, so ignore any error from the initial reset. */ if (IS_DG2(gt->i915) && engine_mask == ALL_ENGINES) - gen11_reset_engines(gt, gt->info.engine_mask, 0); + __gen11_reset_engines(gt, gt->info.engine_mask, 0); if (GRAPHICS_VER(gt->i915) >= 11) - ret = gen11_reset_engines(gt, engine_mask, retry); + ret = __gen11_reset_engines(gt, engine_mask, retry); else - ret = gen6_reset_engines(gt, engine_mask, retry); + ret = __gen6_reset_engines(gt, engine_mask, retry); skip_reset: for_each_engine_masked(engine, gt, engine_mask, tmp) gen8_engine_reset_cancel(engine); + spin_unlock_irqrestore(>->uncore->lock, flags); + return ret; } diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 8b2c11dbe354..1109088fe8f6 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -176,8 +176,8 @@ static int live_lrc_layout(void *arg) continue; hw = shmem_pin_map(engine->default_state); - if (IS_ERR(hw)) { - err = PTR_ERR(hw); + if (!hw) { + err = -ENOMEM; break; } hw += LRC_STATE_OFFSET / sizeof(*hw); @@ -365,8 +365,8 @@ static int live_lrc_fixed(void *arg) continue; hw = shmem_pin_map(engine->default_state); - if (IS_ERR(hw)) { - err = PTR_ERR(hw); + if (!hw) { + err = -ENOMEM; break; } hw += LRC_STATE_OFFSET / sizeof(*hw); diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index b9eb75a2b400..1c35a41620ae 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -3117,9 +3117,9 @@ void intel_gvt_update_reg_whitelist(struct intel_vgpu *vgpu) continue; vaddr = shmem_pin_map(engine->default_state); - if (IS_ERR(vaddr)) { - gvt_err("failed to map %s->default state, err:%zd\n", - engine->name, PTR_ERR(vaddr)); + if (!vaddr) { + gvt_err("failed to map %s->default state\n", + engine->name); return; } diff --git a/drivers/gpu/drm/i915/i915_scatterlist.c b/drivers/gpu/drm/i915/i915_scatterlist.c index 159571b9bd24..dcc081874ec8 100644 --- a/drivers/gpu/drm/i915/i915_scatterlist.c +++ b/drivers/gpu/drm/i915/i915_scatterlist.c @@ -68,6 +68,7 @@ void i915_refct_sgt_init(struct i915_refct_sgt *rsgt, size_t size) * drm_mm_node * @node: The drm_mm_node. * @region_start: An offset to add to the dma addresses of the sg list. + * @page_alignment: Required page alignment for each sg entry. Power of two. * * Create a struct sg_table, initializing it from a struct drm_mm_node, * taking a maximum segment length into account, splitting into segments @@ -77,22 +78,25 @@ void i915_refct_sgt_init(struct i915_refct_sgt *rsgt, size_t size) * error code cast to an error pointer on failure. */ struct i915_refct_sgt *i915_rsgt_from_mm_node(const struct drm_mm_node *node, - u64 region_start) + u64 region_start, + u32 page_alignment) { - const u64 max_segment = SZ_1G; /* Do we have a limit on this? */ - u64 segment_pages = max_segment >> PAGE_SHIFT; + const u32 max_segment = round_down(UINT_MAX, page_alignment); + const u32 segment_pages = max_segment >> PAGE_SHIFT; u64 block_size, offset, prev_end; struct i915_refct_sgt *rsgt; struct sg_table *st; struct scatterlist *sg; + GEM_BUG_ON(!max_segment); + rsgt = kmalloc(sizeof(*rsgt), GFP_KERNEL); if (!rsgt) return ERR_PTR(-ENOMEM); i915_refct_sgt_init(rsgt, node->size << PAGE_SHIFT); st = &rsgt->table; - if (sg_alloc_table(st, DIV_ROUND_UP(node->size, segment_pages), + if (sg_alloc_table(st, DIV_ROUND_UP_ULL(node->size, segment_pages), GFP_KERNEL)) { i915_refct_sgt_put(rsgt); return ERR_PTR(-ENOMEM); @@ -112,12 +116,14 @@ struct i915_refct_sgt *i915_rsgt_from_mm_node(const struct drm_mm_node *node, sg = __sg_next(sg); sg_dma_address(sg) = region_start + offset; + GEM_BUG_ON(!IS_ALIGNED(sg_dma_address(sg), + page_alignment)); sg_dma_len(sg) = 0; sg->length = 0; st->nents++; } - len = min(block_size, max_segment - sg->length); + len = min_t(u64, block_size, max_segment - sg->length); sg->length += len; sg_dma_len(sg) += len; @@ -138,6 +144,7 @@ struct i915_refct_sgt *i915_rsgt_from_mm_node(const struct drm_mm_node *node, * i915_buddy_block list * @res: The struct i915_ttm_buddy_resource. * @region_start: An offset to add to the dma addresses of the sg list. + * @page_alignment: Required page alignment for each sg entry. Power of two. * * Create a struct sg_table, initializing it from struct i915_buddy_block list, * taking a maximum segment length into account, splitting into segments @@ -147,11 +154,12 @@ struct i915_refct_sgt *i915_rsgt_from_mm_node(const struct drm_mm_node *node, * error code cast to an error pointer on failure. */ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res, - u64 region_start) + u64 region_start, + u32 page_alignment) { struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); const u64 size = res->num_pages << PAGE_SHIFT; - const u64 max_segment = rounddown(UINT_MAX, PAGE_SIZE); + const u32 max_segment = round_down(UINT_MAX, page_alignment); struct drm_buddy *mm = bman_res->mm; struct list_head *blocks = &bman_res->blocks; struct drm_buddy_block *block; @@ -161,6 +169,7 @@ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res, resource_size_t prev_end; GEM_BUG_ON(list_empty(blocks)); + GEM_BUG_ON(!max_segment); rsgt = kmalloc(sizeof(*rsgt), GFP_KERNEL); if (!rsgt) @@ -191,12 +200,14 @@ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res, sg = __sg_next(sg); sg_dma_address(sg) = region_start + offset; + GEM_BUG_ON(!IS_ALIGNED(sg_dma_address(sg), + page_alignment)); sg_dma_len(sg) = 0; sg->length = 0; st->nents++; } - len = min(block_size, max_segment - sg->length); + len = min_t(u64, block_size, max_segment - sg->length); sg->length += len; sg_dma_len(sg) += len; diff --git a/drivers/gpu/drm/i915/i915_scatterlist.h b/drivers/gpu/drm/i915/i915_scatterlist.h index 12c6a1684081..9ddb3e743a3e 100644 --- a/drivers/gpu/drm/i915/i915_scatterlist.h +++ b/drivers/gpu/drm/i915/i915_scatterlist.h @@ -213,9 +213,11 @@ static inline void __i915_refct_sgt_init(struct i915_refct_sgt *rsgt, void i915_refct_sgt_init(struct i915_refct_sgt *rsgt, size_t size); struct i915_refct_sgt *i915_rsgt_from_mm_node(const struct drm_mm_node *node, - u64 region_start); + u64 region_start, + u32 page_alignment); struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res, - u64 region_start); + u64 region_start, + u32 page_alignment); #endif diff --git a/drivers/gpu/drm/i915/intel_region_ttm.c b/drivers/gpu/drm/i915/intel_region_ttm.c index 62ff77445b01..575d67bc6ffe 100644 --- a/drivers/gpu/drm/i915/intel_region_ttm.c +++ b/drivers/gpu/drm/i915/intel_region_ttm.c @@ -152,6 +152,7 @@ int intel_region_ttm_fini(struct intel_memory_region *mem) * Convert an opaque TTM resource manager resource to a refcounted sg_table. * @mem: The memory region. * @res: The resource manager resource obtained from the TTM resource manager. + * @page_alignment: Required page alignment for each sg entry. Power of two. * * The gem backends typically use sg-tables for operations on the underlying * io_memory. So provide a way for the backends to translate the @@ -161,16 +162,19 @@ int intel_region_ttm_fini(struct intel_memory_region *mem) */ struct i915_refct_sgt * intel_region_ttm_resource_to_rsgt(struct intel_memory_region *mem, - struct ttm_resource *res) + struct ttm_resource *res, + u32 page_alignment) { if (mem->is_range_manager) { struct ttm_range_mgr_node *range_node = to_ttm_range_mgr_node(res); return i915_rsgt_from_mm_node(&range_node->mm_nodes[0], - mem->region.start); + mem->region.start, + page_alignment); } else { - return i915_rsgt_from_buddy_resource(res, mem->region.start); + return i915_rsgt_from_buddy_resource(res, mem->region.start, + page_alignment); } } diff --git a/drivers/gpu/drm/i915/intel_region_ttm.h b/drivers/gpu/drm/i915/intel_region_ttm.h index cf9d86dcf409..5bb8d8b582ae 100644 --- a/drivers/gpu/drm/i915/intel_region_ttm.h +++ b/drivers/gpu/drm/i915/intel_region_ttm.h @@ -24,7 +24,8 @@ int intel_region_ttm_fini(struct intel_memory_region *mem); struct i915_refct_sgt * intel_region_ttm_resource_to_rsgt(struct intel_memory_region *mem, - struct ttm_resource *res); + struct ttm_resource *res, + u32 page_alignment); void intel_region_ttm_resource_free(struct intel_memory_region *mem, struct ttm_resource *res); diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index 8633bec18fa7..ab9f17fc85bc 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -742,7 +742,7 @@ static int pot_hole(struct i915_address_space *vm, u64 addr; for (addr = round_up(hole_start + min_alignment, step) - min_alignment; - addr <= round_down(hole_end - (2 * min_alignment), step) - min_alignment; + hole_end > addr && hole_end - addr >= 2 * min_alignment; addr += step) { err = i915_vma_pin(vma, 0, 0, addr | flags); if (err) { diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c index 73eb53edb8de..3b18e5905c86 100644 --- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c +++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c @@ -451,7 +451,6 @@ out_put: static int igt_mock_max_segment(void *arg) { - const unsigned int max_segment = rounddown(UINT_MAX, PAGE_SIZE); struct intel_memory_region *mem = arg; struct drm_i915_private *i915 = mem->i915; struct i915_ttm_buddy_resource *res; @@ -460,7 +459,10 @@ static int igt_mock_max_segment(void *arg) struct drm_buddy *mm; struct list_head *blocks; struct scatterlist *sg; + I915_RND_STATE(prng); LIST_HEAD(objects); + unsigned int max_segment; + unsigned int ps; u64 size; int err = 0; @@ -472,7 +474,13 @@ static int igt_mock_max_segment(void *arg) */ size = SZ_8G; - mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0, 0); + ps = PAGE_SIZE; + if (i915_prandom_u64_state(&prng) & 1) + ps = SZ_64K; /* For something like DG2 */ + + max_segment = round_down(UINT_MAX, ps); + + mem = mock_region_create(i915, 0, size, ps, 0, 0); if (IS_ERR(mem)) return PTR_ERR(mem); @@ -498,12 +506,21 @@ static int igt_mock_max_segment(void *arg) } for (sg = obj->mm.pages->sgl; sg; sg = sg_next(sg)) { + dma_addr_t daddr = sg_dma_address(sg); + if (sg->length > max_segment) { pr_err("%s: Created an oversized scatterlist entry, %u > %u\n", __func__, sg->length, max_segment); err = -EINVAL; goto out_close; } + + if (!IS_ALIGNED(daddr, ps)) { + pr_err("%s: Created an unaligned scatterlist entry, addr=%pa, ps=%u\n", + __func__, &daddr, ps); + err = -EINVAL; + goto out_close; + } } out_close: diff --git a/drivers/gpu/drm/i915/selftests/mock_region.c b/drivers/gpu/drm/i915/selftests/mock_region.c index 670557ce1024..bac21fe84ca5 100644 --- a/drivers/gpu/drm/i915/selftests/mock_region.c +++ b/drivers/gpu/drm/i915/selftests/mock_region.c @@ -33,7 +33,8 @@ static int mock_region_get_pages(struct drm_i915_gem_object *obj) return PTR_ERR(obj->mm.res); obj->mm.rsgt = intel_region_ttm_resource_to_rsgt(obj->mm.region, - obj->mm.res); + obj->mm.res, + obj->mm.region->min_page_size); if (IS_ERR(obj->mm.rsgt)) { err = PTR_ERR(obj->mm.rsgt); goto err_free_resource; diff --git a/drivers/infiniband/hw/irdma/cm.c b/drivers/infiniband/hw/irdma/cm.c index 638bf4a1ed94..646fa8677490 100644 --- a/drivers/infiniband/hw/irdma/cm.c +++ b/drivers/infiniband/hw/irdma/cm.c @@ -4231,10 +4231,6 @@ void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr, struct irdma_cm_node *cm_node; struct list_head teardown_list; struct ib_qp_attr attr; - struct irdma_sc_vsi *vsi = &iwdev->vsi; - struct irdma_sc_qp *sc_qp; - struct irdma_qp *qp; - int i; INIT_LIST_HEAD(&teardown_list); @@ -4251,52 +4247,6 @@ void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr, irdma_cm_disconn(cm_node->iwqp); irdma_rem_ref_cm_node(cm_node); } - if (!iwdev->roce_mode) - return; - - INIT_LIST_HEAD(&teardown_list); - for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { - mutex_lock(&vsi->qos[i].qos_mutex); - list_for_each_safe (list_node, list_core_temp, - &vsi->qos[i].qplist) { - u32 qp_ip[4]; - - sc_qp = container_of(list_node, struct irdma_sc_qp, - list); - if (sc_qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_RC) - continue; - - qp = sc_qp->qp_uk.back_qp; - if (!disconnect_all) { - if (nfo->ipv4) - qp_ip[0] = qp->udp_info.local_ipaddr[3]; - else - memcpy(qp_ip, - &qp->udp_info.local_ipaddr[0], - sizeof(qp_ip)); - } - - if (disconnect_all || - (nfo->vlan_id == (qp->udp_info.vlan_tag & VLAN_VID_MASK) && - !memcmp(qp_ip, ipaddr, nfo->ipv4 ? 4 : 16))) { - spin_lock(&iwdev->rf->qptable_lock); - if (iwdev->rf->qp_table[sc_qp->qp_uk.qp_id]) { - irdma_qp_add_ref(&qp->ibqp); - list_add(&qp->teardown_entry, - &teardown_list); - } - spin_unlock(&iwdev->rf->qptable_lock); - } - } - mutex_unlock(&vsi->qos[i].qos_mutex); - } - - list_for_each_safe (list_node, list_core_temp, &teardown_list) { - qp = container_of(list_node, struct irdma_qp, teardown_entry); - attr.qp_state = IB_QPS_ERR; - irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); - irdma_qp_rem_ref(&qp->ibqp); - } } /** diff --git a/drivers/infiniband/hw/irdma/i40iw_hw.c b/drivers/infiniband/hw/irdma/i40iw_hw.c index e46fc110004d..50299f58b6b3 100644 --- a/drivers/infiniband/hw/irdma/i40iw_hw.c +++ b/drivers/infiniband/hw/irdma/i40iw_hw.c @@ -201,6 +201,7 @@ void i40iw_init_hw(struct irdma_sc_dev *dev) dev->hw_attrs.uk_attrs.max_hw_read_sges = I40IW_MAX_SGE_RD; dev->hw_attrs.max_hw_device_pages = I40IW_MAX_PUSH_PAGE_COUNT; dev->hw_attrs.uk_attrs.max_hw_inline = I40IW_MAX_INLINE_DATA_SIZE; + dev->hw_attrs.page_size_cap = SZ_4K | SZ_2M; dev->hw_attrs.max_hw_ird = I40IW_MAX_IRD_SIZE; dev->hw_attrs.max_hw_ord = I40IW_MAX_ORD_SIZE; dev->hw_attrs.max_hw_wqes = I40IW_MAX_WQ_ENTRIES; diff --git a/drivers/infiniband/hw/irdma/icrdma_hw.c b/drivers/infiniband/hw/irdma/icrdma_hw.c index cf53b17510cd..5986fd906308 100644 --- a/drivers/infiniband/hw/irdma/icrdma_hw.c +++ b/drivers/infiniband/hw/irdma/icrdma_hw.c @@ -139,6 +139,7 @@ void icrdma_init_hw(struct irdma_sc_dev *dev) dev->cqp_db = dev->hw_regs[IRDMA_CQPDB]; dev->cq_ack_db = dev->hw_regs[IRDMA_CQACK]; dev->irq_ops = &icrdma_irq_ops; + dev->hw_attrs.page_size_cap = SZ_4K | SZ_2M | SZ_1G; dev->hw_attrs.max_hw_ird = ICRDMA_MAX_IRD_SIZE; dev->hw_attrs.max_hw_ord = ICRDMA_MAX_ORD_SIZE; dev->hw_attrs.max_stat_inst = ICRDMA_MAX_STATS_COUNT; diff --git a/drivers/infiniband/hw/irdma/irdma.h b/drivers/infiniband/hw/irdma/irdma.h index 46c12334c735..4789e85d717b 100644 --- a/drivers/infiniband/hw/irdma/irdma.h +++ b/drivers/infiniband/hw/irdma/irdma.h @@ -127,6 +127,7 @@ struct irdma_hw_attrs { u64 max_hw_outbound_msg_size; u64 max_hw_inbound_msg_size; u64 max_mr_size; + u64 page_size_cap; u32 min_hw_qp_id; u32 min_hw_aeq_size; u32 max_hw_aeq_size; diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index c4412ece5a6d..96135a228f26 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -32,7 +32,7 @@ static int irdma_query_device(struct ib_device *ibdev, props->vendor_part_id = pcidev->device; props->hw_ver = rf->pcidev->revision; - props->page_size_cap = SZ_4K | SZ_2M | SZ_1G; + props->page_size_cap = hw_attrs->page_size_cap; props->max_mr_size = hw_attrs->max_mr_size; props->max_qp = rf->max_qp - rf->used_qps; props->max_qp_wr = hw_attrs->max_qp_wr; @@ -2781,7 +2781,7 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) { iwmr->page_size = ib_umem_find_best_pgsz(region, - SZ_4K | SZ_2M | SZ_1G, + iwdev->rf->sc_dev.hw_attrs.page_size_cap, virt); if (unlikely(!iwmr->page_size)) { kfree(iwmr); diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index 3ad9870db108..aa45a9fee6a0 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -900,6 +900,11 @@ static int goodix_add_acpi_gpio_mappings(struct goodix_ts_data *ts) } else { dev_warn(dev, "Unexpected ACPI resources: gpio_count %d, gpio_int_idx %d\n", ts->gpio_count, ts->gpio_int_idx); + /* + * On some devices _PS0 does a reset for us and + * sometimes this is necessary for things to work. + */ + acpi_device_fix_up_power(ACPI_COMPANION(dev)); return -EINVAL; } diff --git a/drivers/input/touchscreen/usbtouchscreen.c b/drivers/input/touchscreen/usbtouchscreen.c index 43c521f50c85..3dda6eaabdab 100644 --- a/drivers/input/touchscreen/usbtouchscreen.c +++ b/drivers/input/touchscreen/usbtouchscreen.c @@ -1654,6 +1654,9 @@ static int usbtouch_probe(struct usb_interface *intf, if (id->driver_info == DEVTYPE_IGNORE) return -ENODEV; + if (id->driver_info >= ARRAY_SIZE(usbtouch_dev_info)) + return -ENODEV; + endpoint = usbtouch_get_input_endpoint(intf->cur_altsetting); if (!endpoint) return -ENXIO; diff --git a/drivers/input/touchscreen/wm97xx-core.c b/drivers/input/touchscreen/wm97xx-core.c index 2757c7768ffe..f51ab5614532 100644 --- a/drivers/input/touchscreen/wm97xx-core.c +++ b/drivers/input/touchscreen/wm97xx-core.c @@ -758,7 +758,9 @@ batt_err: static int wm97xx_mfd_remove(struct platform_device *pdev) { - return wm97xx_remove(&pdev->dev); + wm97xx_remove(&pdev->dev); + + return 0; } static int __maybe_unused wm97xx_suspend(struct device *dev) diff --git a/drivers/net/amt.c b/drivers/net/amt.c index 732f4c0daa73..febfcf2d92af 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -563,7 +563,7 @@ static struct sk_buff *amt_build_igmp_gq(struct amt_dev *amt) ihv3->nsrcs = 0; ihv3->resv = 0; ihv3->suppress = false; - ihv3->qrv = amt->net->ipv4.sysctl_igmp_qrv; + ihv3->qrv = READ_ONCE(amt->net->ipv4.sysctl_igmp_qrv); ihv3->csum = 0; csum = &ihv3->csum; csum_start = (void *)ihv3; @@ -577,14 +577,14 @@ static struct sk_buff *amt_build_igmp_gq(struct amt_dev *amt) return skb; } -static void __amt_update_gw_status(struct amt_dev *amt, enum amt_status status, - bool validate) +static void amt_update_gw_status(struct amt_dev *amt, enum amt_status status, + bool validate) { if (validate && amt->status >= status) return; netdev_dbg(amt->dev, "Update GW status %s -> %s", status_str[amt->status], status_str[status]); - amt->status = status; + WRITE_ONCE(amt->status, status); } static void __amt_update_relay_status(struct amt_tunnel_list *tunnel, @@ -600,14 +600,6 @@ static void __amt_update_relay_status(struct amt_tunnel_list *tunnel, tunnel->status = status; } -static void amt_update_gw_status(struct amt_dev *amt, enum amt_status status, - bool validate) -{ - spin_lock_bh(&amt->lock); - __amt_update_gw_status(amt, status, validate); - spin_unlock_bh(&amt->lock); -} - static void amt_update_relay_status(struct amt_tunnel_list *tunnel, enum amt_status status, bool validate) { @@ -700,9 +692,7 @@ static void amt_send_discovery(struct amt_dev *amt) if (unlikely(net_xmit_eval(err))) amt->dev->stats.tx_errors++; - spin_lock_bh(&amt->lock); - __amt_update_gw_status(amt, AMT_STATUS_SENT_DISCOVERY, true); - spin_unlock_bh(&amt->lock); + amt_update_gw_status(amt, AMT_STATUS_SENT_DISCOVERY, true); out: rcu_read_unlock(); } @@ -900,6 +890,28 @@ static void amt_send_mld_gq(struct amt_dev *amt, struct amt_tunnel_list *tunnel) } #endif +static bool amt_queue_event(struct amt_dev *amt, enum amt_event event, + struct sk_buff *skb) +{ + int index; + + spin_lock_bh(&amt->lock); + if (amt->nr_events >= AMT_MAX_EVENTS) { + spin_unlock_bh(&amt->lock); + return 1; + } + + index = (amt->event_idx + amt->nr_events) % AMT_MAX_EVENTS; + amt->events[index].event = event; + amt->events[index].skb = skb; + amt->nr_events++; + amt->event_idx %= AMT_MAX_EVENTS; + queue_work(amt_wq, &amt->event_wq); + spin_unlock_bh(&amt->lock); + + return 0; +} + static void amt_secret_work(struct work_struct *work) { struct amt_dev *amt = container_of(to_delayed_work(work), @@ -913,58 +925,72 @@ static void amt_secret_work(struct work_struct *work) msecs_to_jiffies(AMT_SECRET_TIMEOUT)); } -static void amt_discovery_work(struct work_struct *work) +static void amt_event_send_discovery(struct amt_dev *amt) { - struct amt_dev *amt = container_of(to_delayed_work(work), - struct amt_dev, - discovery_wq); - - spin_lock_bh(&amt->lock); if (amt->status > AMT_STATUS_SENT_DISCOVERY) goto out; get_random_bytes(&amt->nonce, sizeof(__be32)); - spin_unlock_bh(&amt->lock); amt_send_discovery(amt); - spin_lock_bh(&amt->lock); out: mod_delayed_work(amt_wq, &amt->discovery_wq, msecs_to_jiffies(AMT_DISCOVERY_TIMEOUT)); - spin_unlock_bh(&amt->lock); } -static void amt_req_work(struct work_struct *work) +static void amt_discovery_work(struct work_struct *work) { struct amt_dev *amt = container_of(to_delayed_work(work), struct amt_dev, - req_wq); + discovery_wq); + + if (amt_queue_event(amt, AMT_EVENT_SEND_DISCOVERY, NULL)) + mod_delayed_work(amt_wq, &amt->discovery_wq, + msecs_to_jiffies(AMT_DISCOVERY_TIMEOUT)); +} + +static void amt_event_send_request(struct amt_dev *amt) +{ u32 exp; - spin_lock_bh(&amt->lock); if (amt->status < AMT_STATUS_RECEIVED_ADVERTISEMENT) goto out; if (amt->req_cnt > AMT_MAX_REQ_COUNT) { netdev_dbg(amt->dev, "Gateway is not ready"); amt->qi = AMT_INIT_REQ_TIMEOUT; - amt->ready4 = false; - amt->ready6 = false; + WRITE_ONCE(amt->ready4, false); + WRITE_ONCE(amt->ready6, false); amt->remote_ip = 0; - __amt_update_gw_status(amt, AMT_STATUS_INIT, false); + amt_update_gw_status(amt, AMT_STATUS_INIT, false); amt->req_cnt = 0; + amt->nonce = 0; goto out; } - spin_unlock_bh(&amt->lock); + + if (!amt->req_cnt) { + WRITE_ONCE(amt->ready4, false); + WRITE_ONCE(amt->ready6, false); + get_random_bytes(&amt->nonce, sizeof(__be32)); + } amt_send_request(amt, false); amt_send_request(amt, true); - spin_lock_bh(&amt->lock); - __amt_update_gw_status(amt, AMT_STATUS_SENT_REQUEST, true); + amt_update_gw_status(amt, AMT_STATUS_SENT_REQUEST, true); amt->req_cnt++; out: exp = min_t(u32, (1 * (1 << amt->req_cnt)), AMT_MAX_REQ_TIMEOUT); mod_delayed_work(amt_wq, &amt->req_wq, msecs_to_jiffies(exp * 1000)); - spin_unlock_bh(&amt->lock); +} + +static void amt_req_work(struct work_struct *work) +{ + struct amt_dev *amt = container_of(to_delayed_work(work), + struct amt_dev, + req_wq); + + if (amt_queue_event(amt, AMT_EVENT_SEND_REQUEST, NULL)) + mod_delayed_work(amt_wq, &amt->req_wq, + msecs_to_jiffies(100)); } static bool amt_send_membership_update(struct amt_dev *amt, @@ -1220,7 +1246,8 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev) /* Gateway only passes IGMP/MLD packets */ if (!report) goto free; - if ((!v6 && !amt->ready4) || (v6 && !amt->ready6)) + if ((!v6 && !READ_ONCE(amt->ready4)) || + (v6 && !READ_ONCE(amt->ready6))) goto free; if (amt_send_membership_update(amt, skb, v6)) goto free; @@ -2236,6 +2263,10 @@ static bool amt_advertisement_handler(struct amt_dev *amt, struct sk_buff *skb) ipv4_is_zeronet(amta->ip4)) return true; + if (amt->status != AMT_STATUS_SENT_DISCOVERY || + amt->nonce != amta->nonce) + return true; + amt->remote_ip = amta->ip4; netdev_dbg(amt->dev, "advertised remote ip = %pI4\n", &amt->remote_ip); mod_delayed_work(amt_wq, &amt->req_wq, 0); @@ -2251,6 +2282,9 @@ static bool amt_multicast_data_handler(struct amt_dev *amt, struct sk_buff *skb) struct ethhdr *eth; struct iphdr *iph; + if (READ_ONCE(amt->status) != AMT_STATUS_SENT_UPDATE) + return true; + hdr_size = sizeof(*amtmd) + sizeof(struct udphdr); if (!pskb_may_pull(skb, hdr_size)) return true; @@ -2325,6 +2359,9 @@ static bool amt_membership_query_handler(struct amt_dev *amt, if (amtmq->reserved || amtmq->version) return true; + if (amtmq->nonce != amt->nonce) + return true; + hdr_size -= sizeof(*eth); if (iptunnel_pull_header(skb, hdr_size, htons(ETH_P_TEB), false)) return true; @@ -2339,6 +2376,9 @@ static bool amt_membership_query_handler(struct amt_dev *amt, iph = ip_hdr(skb); if (iph->version == 4) { + if (READ_ONCE(amt->ready4)) + return true; + if (!pskb_may_pull(skb, sizeof(*iph) + AMT_IPHDR_OPTS + sizeof(*ihv3))) return true; @@ -2349,12 +2389,10 @@ static bool amt_membership_query_handler(struct amt_dev *amt, ihv3 = skb_pull(skb, sizeof(*iph) + AMT_IPHDR_OPTS); skb_reset_transport_header(skb); skb_push(skb, sizeof(*iph) + AMT_IPHDR_OPTS); - spin_lock_bh(&amt->lock); - amt->ready4 = true; + WRITE_ONCE(amt->ready4, true); amt->mac = amtmq->response_mac; amt->req_cnt = 0; amt->qi = ihv3->qqic; - spin_unlock_bh(&amt->lock); skb->protocol = htons(ETH_P_IP); eth->h_proto = htons(ETH_P_IP); ip_eth_mc_map(iph->daddr, eth->h_dest); @@ -2363,6 +2401,9 @@ static bool amt_membership_query_handler(struct amt_dev *amt, struct mld2_query *mld2q; struct ipv6hdr *ip6h; + if (READ_ONCE(amt->ready6)) + return true; + if (!pskb_may_pull(skb, sizeof(*ip6h) + AMT_IP6HDR_OPTS + sizeof(*mld2q))) return true; @@ -2374,12 +2415,10 @@ static bool amt_membership_query_handler(struct amt_dev *amt, mld2q = skb_pull(skb, sizeof(*ip6h) + AMT_IP6HDR_OPTS); skb_reset_transport_header(skb); skb_push(skb, sizeof(*ip6h) + AMT_IP6HDR_OPTS); - spin_lock_bh(&amt->lock); - amt->ready6 = true; + WRITE_ONCE(amt->ready6, true); amt->mac = amtmq->response_mac; amt->req_cnt = 0; amt->qi = mld2q->mld2q_qqic; - spin_unlock_bh(&amt->lock); skb->protocol = htons(ETH_P_IPV6); eth->h_proto = htons(ETH_P_IPV6); ipv6_eth_mc_map(&ip6h->daddr, eth->h_dest); @@ -2392,12 +2431,14 @@ static bool amt_membership_query_handler(struct amt_dev *amt, skb->pkt_type = PACKET_MULTICAST; skb->ip_summed = CHECKSUM_NONE; len = skb->len; + local_bh_disable(); if (__netif_rx(skb) == NET_RX_SUCCESS) { amt_update_gw_status(amt, AMT_STATUS_RECEIVED_QUERY, true); dev_sw_netstats_rx_add(amt->dev, len); } else { amt->dev->stats.rx_dropped++; } + local_bh_enable(); return false; } @@ -2638,7 +2679,9 @@ static bool amt_request_handler(struct amt_dev *amt, struct sk_buff *skb) if (tunnel->ip4 == iph->saddr) goto send; + spin_lock_bh(&amt->lock); if (amt->nr_tunnels >= amt->max_tunnels) { + spin_unlock_bh(&amt->lock); icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); return true; } @@ -2646,8 +2689,10 @@ static bool amt_request_handler(struct amt_dev *amt, struct sk_buff *skb) tunnel = kzalloc(sizeof(*tunnel) + (sizeof(struct hlist_head) * amt->hash_buckets), GFP_ATOMIC); - if (!tunnel) + if (!tunnel) { + spin_unlock_bh(&amt->lock); return true; + } tunnel->source_port = udph->source; tunnel->ip4 = iph->saddr; @@ -2660,10 +2705,9 @@ static bool amt_request_handler(struct amt_dev *amt, struct sk_buff *skb) INIT_DELAYED_WORK(&tunnel->gc_wq, amt_tunnel_expire); - spin_lock_bh(&amt->lock); list_add_tail_rcu(&tunnel->list, &amt->tunnel_list); tunnel->key = amt->key; - amt_update_relay_status(tunnel, AMT_STATUS_RECEIVED_REQUEST, true); + __amt_update_relay_status(tunnel, AMT_STATUS_RECEIVED_REQUEST, true); amt->nr_tunnels++; mod_delayed_work(amt_wq, &tunnel->gc_wq, msecs_to_jiffies(amt_gmi(amt))); @@ -2688,6 +2732,38 @@ send: return false; } +static void amt_gw_rcv(struct amt_dev *amt, struct sk_buff *skb) +{ + int type = amt_parse_type(skb); + int err = 1; + + if (type == -1) + goto drop; + + if (amt->mode == AMT_MODE_GATEWAY) { + switch (type) { + case AMT_MSG_ADVERTISEMENT: + err = amt_advertisement_handler(amt, skb); + break; + case AMT_MSG_MEMBERSHIP_QUERY: + err = amt_membership_query_handler(amt, skb); + if (!err) + return; + break; + default: + netdev_dbg(amt->dev, "Invalid type of Gateway\n"); + break; + } + } +drop: + if (err) { + amt->dev->stats.rx_dropped++; + kfree_skb(skb); + } else { + consume_skb(skb); + } +} + static int amt_rcv(struct sock *sk, struct sk_buff *skb) { struct amt_dev *amt; @@ -2719,8 +2795,12 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb) err = true; goto drop; } - err = amt_advertisement_handler(amt, skb); - break; + if (amt_queue_event(amt, AMT_EVENT_RECEIVE, skb)) { + netdev_dbg(amt->dev, "AMT Event queue full\n"); + err = true; + goto drop; + } + goto out; case AMT_MSG_MULTICAST_DATA: if (iph->saddr != amt->remote_ip) { netdev_dbg(amt->dev, "Invalid Relay IP\n"); @@ -2738,11 +2818,12 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb) err = true; goto drop; } - err = amt_membership_query_handler(amt, skb); - if (err) + if (amt_queue_event(amt, AMT_EVENT_RECEIVE, skb)) { + netdev_dbg(amt->dev, "AMT Event queue full\n"); + err = true; goto drop; - else - goto out; + } + goto out; default: err = true; netdev_dbg(amt->dev, "Invalid type of Gateway\n"); @@ -2780,6 +2861,46 @@ out: return 0; } +static void amt_event_work(struct work_struct *work) +{ + struct amt_dev *amt = container_of(work, struct amt_dev, event_wq); + struct sk_buff *skb; + u8 event; + int i; + + for (i = 0; i < AMT_MAX_EVENTS; i++) { + spin_lock_bh(&amt->lock); + if (amt->nr_events == 0) { + spin_unlock_bh(&amt->lock); + return; + } + event = amt->events[amt->event_idx].event; + skb = amt->events[amt->event_idx].skb; + amt->events[amt->event_idx].event = AMT_EVENT_NONE; + amt->events[amt->event_idx].skb = NULL; + amt->nr_events--; + amt->event_idx++; + amt->event_idx %= AMT_MAX_EVENTS; + spin_unlock_bh(&amt->lock); + + switch (event) { + case AMT_EVENT_RECEIVE: + amt_gw_rcv(amt, skb); + break; + case AMT_EVENT_SEND_DISCOVERY: + amt_event_send_discovery(amt); + break; + case AMT_EVENT_SEND_REQUEST: + amt_event_send_request(amt); + break; + default: + if (skb) + kfree_skb(skb); + break; + } + } +} + static int amt_err_lookup(struct sock *sk, struct sk_buff *skb) { struct amt_dev *amt; @@ -2804,7 +2925,7 @@ static int amt_err_lookup(struct sock *sk, struct sk_buff *skb) break; case AMT_MSG_REQUEST: case AMT_MSG_MEMBERSHIP_UPDATE: - if (amt->status >= AMT_STATUS_RECEIVED_ADVERTISEMENT) + if (READ_ONCE(amt->status) >= AMT_STATUS_RECEIVED_ADVERTISEMENT) mod_delayed_work(amt_wq, &amt->req_wq, 0); break; default: @@ -2867,6 +2988,8 @@ static int amt_dev_open(struct net_device *dev) amt->ready4 = false; amt->ready6 = false; + amt->event_idx = 0; + amt->nr_events = 0; err = amt_socket_create(amt); if (err) @@ -2874,6 +2997,7 @@ static int amt_dev_open(struct net_device *dev) amt->req_cnt = 0; amt->remote_ip = 0; + amt->nonce = 0; get_random_bytes(&amt->key, sizeof(siphash_key_t)); amt->status = AMT_STATUS_INIT; @@ -2892,6 +3016,8 @@ static int amt_dev_stop(struct net_device *dev) struct amt_dev *amt = netdev_priv(dev); struct amt_tunnel_list *tunnel, *tmp; struct socket *sock; + struct sk_buff *skb; + int i; cancel_delayed_work_sync(&amt->req_wq); cancel_delayed_work_sync(&amt->discovery_wq); @@ -2904,6 +3030,15 @@ static int amt_dev_stop(struct net_device *dev) if (sock) udp_tunnel_sock_release(sock); + cancel_work_sync(&amt->event_wq); + for (i = 0; i < AMT_MAX_EVENTS; i++) { + skb = amt->events[i].skb; + if (skb) + kfree_skb(skb); + amt->events[i].event = AMT_EVENT_NONE; + amt->events[i].skb = NULL; + } + amt->ready4 = false; amt->ready6 = false; amt->req_cnt = 0; @@ -3095,7 +3230,7 @@ static int amt_newlink(struct net *net, struct net_device *dev, goto err; } if (amt->mode == AMT_MODE_RELAY) { - amt->qrv = amt->net->ipv4.sysctl_igmp_qrv; + amt->qrv = READ_ONCE(amt->net->ipv4.sysctl_igmp_qrv); amt->qri = 10; dev->needed_headroom = amt->stream_dev->needed_headroom + AMT_RELAY_HLEN; @@ -3146,8 +3281,8 @@ static int amt_newlink(struct net *net, struct net_device *dev, INIT_DELAYED_WORK(&amt->discovery_wq, amt_discovery_work); INIT_DELAYED_WORK(&amt->req_wq, amt_req_work); INIT_DELAYED_WORK(&amt->secret_wq, amt_secret_work); + INIT_WORK(&amt->event_wq, amt_event_work); INIT_LIST_HEAD(&amt->tunnel_list); - return 0; err: dev_put(amt->stream_dev); @@ -3280,7 +3415,7 @@ static int __init amt_init(void) if (err < 0) goto unregister_notifier; - amt_wq = alloc_workqueue("amt", WQ_UNBOUND, 1); + amt_wq = alloc_workqueue("amt", WQ_UNBOUND, 0); if (!amt_wq) { err = -ENOMEM; goto rtnl_unregister; diff --git a/drivers/net/can/c_can/c_can_main.c b/drivers/net/can/c_can/c_can_main.c index a7362af0babb..de38d8f7b5f7 100644 --- a/drivers/net/can/c_can/c_can_main.c +++ b/drivers/net/can/c_can/c_can_main.c @@ -952,15 +952,14 @@ static int c_can_handle_state_change(struct net_device *dev, switch (error_type) { case C_CAN_NO_ERROR: - /* error warning state */ - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = CAN_ERR_CRTL_ACTIVE; cf->data[6] = bec.txerr; cf->data[7] = bec.rxerr; break; case C_CAN_ERROR_WARNING: /* error warning state */ - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = (bec.txerr > bec.rxerr) ? CAN_ERR_CRTL_TX_WARNING : CAN_ERR_CRTL_RX_WARNING; @@ -970,7 +969,7 @@ static int c_can_handle_state_change(struct net_device *dev, break; case C_CAN_ERROR_PASSIVE: /* error passive state */ - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; if (rx_err_passive) cf->data[1] |= CAN_ERR_CRTL_RX_PASSIVE; if (bec.txerr > 127) diff --git a/drivers/net/can/cc770/cc770.c b/drivers/net/can/cc770/cc770.c index bb7224cfc6ab..797a954bb1a0 100644 --- a/drivers/net/can/cc770/cc770.c +++ b/drivers/net/can/cc770/cc770.c @@ -512,6 +512,7 @@ static int cc770_err(struct net_device *dev, u8 status) /* Use extended functions of the CC770 */ if (priv->control_normal_mode & CTRL_EAF) { + cf->can_id |= CAN_ERR_CNT; cf->data[6] = cc770_read_reg(priv, tx_error_counter); cf->data[7] = cc770_read_reg(priv, rx_error_counter); } diff --git a/drivers/net/can/ctucanfd/ctucanfd_base.c b/drivers/net/can/ctucanfd/ctucanfd_base.c index 14ac7c0ee04c..6b281f6eb9b4 100644 --- a/drivers/net/can/ctucanfd/ctucanfd_base.c +++ b/drivers/net/can/ctucanfd/ctucanfd_base.c @@ -847,7 +847,7 @@ static void ctucan_err_interrupt(struct net_device *ndev, u32 isr) case CAN_STATE_ERROR_PASSIVE: priv->can.can_stats.error_passive++; if (skb) { - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = (bec.rxerr > 127) ? CAN_ERR_CRTL_RX_PASSIVE : CAN_ERR_CRTL_TX_PASSIVE; @@ -858,7 +858,7 @@ static void ctucan_err_interrupt(struct net_device *ndev, u32 isr) case CAN_STATE_ERROR_WARNING: priv->can.can_stats.error_warning++; if (skb) { - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] |= (bec.txerr > bec.rxerr) ? CAN_ERR_CRTL_TX_WARNING : CAN_ERR_CRTL_RX_WARNING; @@ -867,6 +867,7 @@ static void ctucan_err_interrupt(struct net_device *ndev, u32 isr) } break; case CAN_STATE_ERROR_ACTIVE: + cf->can_id |= CAN_ERR_CNT; cf->data[1] = CAN_ERR_CRTL_ACTIVE; cf->data[6] = bec.txerr; cf->data[7] = bec.rxerr; diff --git a/drivers/net/can/ctucanfd/ctucanfd_kregs.h b/drivers/net/can/ctucanfd/ctucanfd_kregs.h index edc1c1a24348..0c181ab51bf8 100644 --- a/drivers/net/can/ctucanfd/ctucanfd_kregs.h +++ b/drivers/net/can/ctucanfd/ctucanfd_kregs.h @@ -4,9 +4,9 @@ * CTU CAN FD IP Core * * Copyright (C) 2015-2018 Ondrej Ille <ondrej.ille@gmail.com> FEE CTU - * Copyright (C) 2018-2021 Ondrej Ille <ondrej.ille@gmail.com> self-funded + * Copyright (C) 2018-2022 Ondrej Ille <ondrej.ille@gmail.com> self-funded * Copyright (C) 2018-2019 Martin Jerabek <martin.jerabek01@gmail.com> FEE CTU - * Copyright (C) 2018-2021 Pavel Pisa <pisa@cmp.felk.cvut.cz> FEE CTU/self-funded + * Copyright (C) 2018-2022 Pavel Pisa <pisa@cmp.felk.cvut.cz> FEE CTU/self-funded * * Project advisors: * Jiri Novak <jnovak@fel.cvut.cz> @@ -64,9 +64,12 @@ enum ctu_can_fd_can_registers { CTUCANFD_RX_DATA = 0x6c, CTUCANFD_TX_STATUS = 0x70, CTUCANFD_TX_COMMAND = 0x74, + CTUCANFD_TXTB_INFO = 0x76, CTUCANFD_TX_PRIORITY = 0x78, CTUCANFD_ERR_CAPT = 0x7c, + CTUCANFD_RETR_CTR = 0x7d, CTUCANFD_ALC = 0x7e, + CTUCANFD_TS_INFO = 0x7f, CTUCANFD_TRV_DELAY = 0x80, CTUCANFD_SSP_CFG = 0x82, CTUCANFD_RX_FR_CTR = 0x84, @@ -102,8 +105,12 @@ enum ctu_can_fd_can_registers { #define REG_MODE_STM BIT(2) #define REG_MODE_AFM BIT(3) #define REG_MODE_FDE BIT(4) +#define REG_MODE_TTTM BIT(5) +#define REG_MODE_ROM BIT(6) #define REG_MODE_ACF BIT(7) #define REG_MODE_TSTM BIT(8) +#define REG_MODE_RXBAM BIT(9) +#define REG_MODE_SAM BIT(11) #define REG_MODE_RTRLE BIT(16) #define REG_MODE_RTRTH GENMASK(20, 17) #define REG_MODE_ILBP BIT(21) @@ -123,8 +130,10 @@ enum ctu_can_fd_can_registers { #define REG_STATUS_EWL BIT(6) #define REG_STATUS_IDLE BIT(7) #define REG_STATUS_PEXS BIT(8) +#define REG_STATUS_STCNT BIT(16) /* COMMAND registers */ +#define REG_COMMAND_RXRPMV BIT(1) #define REG_COMMAND_RRB BIT(2) #define REG_COMMAND_CDO BIT(3) #define REG_COMMAND_ERCRST BIT(4) @@ -263,8 +272,12 @@ enum ctu_can_fd_can_registers { #define REG_TX_STATUS_TX2S GENMASK(7, 4) #define REG_TX_STATUS_TX3S GENMASK(11, 8) #define REG_TX_STATUS_TX4S GENMASK(15, 12) +#define REG_TX_STATUS_TX5S GENMASK(19, 16) +#define REG_TX_STATUS_TX6S GENMASK(23, 20) +#define REG_TX_STATUS_TX7S GENMASK(27, 24) +#define REG_TX_STATUS_TX8S GENMASK(31, 28) -/* TX_COMMAND registers */ +/* TX_COMMAND TXTB_INFO registers */ #define REG_TX_COMMAND_TXCE BIT(0) #define REG_TX_COMMAND_TXCR BIT(1) #define REG_TX_COMMAND_TXCA BIT(2) @@ -272,18 +285,29 @@ enum ctu_can_fd_can_registers { #define REG_TX_COMMAND_TXB2 BIT(9) #define REG_TX_COMMAND_TXB3 BIT(10) #define REG_TX_COMMAND_TXB4 BIT(11) +#define REG_TX_COMMAND_TXB5 BIT(12) +#define REG_TX_COMMAND_TXB6 BIT(13) +#define REG_TX_COMMAND_TXB7 BIT(14) +#define REG_TX_COMMAND_TXB8 BIT(15) +#define REG_TX_COMMAND_TXT_BUFFER_COUNT GENMASK(19, 16) /* TX_PRIORITY registers */ #define REG_TX_PRIORITY_TXT1P GENMASK(2, 0) #define REG_TX_PRIORITY_TXT2P GENMASK(6, 4) #define REG_TX_PRIORITY_TXT3P GENMASK(10, 8) #define REG_TX_PRIORITY_TXT4P GENMASK(14, 12) +#define REG_TX_PRIORITY_TXT5P GENMASK(18, 16) +#define REG_TX_PRIORITY_TXT6P GENMASK(22, 20) +#define REG_TX_PRIORITY_TXT7P GENMASK(26, 24) +#define REG_TX_PRIORITY_TXT8P GENMASK(30, 28) -/* ERR_CAPT ALC registers */ +/* ERR_CAPT RETR_CTR ALC TS_INFO registers */ #define REG_ERR_CAPT_ERR_POS GENMASK(4, 0) #define REG_ERR_CAPT_ERR_TYPE GENMASK(7, 5) +#define REG_ERR_CAPT_RETR_CTR_VAL GENMASK(11, 8) #define REG_ERR_CAPT_ALC_BIT GENMASK(20, 16) #define REG_ERR_CAPT_ALC_ID_FIELD GENMASK(23, 21) +#define REG_ERR_CAPT_TS_BITS GENMASK(29, 24) /* TRV_DELAY SSP_CFG registers */ #define REG_TRV_DELAY_TRV_DELAY_VALUE GENMASK(6, 0) diff --git a/drivers/net/can/grcan.c b/drivers/net/can/grcan.c index 4c47c1055eff..24035a6187c9 100644 --- a/drivers/net/can/grcan.c +++ b/drivers/net/can/grcan.c @@ -671,6 +671,7 @@ static void grcan_err(struct net_device *dev, u32 sources, u32 status) /* There are no others at this point */ break; } + cf.can_id |= CAN_ERR_CNT; cf.data[6] = txerr; cf.data[7] = rxerr; priv->can.state = state; diff --git a/drivers/net/can/ifi_canfd/ifi_canfd.c b/drivers/net/can/ifi_canfd/ifi_canfd.c index 968ed6d7316b..64e3be8b73af 100644 --- a/drivers/net/can/ifi_canfd/ifi_canfd.c +++ b/drivers/net/can/ifi_canfd/ifi_canfd.c @@ -492,7 +492,7 @@ static int ifi_canfd_handle_state_change(struct net_device *ndev, switch (new_state) { case CAN_STATE_ERROR_WARNING: /* error warning state */ - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = (bec.txerr > bec.rxerr) ? CAN_ERR_CRTL_TX_WARNING : CAN_ERR_CRTL_RX_WARNING; @@ -501,7 +501,7 @@ static int ifi_canfd_handle_state_change(struct net_device *ndev, break; case CAN_STATE_ERROR_PASSIVE: /* error passive state */ - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] |= CAN_ERR_CRTL_RX_PASSIVE; if (bec.txerr > 127) cf->data[1] |= CAN_ERR_CRTL_TX_PASSIVE; diff --git a/drivers/net/can/janz-ican3.c b/drivers/net/can/janz-ican3.c index 35bfb82d6929..ccb5c5405224 100644 --- a/drivers/net/can/janz-ican3.c +++ b/drivers/net/can/janz-ican3.c @@ -1127,7 +1127,7 @@ static int ican3_handle_cevtind(struct ican3_dev *mod, struct ican3_msg *msg) /* bus error interrupt */ if (isrc == CEVTIND_BEI) { mod->can.can_stats.bus_error++; - cf->can_id |= CAN_ERR_PROT | CAN_ERR_BUSERROR; + cf->can_id |= CAN_ERR_PROT | CAN_ERR_BUSERROR | CAN_ERR_CNT; switch (ecc & ECC_MASK) { case ECC_BIT: @@ -1153,7 +1153,7 @@ static int ican3_handle_cevtind(struct ican3_dev *mod, struct ican3_msg *msg) if (state != mod->can.state && (state == CAN_STATE_ERROR_WARNING || state == CAN_STATE_ERROR_PASSIVE)) { - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; if (state == CAN_STATE_ERROR_WARNING) { mod->can.can_stats.error_warning++; cf->data[1] = (txerr > rxerr) ? diff --git a/drivers/net/can/kvaser_pciefd.c b/drivers/net/can/kvaser_pciefd.c index 017f2d36ffc3..dcd2c9d50d5e 100644 --- a/drivers/net/can/kvaser_pciefd.c +++ b/drivers/net/can/kvaser_pciefd.c @@ -1306,7 +1306,7 @@ static int kvaser_pciefd_rx_error_frame(struct kvaser_pciefd_can *can, shhwtstamps->hwtstamp = ns_to_ktime(div_u64(p->timestamp * 1000, can->kv_pcie->freq_to_ticks_div)); - cf->can_id |= CAN_ERR_BUSERROR; + cf->can_id |= CAN_ERR_BUSERROR | CAN_ERR_CNT; cf->data[6] = bec.txerr; cf->data[7] = bec.rxerr; diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index afaaeb610c00..713a4b0edf86 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -741,7 +741,7 @@ static int m_can_handle_state_change(struct net_device *dev, switch (new_state) { case CAN_STATE_ERROR_WARNING: /* error warning state */ - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = (bec.txerr > bec.rxerr) ? CAN_ERR_CRTL_TX_WARNING : CAN_ERR_CRTL_RX_WARNING; @@ -750,7 +750,7 @@ static int m_can_handle_state_change(struct net_device *dev, break; case CAN_STATE_ERROR_PASSIVE: /* error passive state */ - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; ecr = m_can_read(cdev, M_CAN_ECR); if (ecr & ECR_RP) cf->data[1] |= CAN_ERR_CRTL_RX_PASSIVE; diff --git a/drivers/net/can/pch_can.c b/drivers/net/can/pch_can.c index fde3ac516d26..32804fed116c 100644 --- a/drivers/net/can/pch_can.c +++ b/drivers/net/can/pch_can.c @@ -489,6 +489,7 @@ static void pch_can_error(struct net_device *ndev, u32 status) if (!skb) return; + errc = ioread32(&priv->regs->errc); if (status & PCH_BUS_OFF) { pch_can_set_tx_all(priv, 0); pch_can_set_rx_all(priv, 0); @@ -496,9 +497,12 @@ static void pch_can_error(struct net_device *ndev, u32 status) cf->can_id |= CAN_ERR_BUSOFF; priv->can.can_stats.bus_off++; can_bus_off(ndev); + } else { + cf->can_id |= CAN_ERR_CNT; + cf->data[6] = errc & PCH_TEC; + cf->data[7] = (errc & PCH_REC) >> 8; } - errc = ioread32(&priv->regs->errc); /* Warning interrupt. */ if (status & PCH_EWARN) { state = CAN_STATE_ERROR_WARNING; @@ -556,9 +560,6 @@ static void pch_can_error(struct net_device *ndev, u32 status) break; } - cf->data[6] = errc & PCH_TEC; - cf->data[7] = (errc & PCH_REC) >> 8; - priv->can.state = state; netif_receive_skb(skb); } diff --git a/drivers/net/can/peak_canfd/peak_canfd.c b/drivers/net/can/peak_canfd/peak_canfd.c index b2dea360813d..afb9adb3d5c2 100644 --- a/drivers/net/can/peak_canfd/peak_canfd.c +++ b/drivers/net/can/peak_canfd/peak_canfd.c @@ -373,7 +373,7 @@ static int pucan_handle_status(struct peak_canfd_priv *priv, priv->can.state = CAN_STATE_ERROR_PASSIVE; priv->can.can_stats.error_passive++; if (skb) { - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = (priv->bec.txerr > priv->bec.rxerr) ? CAN_ERR_CRTL_TX_PASSIVE : CAN_ERR_CRTL_RX_PASSIVE; @@ -386,7 +386,7 @@ static int pucan_handle_status(struct peak_canfd_priv *priv, priv->can.state = CAN_STATE_ERROR_WARNING; priv->can.can_stats.error_warning++; if (skb) { - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = (priv->bec.txerr > priv->bec.rxerr) ? CAN_ERR_CRTL_TX_WARNING : CAN_ERR_CRTL_RX_WARNING; @@ -430,7 +430,7 @@ static int pucan_handle_cache_critical(struct peak_canfd_priv *priv) return -ENOMEM; } - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = CAN_ERR_CRTL_RX_OVERFLOW; cf->data[6] = priv->bec.txerr; diff --git a/drivers/net/can/rcar/rcar_can.c b/drivers/net/can/rcar/rcar_can.c index d45762f1cf6b..d11db2112a4a 100644 --- a/drivers/net/can/rcar/rcar_can.c +++ b/drivers/net/can/rcar/rcar_can.c @@ -232,11 +232,8 @@ static void rcar_can_error(struct net_device *ndev) if (eifr & (RCAR_CAN_EIFR_EWIF | RCAR_CAN_EIFR_EPIF)) { txerr = readb(&priv->regs->tecr); rxerr = readb(&priv->regs->recr); - if (skb) { + if (skb) cf->can_id |= CAN_ERR_CRTL; - cf->data[6] = txerr; - cf->data[7] = rxerr; - } } if (eifr & RCAR_CAN_EIFR_BEIF) { int rx_errors = 0, tx_errors = 0; @@ -336,6 +333,10 @@ static void rcar_can_error(struct net_device *ndev) can_bus_off(ndev); if (skb) cf->can_id |= CAN_ERR_BUSOFF; + } else if (skb) { + cf->can_id |= CAN_ERR_CNT; + cf->data[6] = txerr; + cf->data[7] = rxerr; } if (eifr & RCAR_CAN_EIFR_ORIF) { netdev_dbg(priv->ndev, "Receive overrun error interrupt\n"); diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index ba42cef10a53..d3e569a02b4d 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -1052,7 +1052,7 @@ static void rcar_canfd_error(struct net_device *ndev, u32 cerfl, netdev_dbg(ndev, "Error warning interrupt\n"); priv->can.state = CAN_STATE_ERROR_WARNING; priv->can.can_stats.error_warning++; - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = txerr > rxerr ? CAN_ERR_CRTL_TX_WARNING : CAN_ERR_CRTL_RX_WARNING; cf->data[6] = txerr; @@ -1062,7 +1062,7 @@ static void rcar_canfd_error(struct net_device *ndev, u32 cerfl, netdev_dbg(ndev, "Error passive interrupt\n"); priv->can.state = CAN_STATE_ERROR_PASSIVE; priv->can.can_stats.error_passive++; - cf->can_id |= CAN_ERR_CRTL; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = txerr > rxerr ? CAN_ERR_CRTL_TX_PASSIVE : CAN_ERR_CRTL_RX_PASSIVE; cf->data[6] = txerr; @@ -1843,6 +1843,7 @@ static int rcar_canfd_probe(struct platform_device *pdev) of_child = of_get_child_by_name(pdev->dev.of_node, name); if (of_child && of_device_is_available(of_child)) channels_mask |= BIT(i); + of_node_put(of_child); } if (chip_id != RENESAS_RZG2L) { diff --git a/drivers/net/can/sja1000/sja1000.c b/drivers/net/can/sja1000/sja1000.c index 2e7638f98cf1..75a2f9bf8c16 100644 --- a/drivers/net/can/sja1000/sja1000.c +++ b/drivers/net/can/sja1000/sja1000.c @@ -183,8 +183,9 @@ static void chipset_init(struct net_device *dev) { struct sja1000_priv *priv = netdev_priv(dev); - /* set clock divider and output control register */ - priv->write_reg(priv, SJA1000_CDR, priv->cdr | CDR_PELICAN); + if (!(priv->flags & SJA1000_QUIRK_NO_CDR_REG)) + /* set clock divider and output control register */ + priv->write_reg(priv, SJA1000_CDR, priv->cdr | CDR_PELICAN); /* set acceptance filter (accept all) */ priv->write_reg(priv, SJA1000_ACCC0, 0x00); @@ -209,7 +210,8 @@ static void sja1000_start(struct net_device *dev) set_reset_mode(dev); /* Initialize chip if uninitialized at this stage */ - if (!(priv->read_reg(priv, SJA1000_CDR) & CDR_PELICAN)) + if (!(priv->flags & SJA1000_QUIRK_NO_CDR_REG || + priv->read_reg(priv, SJA1000_CDR) & CDR_PELICAN)) chipset_init(dev); /* Clear error counters and error code capture */ @@ -402,9 +404,6 @@ static int sja1000_err(struct net_device *dev, uint8_t isrc, uint8_t status) txerr = priv->read_reg(priv, SJA1000_TXERR); rxerr = priv->read_reg(priv, SJA1000_RXERR); - cf->data[6] = txerr; - cf->data[7] = rxerr; - if (isrc & IRQ_DOI) { /* data overrun interrupt */ netdev_dbg(dev, "data overrun interrupt\n"); @@ -426,6 +425,11 @@ static int sja1000_err(struct net_device *dev, uint8_t isrc, uint8_t status) else state = CAN_STATE_ERROR_ACTIVE; } + if (state != CAN_STATE_BUS_OFF) { + cf->can_id |= CAN_ERR_CNT; + cf->data[6] = txerr; + cf->data[7] = rxerr; + } if (isrc & IRQ_BEI) { /* bus error interrupt */ priv->can.can_stats.bus_error++; diff --git a/drivers/net/can/sja1000/sja1000.h b/drivers/net/can/sja1000/sja1000.h index 9d46398f8154..7f736f1df547 100644 --- a/drivers/net/can/sja1000/sja1000.h +++ b/drivers/net/can/sja1000/sja1000.h @@ -145,7 +145,8 @@ /* * Flags for sja1000priv.flags */ -#define SJA1000_CUSTOM_IRQ_HANDLER 0x1 +#define SJA1000_CUSTOM_IRQ_HANDLER BIT(0) +#define SJA1000_QUIRK_NO_CDR_REG BIT(1) /* * SJA1000 private data structure diff --git a/drivers/net/can/sja1000/sja1000_platform.c b/drivers/net/can/sja1000/sja1000_platform.c index f9ec7bd8dfac..81bc741905fd 100644 --- a/drivers/net/can/sja1000/sja1000_platform.c +++ b/drivers/net/can/sja1000/sja1000_platform.c @@ -31,7 +31,7 @@ MODULE_LICENSE("GPL v2"); struct sja1000_of_data { size_t priv_sz; - int (*init)(struct sja1000_priv *priv, struct device_node *of); + void (*init)(struct sja1000_priv *priv, struct device_node *of); }; struct technologic_priv { @@ -94,15 +94,13 @@ static void sp_technologic_write_reg16(const struct sja1000_priv *priv, spin_unlock_irqrestore(&tp->io_lock, flags); } -static int sp_technologic_init(struct sja1000_priv *priv, struct device_node *of) +static void sp_technologic_init(struct sja1000_priv *priv, struct device_node *of) { struct technologic_priv *tp = priv->priv; priv->read_reg = sp_technologic_read_reg16; priv->write_reg = sp_technologic_write_reg16; spin_lock_init(&tp->io_lock); - - return 0; } static void sp_populate(struct sja1000_priv *priv, @@ -210,7 +208,6 @@ static int sp_probe(struct platform_device *pdev) struct resource *res_mem, *res_irq = NULL; struct sja1000_platform_data *pdata; struct device_node *of = pdev->dev.of_node; - const struct of_device_id *of_id; const struct sja1000_of_data *of_data = NULL; size_t priv_sz = 0; @@ -243,11 +240,9 @@ static int sp_probe(struct platform_device *pdev) return -ENODEV; } - of_id = of_match_device(sp_of_table, &pdev->dev); - if (of_id && of_id->data) { - of_data = of_id->data; + of_data = device_get_match_data(&pdev->dev); + if (of_data) priv_sz = of_data->priv_sz; - } dev = alloc_sja1000dev(priv_sz); if (!dev) @@ -269,11 +264,8 @@ static int sp_probe(struct platform_device *pdev) if (of) { sp_populate_of(priv, of); - if (of_data && of_data->init) { - err = of_data->init(priv, of); - if (err) - goto exit_free; - } + if (of_data && of_data->init) + of_data->init(priv, of); } else { sp_populate(priv, pdata, res_mem->flags); } diff --git a/drivers/net/can/slcan/slcan-core.c b/drivers/net/can/slcan/slcan-core.c index 54d29a410ad5..dc28e715bbe1 100644 --- a/drivers/net/can/slcan/slcan-core.c +++ b/drivers/net/can/slcan/slcan-core.c @@ -67,13 +67,14 @@ MODULE_AUTHOR("Oliver Hartkopp <socketcan@hartkopp.net>"); #define SLCAN_MAGIC 0x53CA static int maxdev = 10; /* MAX number of SLCAN channels; - This can be overridden with - insmod slcan.ko maxdev=nnn */ + * This can be overridden with + * insmod slcan.ko maxdev=nnn + */ module_param(maxdev, int, 0); MODULE_PARM_DESC(maxdev, "Maximum number of slcan interfaces"); /* maximum rx buffer len: extended CAN frame with timestamp */ -#define SLC_MTU (sizeof("T1111222281122334455667788EA5F\r")+1) +#define SLC_MTU (sizeof("T1111222281122334455667788EA5F\r") + 1) #define SLC_CMD_LEN 1 #define SLC_SFF_ID_LEN 3 @@ -139,12 +140,11 @@ int slcan_enable_err_rst_on_open(struct net_device *ndev, bool on) return 0; } - /************************************************************************ - * SLCAN ENCAPSULATION FORMAT * - ************************************************************************/ +/************************************************************************* + * SLCAN ENCAPSULATION FORMAT * + *************************************************************************/ -/* - * A CAN frame has a can_id (11 bit standard frame format OR 29 bit extended +/* A CAN frame has a can_id (11 bit standard frame format OR 29 bit extended * frame format) a data length code (len) which can be from 0 to 8 * and up to <len> data bytes as payload. * Additionally a CAN frame may become a remote transmission frame if the @@ -174,9 +174,9 @@ int slcan_enable_err_rst_on_open(struct net_device *ndev, bool on) * */ - /************************************************************************ - * STANDARD SLCAN DECAPSULATION * - ************************************************************************/ +/************************************************************************* + * STANDARD SLCAN DECAPSULATION * + *************************************************************************/ /* Send one completely decapsulated can_frame to the network layer */ static void slc_bump_frame(struct slcan *sl) @@ -306,19 +306,18 @@ static void slc_bump_state(struct slcan *sl) return; skb = alloc_can_err_skb(dev, &cf); - if (skb) { - cf->data[6] = txerr; - cf->data[7] = rxerr; - } else { - cf = NULL; - } tx_state = txerr >= rxerr ? state : 0; rx_state = txerr <= rxerr ? state : 0; can_change_state(dev, cf, tx_state, rx_state); - if (state == CAN_STATE_BUS_OFF) + if (state == CAN_STATE_BUS_OFF) { can_bus_off(dev); + } else if (skb) { + cf->can_id |= CAN_ERR_CNT; + cf->data[6] = txerr; + cf->data[7] = rxerr; + } if (skb) netif_rx(skb); @@ -469,26 +468,26 @@ static void slcan_unesc(struct slcan *sl, unsigned char s) { if ((s == '\r') || (s == '\a')) { /* CR or BEL ends the pdu */ if (!test_and_clear_bit(SLF_ERROR, &sl->flags) && - (sl->rcount > 4)) { + sl->rcount > 4) slc_bump(sl); - } + sl->rcount = 0; } else { if (!test_bit(SLF_ERROR, &sl->flags)) { if (sl->rcount < SLC_MTU) { sl->rbuff[sl->rcount++] = s; return; - } else { - sl->dev->stats.rx_over_errors++; - set_bit(SLF_ERROR, &sl->flags); } + + sl->dev->stats.rx_over_errors++; + set_bit(SLF_ERROR, &sl->flags); } } } - /************************************************************************ - * STANDARD SLCAN ENCAPSULATION * - ************************************************************************/ +/************************************************************************* + * STANDARD SLCAN ENCAPSULATION * + *************************************************************************/ /* Encapsulate one can_frame and stuff into a TTY queue. */ static void slc_encaps(struct slcan *sl, struct can_frame *cf) @@ -575,7 +574,8 @@ static void slcan_transmit(struct work_struct *work) } /* Now serial buffer is almost free & we can start - * transmission of another packet */ + * transmission of another packet + */ sl->dev->stats.tx_packets++; clear_bit(TTY_DO_WRITE_WAKEUP, &sl->tty->flags); spin_unlock_bh(&sl->lock); @@ -589,8 +589,7 @@ static void slcan_transmit(struct work_struct *work) spin_unlock_bh(&sl->lock); } -/* - * Called by the driver when there's room for more data. +/* Called by the driver when there's room for more data. * Schedule the transmit. */ static void slcan_write_wakeup(struct tty_struct *tty) @@ -618,13 +617,13 @@ static netdev_tx_t slc_xmit(struct sk_buff *skb, struct net_device *dev) netdev_warn(dev, "xmit: iface is down\n"); goto out; } - if (sl->tty == NULL) { + if (!sl->tty) { spin_unlock(&sl->lock); goto out; } netif_stop_queue(sl->dev); - slc_encaps(sl, (struct can_frame *) skb->data); /* encaps & send */ + slc_encaps(sl, (struct can_frame *)skb->data); /* encaps & send */ spin_unlock(&sl->lock); out: @@ -632,7 +631,6 @@ out: return NETDEV_TX_OK; } - /****************************************** * Routines looking at netdevice side. ******************************************/ @@ -647,7 +645,7 @@ static int slcan_transmit_cmd(struct slcan *sl, const unsigned char *cmd) return -ENODEV; } - n = snprintf(sl->xbuff, sizeof(sl->xbuff), "%s", cmd); + n = scnprintf(sl->xbuff, sizeof(sl->xbuff), "%s", cmd); set_bit(TTY_DO_WRITE_WAKEUP, &sl->tty->flags); actual = sl->tty->ops->write(sl->tty, sl->xbuff, n); sl->xleft = n - actual; @@ -689,15 +687,14 @@ static int slc_close(struct net_device *dev) clear_bit(TTY_DO_WRITE_WAKEUP, &sl->tty->flags); } netif_stop_queue(dev); + sl->rcount = 0; + sl->xleft = 0; + spin_unlock_bh(&sl->lock); close_candev(dev); sl->can.state = CAN_STATE_STOPPED; if (sl->can.bittiming.bitrate == CAN_BITRATE_UNKNOWN) sl->can.bittiming.bitrate = CAN_BITRATE_UNSET; - sl->rcount = 0; - sl->xleft = 0; - spin_unlock_bh(&sl->lock); - return 0; } @@ -708,7 +705,7 @@ static int slc_open(struct net_device *dev) unsigned char cmd[SLC_MTU]; int err, s; - if (sl->tty == NULL) + if (!sl->tty) return -ENODEV; /* The baud rate is not set with the command @@ -736,7 +733,6 @@ static int slc_open(struct net_device *dev) /* The CAN framework has already validate the bitrate value, * so we can avoid to check if `s' has been properly set. */ - snprintf(cmd, sizeof(cmd), "C\rS%d\r", s); err = slcan_transmit_cmd(sl, cmd); if (err) { @@ -792,23 +788,21 @@ static const struct net_device_ops slc_netdev_ops = { }; /****************************************** - Routines looking at TTY side. + * Routines looking at TTY side. ******************************************/ -/* - * Handle the 'receiver data ready' interrupt. +/* Handle the 'receiver data ready' interrupt. * This function is called by the 'tty_io' module in the kernel when * a block of SLCAN data has been received, which can now be decapsulated * and sent on to some IP layer for further processing. This will not * be re-entered while running but other ldisc functions may be called * in parallel */ - static void slcan_receive_buf(struct tty_struct *tty, const unsigned char *cp, const char *fp, int count) { - struct slcan *sl = (struct slcan *) tty->disc_data; + struct slcan *sl = (struct slcan *)tty->disc_data; if (!sl || sl->magic != SLCAN_MAGIC || !netif_running(sl->dev)) return; @@ -838,7 +832,7 @@ static void slc_sync(void) for (i = 0; i < maxdev; i++) { dev = slcan_devs[i]; - if (dev == NULL) + if (!dev) break; sl = netdev_priv(dev); @@ -858,9 +852,8 @@ static struct slcan *slc_alloc(void) for (i = 0; i < maxdev; i++) { dev = slcan_devs[i]; - if (dev == NULL) + if (!dev) break; - } /* Sorry, too many, all slots in use */ @@ -890,8 +883,7 @@ static struct slcan *slc_alloc(void) return sl; } -/* - * Open the high-level part of the SLCAN channel. +/* Open the high-level part of the SLCAN channel. * This function is called by the TTY module when the * SLCAN line discipline is called for. Because we are * sure the tty line exists, we only have to link it to @@ -899,7 +891,6 @@ static struct slcan *slc_alloc(void) * * Called in process context serialized from other ldisc calls. */ - static int slcan_open(struct tty_struct *tty) { struct slcan *sl; @@ -908,12 +899,12 @@ static int slcan_open(struct tty_struct *tty) if (!capable(CAP_NET_ADMIN)) return -EPERM; - if (tty->ops->write == NULL) + if (!tty->ops->write) return -EOPNOTSUPP; /* RTnetlink lock is misused here to serialize concurrent - opens of slcan channels. There are better ways, but it is - the simplest one. + * opens of slcan channels. There are better ways, but it is + * the simplest one. */ rtnl_lock(); @@ -930,7 +921,7 @@ static int slcan_open(struct tty_struct *tty) /* OK. Find a free SLCAN channel to use. */ err = -ENFILE; sl = slc_alloc(); - if (sl == NULL) + if (!sl) goto err_exit; sl->tty = tty; @@ -974,17 +965,15 @@ err_exit: return err; } -/* - * Close down a SLCAN channel. +/* Close down a SLCAN channel. * This means flushing out any pending queues, and then returning. This * call is serialized against other ldisc functions. * * We also use this method for a hangup event. */ - static void slcan_close(struct tty_struct *tty) { - struct slcan *sl = (struct slcan *) tty->disc_data; + struct slcan *sl = (struct slcan *)tty->disc_data; /* First make sure we're connected. */ if (!sl || sl->magic != SLCAN_MAGIC || sl->tty != tty) @@ -1014,7 +1003,7 @@ static void slcan_hangup(struct tty_struct *tty) static int slcan_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { - struct slcan *sl = (struct slcan *) tty->disc_data; + struct slcan *sl = (struct slcan *)tty->disc_data; unsigned int tmp; /* First make sure we're connected. */ @@ -1065,7 +1054,7 @@ static int __init slcan_init(void) /* Fill in our line protocol discipline, and register it */ status = tty_register_ldisc(&slc_ldisc); if (status) { - printk(KERN_ERR "slcan: can't register line discipline\n"); + pr_err("slcan: can't register line discipline\n"); kfree(slcan_devs); } return status; @@ -1079,7 +1068,7 @@ static void __exit slcan_exit(void) unsigned long timeout = jiffies + HZ; int busy = 0; - if (slcan_devs == NULL) + if (!slcan_devs) return; /* First of all: check for active disciplines and hangup them. @@ -1104,7 +1093,8 @@ static void __exit slcan_exit(void) } while (busy && time_before(jiffies, timeout)); /* FIXME: hangup is async so we should wait when doing this second - phase */ + * phase + */ for (i = 0; i < maxdev; i++) { dev = slcan_devs[i]; @@ -1112,9 +1102,8 @@ static void __exit slcan_exit(void) continue; sl = netdev_priv(dev); - if (sl->tty) { + if (sl->tty) netdev_err(dev, "tty discipline still running\n"); - } slc_close(dev); unregister_candev(dev); diff --git a/drivers/net/can/spi/hi311x.c b/drivers/net/can/spi/hi311x.c index ebc4ebb44c98..167114aae6dd 100644 --- a/drivers/net/can/spi/hi311x.c +++ b/drivers/net/can/spi/hi311x.c @@ -667,8 +667,6 @@ static irqreturn_t hi3110_can_ist(int irq, void *dev_id) txerr = hi3110_read(spi, HI3110_READ_TEC); rxerr = hi3110_read(spi, HI3110_READ_REC); - cf->data[6] = txerr; - cf->data[7] = rxerr; tx_state = txerr >= rxerr ? new_state : 0; rx_state = txerr <= rxerr ? new_state : 0; can_change_state(net, cf, tx_state, rx_state); @@ -681,6 +679,10 @@ static irqreturn_t hi3110_can_ist(int irq, void *dev_id) hi3110_hw_sleep(spi); break; } + } else { + cf->can_id |= CAN_ERR_CNT; + cf->data[6] = txerr; + cf->data[7] = rxerr; } } diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c index 9b47b07162fe..7fc86ed405c6 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c @@ -1099,6 +1099,7 @@ static int mcp251xfd_handle_cerrif(struct mcp251xfd_priv *priv) err = mcp251xfd_get_berr_counter(priv->ndev, &bec); if (err) return err; + cf->can_id |= CAN_ERR_CNT; cf->data[6] = bec.txerr; cf->data[7] = bec.rxerr; } @@ -1690,8 +1691,8 @@ static int mcp251xfd_register_chip_detect(struct mcp251xfd_priv *priv) u32 osc; int err; - /* The OSC_LPMEN is only supported on MCP2518FD, so use it to - * autodetect the model. + /* The OSC_LPMEN is only supported on MCP2518FD and MCP251863, + * so use it to autodetect the model. */ err = regmap_update_bits(priv->map_reg, MCP251XFD_REG_OSC, MCP251XFD_REG_OSC_LPMEN, @@ -1703,10 +1704,18 @@ static int mcp251xfd_register_chip_detect(struct mcp251xfd_priv *priv) if (err) return err; - if (osc & MCP251XFD_REG_OSC_LPMEN) - devtype_data = &mcp251xfd_devtype_data_mcp2518fd; - else + if (osc & MCP251XFD_REG_OSC_LPMEN) { + /* We cannot distinguish between MCP2518FD and + * MCP251863. If firmware specifies MCP251863, keep + * it, otherwise set to MCP2518FD. + */ + if (mcp251xfd_is_251863(priv)) + devtype_data = &mcp251xfd_devtype_data_mcp251863; + else + devtype_data = &mcp251xfd_devtype_data_mcp2518fd; + } else { devtype_data = &mcp251xfd_devtype_data_mcp2517fd; + } if (!mcp251xfd_is_251XFD(priv) && priv->devtype_data.model != devtype_data->model) { diff --git a/drivers/net/can/sun4i_can.c b/drivers/net/can/sun4i_can.c index 155b90f6c767..b90dfb429ccd 100644 --- a/drivers/net/can/sun4i_can.c +++ b/drivers/net/can/sun4i_can.c @@ -535,11 +535,6 @@ static int sun4i_can_err(struct net_device *dev, u8 isrc, u8 status) rxerr = (errc >> 16) & 0xFF; txerr = errc & 0xFF; - if (skb) { - cf->data[6] = txerr; - cf->data[7] = rxerr; - } - if (isrc & SUN4I_INT_DATA_OR) { /* data overrun interrupt */ netdev_dbg(dev, "data overrun interrupt\n"); @@ -570,6 +565,11 @@ static int sun4i_can_err(struct net_device *dev, u8 isrc, u8 status) else state = CAN_STATE_ERROR_ACTIVE; } + if (skb && state != CAN_STATE_BUS_OFF) { + cf->can_id |= CAN_ERR_CNT; + cf->data[6] = txerr; + cf->data[7] = rxerr; + } if (isrc & SUN4I_INT_BUS_ERR) { /* bus error interrupt */ netdev_dbg(dev, "bus error interrupt\n"); diff --git a/drivers/net/can/ti_hecc.c b/drivers/net/can/ti_hecc.c index debe17bfd0f0..afa38771520e 100644 --- a/drivers/net/can/ti_hecc.c +++ b/drivers/net/can/ti_hecc.c @@ -662,6 +662,7 @@ static void ti_hecc_change_state(struct net_device *ndev, can_change_state(priv->ndev, cf, tx_state, rx_state); if (max(tx_state, rx_state) != CAN_STATE_BUS_OFF) { + cf->can_id |= CAN_ERR_CNT; cf->data[6] = hecc_read(priv, HECC_CANTEC); cf->data[7] = hecc_read(priv, HECC_CANREC); } diff --git a/drivers/net/can/usb/esd_usb.c b/drivers/net/can/usb/esd_usb.c index 8a4bf2961f3d..177ed33e08d9 100644 --- a/drivers/net/can/usb/esd_usb.c +++ b/drivers/net/can/usb/esd_usb.c @@ -265,7 +265,8 @@ static void esd_usb_rx_event(struct esd_usb_net_priv *priv, priv->can.can_stats.bus_error++; stats->rx_errors++; - cf->can_id |= CAN_ERR_PROT | CAN_ERR_BUSERROR; + cf->can_id |= CAN_ERR_PROT | CAN_ERR_BUSERROR | + CAN_ERR_CNT; switch (ecc & SJA1000_ECC_MASK) { case SJA1000_ECC_BIT: diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c index 5d70844ac030..dd65c101bfb8 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c @@ -917,8 +917,11 @@ static void kvaser_usb_hydra_update_state(struct kvaser_usb_net_priv *priv, new_state < CAN_STATE_BUS_OFF) priv->can.can_stats.restarts++; - cf->data[6] = bec->txerr; - cf->data[7] = bec->rxerr; + if (new_state != CAN_STATE_BUS_OFF) { + cf->can_id |= CAN_ERR_CNT; + cf->data[6] = bec->txerr; + cf->data[7] = bec->rxerr; + } netif_rx(skb); } @@ -1069,8 +1072,11 @@ kvaser_usb_hydra_error_frame(struct kvaser_usb_net_priv *priv, shhwtstamps->hwtstamp = hwtstamp; cf->can_id |= CAN_ERR_BUSERROR; - cf->data[6] = bec.txerr; - cf->data[7] = bec.rxerr; + if (new_state != CAN_STATE_BUS_OFF) { + cf->can_id |= CAN_ERR_CNT; + cf->data[6] = bec.txerr; + cf->data[7] = bec.rxerr; + } netif_rx(skb); diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c index cc809ecd1e62..07f687f29b34 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c @@ -853,8 +853,11 @@ static void kvaser_usb_leaf_rx_error(const struct kvaser_usb *dev, break; } - cf->data[6] = es->txerr; - cf->data[7] = es->rxerr; + if (new_state != CAN_STATE_BUS_OFF) { + cf->can_id |= CAN_ERR_CNT; + cf->data[6] = es->txerr; + cf->data[7] = es->rxerr; + } netif_rx(skb); } diff --git a/drivers/net/can/usb/peak_usb/pcan_usb.c b/drivers/net/can/usb/peak_usb/pcan_usb.c index 091c631ebe23..d07b7ee79e3e 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb.c @@ -506,6 +506,7 @@ static int pcan_usb_decode_error(struct pcan_usb_msg_context *mc, u8 n, /* Supply TX/RX error counters in case of * controller error. */ + cf->can_id = CAN_ERR_CNT; cf->data[6] = mc->pdev->bec.txerr; cf->data[7] = mc->pdev->bec.rxerr; } diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c index b850ff8fe4bd..27b0a72fd885 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c @@ -57,7 +57,7 @@ MODULE_DEVICE_TABLE(usb, peak_usb_table); * dump memory */ #define DUMP_WIDTH 16 -void pcan_dump_mem(char *prompt, void *p, int l) +void pcan_dump_mem(const char *prompt, const void *p, int l) { pr_info("%s dumping %s (%d bytes):\n", PCAN_USB_DRIVER_NAME, prompt ? prompt : "memory", l); diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.h b/drivers/net/can/usb/peak_usb/pcan_usb_core.h index f60af573a2e0..9c90487b9c92 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.h +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.h @@ -132,7 +132,7 @@ struct peak_usb_device { struct peak_usb_device *next_siblings; }; -void pcan_dump_mem(char *prompt, void *p, int l); +void pcan_dump_mem(const char *prompt, const void *p, int l); /* common timestamp management */ void peak_usb_init_time_ref(struct peak_time_ref *time_ref, diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c index 65487ec33566..3d7e0e370505 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c @@ -33,6 +33,10 @@ #define PCAN_UFD_RX_BUFFER_SIZE 2048 #define PCAN_UFD_TX_BUFFER_SIZE 512 +/* struct pcan_ufd_fw_info::type */ +#define PCAN_USBFD_TYPE_STD 1 +#define PCAN_USBFD_TYPE_EXT 2 /* includes EP numbers */ + /* read some versions info from the hw device */ struct __packed pcan_ufd_fw_info { __le16 size_of; /* sizeof this */ @@ -44,6 +48,13 @@ struct __packed pcan_ufd_fw_info { __le32 dev_id[2]; /* "device id" per CAN */ __le32 ser_no; /* S/N */ __le32 flags; /* special functions */ + + /* extended data when type == PCAN_USBFD_TYPE_EXT */ + u8 cmd_out_ep; /* ep for cmd */ + u8 cmd_in_ep; /* ep for replies */ + u8 data_out_ep[2]; /* ep for CANx TX */ + u8 data_in_ep; /* ep for CAN RX */ + u8 dummy[3]; }; /* handle device specific info used by the netdevices */ @@ -171,6 +182,9 @@ static inline void *pcan_usb_fd_cmd_buffer(struct peak_usb_device *dev) /* send PCAN-USB Pro FD commands synchronously */ static int pcan_usb_fd_send_cmd(struct peak_usb_device *dev, void *cmd_tail) { + struct pcan_usb_fd_device *pdev = + container_of(dev, struct pcan_usb_fd_device, dev); + struct pcan_ufd_fw_info *fw_info = &pdev->usb_if->fw_info; void *cmd_head = pcan_usb_fd_cmd_buffer(dev); int err = 0; u8 *packet_ptr; @@ -200,7 +214,7 @@ static int pcan_usb_fd_send_cmd(struct peak_usb_device *dev, void *cmd_tail) do { err = usb_bulk_msg(dev->udev, usb_sndbulkpipe(dev->udev, - PCAN_USBPRO_EP_CMDOUT), + fw_info->cmd_out_ep), packet_ptr, packet_len, NULL, PCAN_UFD_CMD_TIMEOUT_MS); if (err) { @@ -426,6 +440,9 @@ static int pcan_usb_fd_set_bittiming_fast(struct peak_usb_device *dev, static int pcan_usb_fd_restart_async(struct peak_usb_device *dev, struct urb *urb, u8 *buf) { + struct pcan_usb_fd_device *pdev = + container_of(dev, struct pcan_usb_fd_device, dev); + struct pcan_ufd_fw_info *fw_info = &pdev->usb_if->fw_info; u8 *pc = buf; /* build the entire cmds list in the provided buffer, to go back into @@ -439,7 +456,7 @@ static int pcan_usb_fd_restart_async(struct peak_usb_device *dev, /* complete the URB */ usb_fill_bulk_urb(urb, dev->udev, - usb_sndbulkpipe(dev->udev, PCAN_USBPRO_EP_CMDOUT), + usb_sndbulkpipe(dev->udev, fw_info->cmd_out_ep), buf, pc - buf, pcan_usb_pro_restart_complete, dev); @@ -839,6 +856,15 @@ static int pcan_usb_fd_get_berr_counter(const struct net_device *netdev, return 0; } +/* probe function for all PCAN-USB FD family usb interfaces */ +static int pcan_usb_fd_probe(struct usb_interface *intf) +{ + struct usb_host_interface *iface_desc = &intf->altsetting[0]; + + /* CAN interface is always interface #0 */ + return iface_desc->desc.bInterfaceNumber; +} + /* stop interface (last chance before set bus off) */ static int pcan_usb_fd_stop(struct peak_usb_device *dev) { @@ -860,6 +886,7 @@ static int pcan_usb_fd_init(struct peak_usb_device *dev) { struct pcan_usb_fd_device *pdev = container_of(dev, struct pcan_usb_fd_device, dev); + struct pcan_ufd_fw_info *fw_info; int i, err = -ENOMEM; /* do this for 1st channel only */ @@ -878,10 +905,12 @@ static int pcan_usb_fd_init(struct peak_usb_device *dev) /* number of ts msgs to ignore before taking one into account */ pdev->usb_if->cm_ignore_count = 5; + fw_info = &pdev->usb_if->fw_info; + err = pcan_usb_pro_send_req(dev, PCAN_USBPRO_REQ_INFO, PCAN_USBPRO_INFO_FW, - &pdev->usb_if->fw_info, - sizeof(pdev->usb_if->fw_info)); + fw_info, + sizeof(*fw_info)); if (err) { dev_err(dev->netdev->dev.parent, "unable to read %s firmware info (err %d)\n", @@ -895,14 +924,14 @@ static int pcan_usb_fd_init(struct peak_usb_device *dev) */ dev_info(dev->netdev->dev.parent, "PEAK-System %s v%u fw v%u.%u.%u (%u channels)\n", - dev->adapter->name, pdev->usb_if->fw_info.hw_version, - pdev->usb_if->fw_info.fw_version[0], - pdev->usb_if->fw_info.fw_version[1], - pdev->usb_if->fw_info.fw_version[2], + dev->adapter->name, fw_info->hw_version, + fw_info->fw_version[0], + fw_info->fw_version[1], + fw_info->fw_version[2], dev->adapter->ctrl_count); /* check for ability to switch between ISO/non-ISO modes */ - if (pdev->usb_if->fw_info.fw_version[0] >= 2) { + if (fw_info->fw_version[0] >= 2) { /* firmware >= 2.x supports ISO/non-ISO switching */ dev->can.ctrlmode_supported |= CAN_CTRLMODE_FD_NON_ISO; } else { @@ -910,6 +939,14 @@ static int pcan_usb_fd_init(struct peak_usb_device *dev) dev->can.ctrlmode |= CAN_CTRLMODE_FD_NON_ISO; } + /* if vendor rsp is of type 2, then it contains EP numbers to + * use for cmds pipes. If not, then default EP should be used. + */ + if (fw_info->type != cpu_to_le16(PCAN_USBFD_TYPE_EXT)) { + fw_info->cmd_out_ep = PCAN_USBPRO_EP_CMDOUT; + fw_info->cmd_in_ep = PCAN_USBPRO_EP_CMDIN; + } + /* tell the hardware the can driver is running */ err = pcan_usb_fd_drv_loaded(dev, 1); if (err) { @@ -930,12 +967,23 @@ static int pcan_usb_fd_init(struct peak_usb_device *dev) /* do a copy of the ctrlmode[_supported] too */ dev->can.ctrlmode = ppdev->dev.can.ctrlmode; dev->can.ctrlmode_supported = ppdev->dev.can.ctrlmode_supported; + + fw_info = &pdev->usb_if->fw_info; } pdev->usb_if->dev[dev->ctrl_idx] = dev; dev->device_number = le32_to_cpu(pdev->usb_if->fw_info.dev_id[dev->ctrl_idx]); + /* if vendor rsp is of type 2, then it contains EP numbers to + * use for data pipes. If not, then statically defined EP are used + * (see peak_usb_create_dev()). + */ + if (fw_info->type == cpu_to_le16(PCAN_USBFD_TYPE_EXT)) { + dev->ep_msg_in = fw_info->data_in_ep; + dev->ep_msg_out = fw_info->data_out_ep[dev->ctrl_idx]; + } + /* set clock domain */ for (i = 0; i < ARRAY_SIZE(pcan_usb_fd_clk_freq); i++) if (dev->adapter->clock.freq == pcan_usb_fd_clk_freq[i]) @@ -1091,7 +1139,7 @@ const struct peak_usb_adapter pcan_usb_fd = { .tx_buffer_size = PCAN_UFD_TX_BUFFER_SIZE, /* device callbacks */ - .intf_probe = pcan_usb_pro_probe, /* same as PCAN-USB Pro */ + .intf_probe = pcan_usb_fd_probe, .dev_init = pcan_usb_fd_init, .dev_exit = pcan_usb_fd_exit, diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_pro.c b/drivers/net/can/usb/peak_usb/pcan_usb_pro.c index ebe087f258e3..457887113e75 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_pro.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_pro.c @@ -439,7 +439,7 @@ static int pcan_usb_pro_get_device_id(struct peak_usb_device *dev, return err; pdn = (struct pcan_usb_pro_devid *)pc; - *device_id = le32_to_cpu(pdn->serial_num); + *device_id = le32_to_cpu(pdn->dev_num); return err; } diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_pro.h b/drivers/net/can/usb/peak_usb/pcan_usb_pro.h index 5d4cf14eb9d9..a34e0fc021c9 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_pro.h +++ b/drivers/net/can/usb/peak_usb/pcan_usb_pro.h @@ -112,7 +112,7 @@ struct __packed pcan_usb_pro_devid { u8 data_type; u8 channel; __le16 dummy; - __le32 serial_num; + __le32 dev_num; }; #define PCAN_USBPRO_LED_DEVICE 0x00 diff --git a/drivers/net/can/usb/usb_8dev.c b/drivers/net/can/usb/usb_8dev.c index f3363575bf32..8b7cd69e20b0 100644 --- a/drivers/net/can/usb/usb_8dev.c +++ b/drivers/net/can/usb/usb_8dev.c @@ -438,9 +438,11 @@ static void usb_8dev_rx_err_msg(struct usb_8dev_priv *priv, if (rx_errors) stats->rx_errors++; - - cf->data[6] = txerr; - cf->data[7] = rxerr; + if (priv->can.state != CAN_STATE_BUS_OFF) { + cf->can_id |= CAN_ERR_CNT; + cf->data[6] = txerr; + cf->data[7] = rxerr; + } priv->bec.txerr = txerr; priv->bec.rxerr = rxerr; diff --git a/drivers/net/can/xilinx_can.c b/drivers/net/can/xilinx_can.c index 0de2f97d9f62..caa6b4cee63f 100644 --- a/drivers/net/can/xilinx_can.c +++ b/drivers/net/can/xilinx_can.c @@ -965,6 +965,7 @@ static void xcan_set_error_state(struct net_device *ndev, can_change_state(ndev, cf, tx_state, rx_state); if (cf) { + cf->can_id |= CAN_ERR_CNT; cf->data[6] = txerr; cf->data[7] = rxerr; } diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig index 702d68ae435a..d8ae0e8af2a0 100644 --- a/drivers/net/dsa/Kconfig +++ b/drivers/net/dsa/Kconfig @@ -60,14 +60,6 @@ source "drivers/net/dsa/sja1105/Kconfig" source "drivers/net/dsa/xrs700x/Kconfig" -config NET_DSA_QCA8K - tristate "Qualcomm Atheros QCA8K Ethernet switch family support" - select NET_DSA_TAG_QCA - select REGMAP - help - This enables support for the Qualcomm Atheros QCA8K Ethernet - switch chips. - source "drivers/net/dsa/realtek/Kconfig" config NET_DSA_RZN1_A5PSW diff --git a/drivers/net/dsa/Makefile b/drivers/net/dsa/Makefile index b32907afa702..16eb879e0cb4 100644 --- a/drivers/net/dsa/Makefile +++ b/drivers/net/dsa/Makefile @@ -8,7 +8,6 @@ endif obj-$(CONFIG_NET_DSA_LANTIQ_GSWIP) += lantiq_gswip.o obj-$(CONFIG_NET_DSA_MT7530) += mt7530.o obj-$(CONFIG_NET_DSA_MV88E6060) += mv88e6060.o -obj-$(CONFIG_NET_DSA_QCA8K) += qca8k.o obj-$(CONFIG_NET_DSA_RZN1_A5PSW) += rzn1_a5psw.o obj-$(CONFIG_NET_DSA_SMSC_LAN9303) += lan9303-core.o obj-$(CONFIG_NET_DSA_SMSC_LAN9303_I2C) += lan9303_i2c.o diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c index 28d7cb2ce98f..fd12a68c1dcd 100644 --- a/drivers/net/dsa/microchip/ksz_common.c +++ b/drivers/net/dsa/microchip/ksz_common.c @@ -150,6 +150,7 @@ static const struct ksz_dev_ops ksz8_dev_ops = { .port_setup = ksz8_port_setup, .r_phy = ksz8_r_phy, .w_phy = ksz8_w_phy, + .r_mib_cnt = ksz8_r_mib_cnt, .r_mib_pkt = ksz8_r_mib_pkt, .freeze_mib = ksz8_freeze_mib, .port_init_cnt = ksz8_port_init_cnt, @@ -1578,18 +1579,21 @@ int ksz_switch_register(struct ksz_device *dev) ports = of_get_child_by_name(dev->dev->of_node, "ethernet-ports"); if (!ports) ports = of_get_child_by_name(dev->dev->of_node, "ports"); - if (ports) + if (ports) { for_each_available_child_of_node(ports, port) { if (of_property_read_u32(port, "reg", &port_num)) continue; if (!(dev->port_mask & BIT(port_num))) { of_node_put(port); + of_node_put(ports); return -EINVAL; } of_get_phy_mode(port, &dev->ports[port_num].interface); } + of_node_put(ports); + } dev->synclko_125 = of_property_read_bool(dev->dev->of_node, "microchip,synclko-125"); dev->synclko_disable = of_property_read_bool(dev->dev->of_node, diff --git a/drivers/net/dsa/microchip/ksz_spi.c b/drivers/net/dsa/microchip/ksz_spi.c index 4844830dca72..05bd089795f8 100644 --- a/drivers/net/dsa/microchip/ksz_spi.c +++ b/drivers/net/dsa/microchip/ksz_spi.c @@ -215,7 +215,7 @@ static struct spi_driver ksz_spi_driver = { .driver = { .name = "ksz-switch", .owner = THIS_MODULE, - .of_match_table = of_match_ptr(ksz_dt_ids), + .of_match_table = ksz_dt_ids, }, .id_table = ksz_spi_ids, .probe = ksz_spi_probe, diff --git a/drivers/net/dsa/qca/Kconfig b/drivers/net/dsa/qca/Kconfig index 13b7e679b8b5..ba339747362c 100644 --- a/drivers/net/dsa/qca/Kconfig +++ b/drivers/net/dsa/qca/Kconfig @@ -7,3 +7,11 @@ config NET_DSA_AR9331 help This enables support for the Qualcomm Atheros AR9331 built-in Ethernet switch. + +config NET_DSA_QCA8K + tristate "Qualcomm Atheros QCA8K Ethernet switch family support" + select NET_DSA_TAG_QCA + select REGMAP + help + This enables support for the Qualcomm Atheros QCA8K Ethernet + switch chips. diff --git a/drivers/net/dsa/qca/Makefile b/drivers/net/dsa/qca/Makefile index 274022319066..40bb7c27285b 100644 --- a/drivers/net/dsa/qca/Makefile +++ b/drivers/net/dsa/qca/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_NET_DSA_AR9331) += ar9331.o +obj-$(CONFIG_NET_DSA_QCA8K) += qca8k.o diff --git a/drivers/net/dsa/qca8k.c b/drivers/net/dsa/qca/qca8k.c index 1cbb05b0323f..1cbb05b0323f 100644 --- a/drivers/net/dsa/qca8k.c +++ b/drivers/net/dsa/qca/qca8k.c diff --git a/drivers/net/dsa/qca8k.h b/drivers/net/dsa/qca/qca8k.h index ec58d0e80a70..ec58d0e80a70 100644 --- a/drivers/net/dsa/qca8k.h +++ b/drivers/net/dsa/qca/qca8k.h diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index b253e27bcfb4..b03d0d0c3dbf 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -3382,12 +3382,28 @@ static const struct of_device_id sja1105_dt_ids[] = { }; MODULE_DEVICE_TABLE(of, sja1105_dt_ids); +static const struct spi_device_id sja1105_spi_ids[] = { + { "sja1105e" }, + { "sja1105t" }, + { "sja1105p" }, + { "sja1105q" }, + { "sja1105r" }, + { "sja1105s" }, + { "sja1110a" }, + { "sja1110b" }, + { "sja1110c" }, + { "sja1110d" }, + { }, +}; +MODULE_DEVICE_TABLE(spi, sja1105_spi_ids); + static struct spi_driver sja1105_driver = { .driver = { .name = "sja1105", .owner = THIS_MODULE, .of_match_table = of_match_ptr(sja1105_dt_ids), }, + .id_table = sja1105_spi_ids, .probe = sja1105_probe, .remove = sja1105_remove, .shutdown = sja1105_shutdown, diff --git a/drivers/net/dsa/vitesse-vsc73xx-spi.c b/drivers/net/dsa/vitesse-vsc73xx-spi.c index 3110895358d8..97a92e6da60d 100644 --- a/drivers/net/dsa/vitesse-vsc73xx-spi.c +++ b/drivers/net/dsa/vitesse-vsc73xx-spi.c @@ -205,10 +205,20 @@ static const struct of_device_id vsc73xx_of_match[] = { }; MODULE_DEVICE_TABLE(of, vsc73xx_of_match); +static const struct spi_device_id vsc73xx_spi_ids[] = { + { "vsc7385" }, + { "vsc7388" }, + { "vsc7395" }, + { "vsc7398" }, + { }, +}; +MODULE_DEVICE_TABLE(spi, vsc73xx_spi_ids); + static struct spi_driver vsc73xx_spi_driver = { .probe = vsc73xx_spi_probe, .remove = vsc73xx_spi_remove, .shutdown = vsc73xx_spi_shutdown, + .id_table = vsc73xx_spi_ids, .driver = { .name = "vsc73xx-spi", .of_match_table = vsc73xx_of_match, diff --git a/drivers/net/ethernet/amd/amd8111e.c b/drivers/net/ethernet/amd/amd8111e.c index 05ac8d9ccb2f..5d1baa01360f 100644 --- a/drivers/net/ethernet/amd/amd8111e.c +++ b/drivers/net/ethernet/amd/amd8111e.c @@ -1830,9 +1830,6 @@ static int amd8111e_probe_one(struct pci_dev *pdev, dev->max_mtu = AMD8111E_MAX_MTU; netif_napi_add_weight(dev, &lp->napi, amd8111e_rx_poll, 32); -#if AMD8111E_VLAN_TAG_USED - dev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; -#endif /* Probe the external PHY */ amd8111e_probe_ext_phy(dev); diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c index 948584761e66..be4b1f8eef29 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c +++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c @@ -2734,8 +2734,8 @@ static int atl1c_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netif_napi_add(netdev, &adapter->rrd_ring[i].napi, atl1c_clean_rx, 64); for (i = 0; i < adapter->tx_queue_count; ++i) - netif_napi_add(netdev, &adapter->tpd_ring[i].napi, - atl1c_clean_tx, 64); + netif_napi_add_tx(netdev, &adapter->tpd_ring[i].napi, + atl1c_clean_tx); timer_setup(&adapter->phy_config_timer, atl1c_phy_config, 0); /* setup the private structure */ err = atl1c_sw_init(adapter); diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index 7ca077b65eaa..9c410f93a103 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -717,14 +717,15 @@ #define MACB_CAPS_BD_RD_PREFETCH 0x00000080 #define MACB_CAPS_NEEDS_RSTONUBR 0x00000100 #define MACB_CAPS_MIIONRGMII 0x00000200 +#define MACB_CAPS_NEED_TSUCLK 0x00000400 +#define MACB_CAPS_PCS 0x01000000 +#define MACB_CAPS_HIGH_SPEED 0x02000000 #define MACB_CAPS_CLK_HW_CHG 0x04000000 #define MACB_CAPS_MACB_IS_EMAC 0x08000000 #define MACB_CAPS_FIFO_MODE 0x10000000 #define MACB_CAPS_GIGABIT_MODE_AVAILABLE 0x20000000 #define MACB_CAPS_SG_DISABLED 0x40000000 #define MACB_CAPS_MACB_IS_GEM 0x80000000 -#define MACB_CAPS_PCS 0x01000000 -#define MACB_CAPS_HIGH_SPEED 0x02000000 /* LSO settings */ #define MACB_LSO_UFO_ENABLE 0x01 diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index d5f7a9fb36d5..4cd4f57ca2aa 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -3482,7 +3482,8 @@ static int gem_add_flow_filter(struct net_device *netdev, fs->flow_type, (int)fs->ring_cookie, fs->location, htonl(fs->h_u.tcp_ip4_spec.ip4src), htonl(fs->h_u.tcp_ip4_spec.ip4dst), - htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); + be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), + be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); spin_lock_irqsave(&bp->rx_fs_lock, flags); @@ -3535,8 +3536,8 @@ static int gem_del_flow_filter(struct net_device *netdev, fs->flow_type, (int)fs->ring_cookie, fs->location, htonl(fs->h_u.tcp_ip4_spec.ip4src), htonl(fs->h_u.tcp_ip4_spec.ip4dst), - htons(fs->h_u.tcp_ip4_spec.psrc), - htons(fs->h_u.tcp_ip4_spec.pdst)); + be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), + be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); gem_writel_n(bp, SCRT2, fs->location, 0); @@ -4772,6 +4773,16 @@ static const struct macb_config sama7g5_emac_config = { .usrio = &sama7g5_usrio, }; +static const struct macb_config versal_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | MACB_CAPS_NEED_TSUCLK, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = init_reset_optional, + .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, +}; + static const struct of_device_id macb_dt_ids[] = { { .compatible = "cdns,at32ap7000-macb" }, { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, @@ -4793,6 +4804,7 @@ static const struct of_device_id macb_dt_ids[] = { { .compatible = "microchip,mpfs-macb", .data = &mpfs_config }, { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, + { .compatible = "cdns,versal-gem", .data = &versal_config}, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, macb_dt_ids); @@ -5202,7 +5214,7 @@ static int __maybe_unused macb_runtime_suspend(struct device *dev) if (!(device_may_wakeup(dev))) macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk); - else + else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk); return 0; @@ -5218,8 +5230,10 @@ static int __maybe_unused macb_runtime_resume(struct device *dev) clk_prepare_enable(bp->hclk); clk_prepare_enable(bp->tx_clk); clk_prepare_enable(bp->rx_clk); + clk_prepare_enable(bp->tsu_clk); + } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { + clk_prepare_enable(bp->tsu_clk); } - clk_prepare_enable(bp->tsu_clk); return 0; } diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c index 7c760aa65540..ddfe9208529a 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c @@ -1236,8 +1236,8 @@ static struct sock *chtls_recv_sock(struct sock *lsk, csk->sndbuf = newsk->sk_sndbuf; csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx; RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk), - sock_net(newsk)-> - ipv4.sysctl_tcp_window_scaling, + READ_ONCE(sock_net(newsk)-> + ipv4.sysctl_tcp_window_scaling), tp->window_clamp); neigh_release(n); inet_inherit_port(&tcp_hashinfo, lsk, newsk); @@ -1384,7 +1384,7 @@ static void chtls_pass_accept_request(struct sock *sk, #endif } if (req->tcpopt.wsf <= 14 && - sock_net(sk)->ipv4.sysctl_tcp_window_scaling) { + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) { inet_rsk(oreq)->wscale_ok = 1; inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf; } diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c index 528eb0f223b1..b4f5e57d0285 100644 --- a/drivers/net/ethernet/emulex/benet/be_cmds.c +++ b/drivers/net/ethernet/emulex/benet/be_cmds.c @@ -2287,7 +2287,7 @@ err: /* Uses sync mcc */ int be_cmd_read_port_transceiver_data(struct be_adapter *adapter, - u8 page_num, u8 *data) + u8 page_num, u32 off, u32 len, u8 *data) { struct be_dma_mem cmd; struct be_mcc_wrb *wrb; @@ -2321,10 +2321,10 @@ int be_cmd_read_port_transceiver_data(struct be_adapter *adapter, req->port = cpu_to_le32(adapter->hba_port_num); req->page_num = cpu_to_le32(page_num); status = be_mcc_notify_wait(adapter); - if (!status) { + if (!status && len > 0) { struct be_cmd_resp_port_type *resp = cmd.va; - memcpy(data, resp->page_data, PAGE_DATA_LEN); + memcpy(data, resp->page_data + off, len); } err: mutex_unlock(&adapter->mcc_lock); @@ -2415,7 +2415,7 @@ int be_cmd_query_cable_type(struct be_adapter *adapter) int status; status = be_cmd_read_port_transceiver_data(adapter, TR_PAGE_A0, - page_data); + 0, PAGE_DATA_LEN, page_data); if (!status) { switch (adapter->phy.interface_type) { case PHY_TYPE_QSFP: @@ -2440,7 +2440,7 @@ int be_cmd_query_sfp_info(struct be_adapter *adapter) int status; status = be_cmd_read_port_transceiver_data(adapter, TR_PAGE_A0, - page_data); + 0, PAGE_DATA_LEN, page_data); if (!status) { strlcpy(adapter->phy.vendor_name, page_data + SFP_VENDOR_NAME_OFFSET, SFP_VENDOR_NAME_LEN - 1); diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.h b/drivers/net/ethernet/emulex/benet/be_cmds.h index db1f3b908582..e2085c68c0ee 100644 --- a/drivers/net/ethernet/emulex/benet/be_cmds.h +++ b/drivers/net/ethernet/emulex/benet/be_cmds.h @@ -2427,7 +2427,7 @@ int be_cmd_set_beacon_state(struct be_adapter *adapter, u8 port_num, u8 beacon, int be_cmd_get_beacon_state(struct be_adapter *adapter, u8 port_num, u32 *state); int be_cmd_read_port_transceiver_data(struct be_adapter *adapter, - u8 page_num, u8 *data); + u8 page_num, u32 off, u32 len, u8 *data); int be_cmd_query_cable_type(struct be_adapter *adapter); int be_cmd_query_sfp_info(struct be_adapter *adapter); int lancer_cmd_read_object(struct be_adapter *adapter, struct be_dma_mem *cmd, diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c index dfa784339781..bd0df189d871 100644 --- a/drivers/net/ethernet/emulex/benet/be_ethtool.c +++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c @@ -1344,7 +1344,7 @@ static int be_get_module_info(struct net_device *netdev, return -EOPNOTSUPP; status = be_cmd_read_port_transceiver_data(adapter, TR_PAGE_A0, - page_data); + 0, PAGE_DATA_LEN, page_data); if (!status) { if (!page_data[SFP_PLUS_SFF_8472_COMP]) { modinfo->type = ETH_MODULE_SFF_8079; @@ -1362,25 +1362,32 @@ static int be_get_module_eeprom(struct net_device *netdev, { struct be_adapter *adapter = netdev_priv(netdev); int status; + u32 begin, end; if (!check_privilege(adapter, MAX_PRIVILEGES)) return -EOPNOTSUPP; - status = be_cmd_read_port_transceiver_data(adapter, TR_PAGE_A0, - data); - if (status) - goto err; + begin = eeprom->offset; + end = eeprom->offset + eeprom->len; + + if (begin < PAGE_DATA_LEN) { + status = be_cmd_read_port_transceiver_data(adapter, TR_PAGE_A0, begin, + min_t(u32, end, PAGE_DATA_LEN) - begin, + data); + if (status) + goto err; + + data += PAGE_DATA_LEN - begin; + begin = PAGE_DATA_LEN; + } - if (eeprom->offset + eeprom->len > PAGE_DATA_LEN) { - status = be_cmd_read_port_transceiver_data(adapter, - TR_PAGE_A2, - data + - PAGE_DATA_LEN); + if (end > PAGE_DATA_LEN) { + status = be_cmd_read_port_transceiver_data(adapter, TR_PAGE_A2, + begin - PAGE_DATA_LEN, + end - begin, data); if (status) goto err; } - if (eeprom->offset) - memcpy(data, data + eeprom->offset, eeprom->len); err: return be_cmd_status(status); } diff --git a/drivers/net/ethernet/intel/e1000e/hw.h b/drivers/net/ethernet/intel/e1000e/hw.h index 13382df2f2ef..bcf680e83811 100644 --- a/drivers/net/ethernet/intel/e1000e/hw.h +++ b/drivers/net/ethernet/intel/e1000e/hw.h @@ -630,7 +630,6 @@ struct e1000_phy_info { bool disable_polarity_correction; bool is_mdix; bool polarity_correction; - bool reset_disable; bool speed_downgraded; bool autoneg_wait_to_complete; }; diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c index e6c8e6d5234f..9466f65a6da7 100644 --- a/drivers/net/ethernet/intel/e1000e/ich8lan.c +++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c @@ -2050,10 +2050,6 @@ static s32 e1000_check_reset_block_ich8lan(struct e1000_hw *hw) bool blocked = false; int i = 0; - /* Check the PHY (LCD) reset flag */ - if (hw->phy.reset_disable) - return true; - while ((blocked = !(er32(FWSM) & E1000_ICH_FWSM_RSPCIPHY)) && (i++ < 30)) usleep_range(10000, 11000); diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.h b/drivers/net/ethernet/intel/e1000e/ich8lan.h index 638a3ddd7ada..2504b11c3169 100644 --- a/drivers/net/ethernet/intel/e1000e/ich8lan.h +++ b/drivers/net/ethernet/intel/e1000e/ich8lan.h @@ -271,7 +271,6 @@ #define I217_CGFREG_ENABLE_MTA_RESET 0x0002 #define I217_MEMPWR PHY_REG(772, 26) #define I217_MEMPWR_DISABLE_SMB_RELEASE 0x0010 -#define I217_MEMPWR_MOEM 0x1000 /* Receive Address Initial CRC Calculation */ #define E1000_PCH_RAICC(_n) (0x05F50 + ((_n) * 4)) diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 38e60def8520..70d933f52e93 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -6494,6 +6494,10 @@ static void e1000e_s0ix_exit_flow(struct e1000_adapter *adapter) if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID && hw->mac.type >= e1000_pch_adp) { + /* Keep the GPT clock enabled for CSME */ + mac_data = er32(FEXTNVM); + mac_data |= BIT(3); + ew32(FEXTNVM, mac_data); /* Request ME unconfigure the device from S0ix */ mac_data = er32(H2ME); mac_data &= ~E1000_H2ME_START_DPG; @@ -6987,21 +6991,8 @@ static __maybe_unused int e1000e_pm_suspend(struct device *dev) struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev)); struct e1000_adapter *adapter = netdev_priv(netdev); struct pci_dev *pdev = to_pci_dev(dev); - struct e1000_hw *hw = &adapter->hw; - u16 phy_data; int rc; - if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID && - hw->mac.type >= e1000_pch_adp) { - /* Mask OEM Bits / Gig Disable / Restart AN (772_26[12] = 1) */ - e1e_rphy(hw, I217_MEMPWR, &phy_data); - phy_data |= I217_MEMPWR_MOEM; - e1e_wphy(hw, I217_MEMPWR, phy_data); - - /* Disable LCD reset */ - hw->phy.reset_disable = true; - } - e1000e_flush_lpic(pdev); e1000e_pm_freeze(dev); @@ -7023,8 +7014,6 @@ static __maybe_unused int e1000e_pm_resume(struct device *dev) struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev)); struct e1000_adapter *adapter = netdev_priv(netdev); struct pci_dev *pdev = to_pci_dev(dev); - struct e1000_hw *hw = &adapter->hw; - u16 phy_data; int rc; /* Introduce S0ix implementation */ @@ -7035,17 +7024,6 @@ static __maybe_unused int e1000e_pm_resume(struct device *dev) if (rc) return rc; - if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID && - hw->mac.type >= e1000_pch_adp) { - /* Unmask OEM Bits / Gig Disable / Restart AN 772_26[12] = 0 */ - e1e_rphy(hw, I217_MEMPWR, &phy_data); - phy_data &= ~I217_MEMPWR_MOEM; - e1e_wphy(hw, I217_MEMPWR, phy_data); - - /* Enable LCD reset */ - hw->phy.reset_disable = false; - } - return e1000e_pm_thaw(dev); } diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index 97c574a33ba0..d86b6d349ea9 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -1292,4 +1292,18 @@ int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, struct i40e_cloud_filter *filter, bool add); + +/** + * i40e_is_tc_mqprio_enabled - check if TC MQPRIO is enabled on PF + * @pf: pointer to a pf. + * + * Check and return value of flag I40E_FLAG_TC_MQPRIO. + * + * Return: I40E_FLAG_TC_MQPRIO set state. + **/ +static inline u32 i40e_is_tc_mqprio_enabled(struct i40e_pf *pf) +{ + return pf->flags & I40E_FLAG_TC_MQPRIO; +} + #endif /* _I40E_H_ */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index d9a934ca3eba..156e92c43780 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -5021,7 +5021,7 @@ static int i40e_set_channels(struct net_device *dev, /* We do not support setting channels via ethtool when TCs are * configured through mqprio */ - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) return -EINVAL; /* verify they are not requesting separate vectors */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 151e9b6b9df4..fb9f476fb33c 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -5456,7 +5456,7 @@ static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) u8 num_tc = 0; struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; /* If neither MQPRIO nor DCB is enabled, then always use single TC */ @@ -5488,7 +5488,7 @@ static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) **/ static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) { - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) return i40e_mqprio_get_enabled_tc(pf); /* If neither MQPRIO nor DCB is enabled for this PF then just return @@ -5585,7 +5585,7 @@ static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, int i; /* There is no need to reset BW when mqprio mode is on. */ - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) return 0; if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) { ret = i40e_set_bw_limit(vsi, vsi->seid, 0); @@ -5657,7 +5657,7 @@ static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) vsi->tc_config.tc_info[i].qoffset); } - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) return; /* Assign UP2TC map for the VSI */ @@ -5818,7 +5818,7 @@ static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) ctxt.vf_num = 0; ctxt.uplink_seid = vsi->uplink_seid; ctxt.info = vsi->info; - if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) { + if (i40e_is_tc_mqprio_enabled(pf)) { ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); if (ret) goto out; @@ -6542,7 +6542,7 @@ int i40e_create_queue_channel(struct i40e_vsi *vsi, pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; if (vsi->type == I40E_VSI_MAIN) { - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); else i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG); @@ -7936,7 +7936,7 @@ static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev) netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n"); return ERR_PTR(-EINVAL); } - if ((pf->flags & I40E_FLAG_TC_MQPRIO)) { + if (i40e_is_tc_mqprio_enabled(pf)) { netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n"); return ERR_PTR(-EINVAL); } @@ -8189,7 +8189,7 @@ config_tc: /* Quiesce VSI queues */ i40e_quiesce_vsi(vsi); - if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO)) + if (!hw && !i40e_is_tc_mqprio_enabled(pf)) i40e_remove_queue_channels(vsi); /* Configure VSI for enabled TCs */ @@ -8213,7 +8213,7 @@ config_tc: "Setup channel (id:%u) utilizing num_queues %d\n", vsi->seid, vsi->tc_config.tc_info[0].qcount); - if (pf->flags & I40E_FLAG_TC_MQPRIO) { + if (i40e_is_tc_mqprio_enabled(pf)) { if (vsi->mqprio_qopt.max_rate[0]) { u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; @@ -10773,7 +10773,7 @@ static int i40e_reset(struct i40e_pf *pf) **/ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) { - int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state); + const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf); struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; struct i40e_hw *hw = &pf->hw; i40e_status ret; @@ -10781,13 +10781,11 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) int v; if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && - i40e_check_recovery_mode(pf)) { + is_recovery_mode_reported) i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev); - } if (test_bit(__I40E_DOWN, pf->state) && - !test_bit(__I40E_RECOVERY_MODE, pf->state) && - !old_recovery_mode_bit) + !test_bit(__I40E_RECOVERY_MODE, pf->state)) goto clear_recovery; dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); @@ -10814,13 +10812,12 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) * accordingly with regard to resources initialization * and deinitialization */ - if (test_bit(__I40E_RECOVERY_MODE, pf->state) || - old_recovery_mode_bit) { + if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { if (i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities)) goto end_unlock; - if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { + if (is_recovery_mode_reported) { /* we're staying in recovery mode so we'll reinitialize * misc vector here */ @@ -10870,7 +10867,7 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) * unless I40E_FLAG_TC_MQPRIO was enabled or DCB * is not supported with new link speed */ - if (pf->flags & I40E_FLAG_TC_MQPRIO) { + if (i40e_is_tc_mqprio_enabled(pf)) { i40e_aq_set_dcb_parameters(hw, false, NULL); } else { if (I40E_IS_X710TL_DEVICE(hw->device_id) && diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index fda1198d2c00..69703801d1b0 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -64,7 +64,6 @@ struct iavf_vsi { u16 id; DECLARE_BITMAP(state, __IAVF_VSI_STATE_SIZE__); int base_vector; - u16 work_limit; u16 qs_handle; void *priv; /* client driver data reference. */ }; @@ -160,8 +159,12 @@ struct iavf_vlan { struct iavf_vlan_filter { struct list_head list; struct iavf_vlan vlan; - bool remove; /* filter needs to be removed */ - bool add; /* filter needs to be added */ + struct { + u8 is_new_vlan:1; /* filter is new, wait for PF answer */ + u8 remove:1; /* filter needs to be removed */ + u8 add:1; /* filter needs to be added */ + u8 padding:5; + }; }; #define IAVF_MAX_TRAFFIC_CLASS 4 @@ -464,6 +467,10 @@ static inline const char *iavf_state_str(enum iavf_state_t state) return "__IAVF_INIT_VERSION_CHECK"; case __IAVF_INIT_GET_RESOURCES: return "__IAVF_INIT_GET_RESOURCES"; + case __IAVF_INIT_EXTENDED_CAPS: + return "__IAVF_INIT_EXTENDED_CAPS"; + case __IAVF_INIT_CONFIG_ADAPTER: + return "__IAVF_INIT_CONFIG_ADAPTER"; case __IAVF_INIT_SW: return "__IAVF_INIT_SW"; case __IAVF_INIT_FAILED: @@ -523,6 +530,7 @@ int iavf_get_vf_config(struct iavf_adapter *adapter); int iavf_get_vf_vlan_v2_caps(struct iavf_adapter *adapter); int iavf_send_vf_offload_vlan_v2_msg(struct iavf_adapter *adapter); void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter); +u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter); void iavf_irq_enable(struct iavf_adapter *adapter, bool flush); void iavf_configure_queues(struct iavf_adapter *adapter); void iavf_deconfigure_queues(struct iavf_adapter *adapter); diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index 3bb56714beb0..e535d4c3da49 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -692,12 +692,8 @@ static int __iavf_get_coalesce(struct net_device *netdev, struct ethtool_coalesce *ec, int queue) { struct iavf_adapter *adapter = netdev_priv(netdev); - struct iavf_vsi *vsi = &adapter->vsi; struct iavf_ring *rx_ring, *tx_ring; - ec->tx_max_coalesced_frames = vsi->work_limit; - ec->rx_max_coalesced_frames = vsi->work_limit; - /* Rx and Tx usecs per queue value. If user doesn't specify the * queue, return queue 0's value to represent. */ @@ -825,12 +821,8 @@ static int __iavf_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *ec, int queue) { struct iavf_adapter *adapter = netdev_priv(netdev); - struct iavf_vsi *vsi = &adapter->vsi; int i; - if (ec->tx_max_coalesced_frames_irq || ec->rx_max_coalesced_frames_irq) - vsi->work_limit = ec->tx_max_coalesced_frames_irq; - if (ec->rx_coalesce_usecs == 0) { if (ec->use_adaptive_rx_coalesce) netif_info(adapter, drv, netdev, "rx-usecs=0, need to disable adaptive-rx for a complete disable\n"); @@ -1969,8 +1961,6 @@ static int iavf_set_rxfh(struct net_device *netdev, const u32 *indir, static const struct ethtool_ops iavf_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | - ETHTOOL_COALESCE_MAX_FRAMES | - ETHTOOL_COALESCE_MAX_FRAMES_IRQ | ETHTOOL_COALESCE_USE_ADAPTIVE, .get_drvinfo = iavf_get_drvinfo, .get_link = ethtool_op_get_link, diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 69ade653f5d4..e78c38d02432 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -843,7 +843,7 @@ static void iavf_restore_filters(struct iavf_adapter *adapter) * iavf_get_num_vlans_added - get number of VLANs added * @adapter: board private structure */ -static u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter) +u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter) { return bitmap_weight(adapter->vsi.active_cvlans, VLAN_N_VID) + bitmap_weight(adapter->vsi.active_svlans, VLAN_N_VID); @@ -906,11 +906,6 @@ static int iavf_vlan_rx_add_vid(struct net_device *netdev, if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)))) return -ENOMEM; - if (proto == cpu_to_be16(ETH_P_8021Q)) - set_bit(vid, adapter->vsi.active_cvlans); - else - set_bit(vid, adapter->vsi.active_svlans); - return 0; } @@ -2331,7 +2326,6 @@ int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter) adapter->vsi.back = adapter; adapter->vsi.base_vector = 1; - adapter->vsi.work_limit = IAVF_DEFAULT_IRQ_WORK; vsi->netdev = adapter->netdev; vsi->qs_handle = adapter->vsi_res->qset_handle; if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { @@ -3044,6 +3038,9 @@ continue_reset: adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; iavf_misc_irq_enable(adapter); + bitmap_clear(adapter->vsi.active_cvlans, 0, VLAN_N_VID); + bitmap_clear(adapter->vsi.active_svlans, 0, VLAN_N_VID); + mod_delayed_work(iavf_wq, &adapter->watchdog_task, 2); /* We were running when the reset started, so we need to restore some @@ -3835,6 +3832,29 @@ static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc, } /** + * iavf_find_cf - Find the cloud filter in the list + * @adapter: Board private structure + * @cookie: filter specific cookie + * + * Returns ptr to the filter object or NULL. Must be called while holding the + * cloud_filter_list_lock. + */ +static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter, + unsigned long *cookie) +{ + struct iavf_cloud_filter *filter = NULL; + + if (!cookie) + return NULL; + + list_for_each_entry(filter, &adapter->cloud_filter_list, list) { + if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) + return filter; + } + return NULL; +} + +/** * iavf_configure_clsflower - Add tc flower filters * @adapter: board private structure * @cls_flower: Pointer to struct flow_cls_offload @@ -3865,6 +3885,15 @@ static int iavf_configure_clsflower(struct iavf_adapter *adapter, filter->cookie = cls_flower->cookie; + /* bail out here if filter already exists */ + spin_lock_bh(&adapter->cloud_filter_list_lock); + if (iavf_find_cf(adapter, &cls_flower->cookie)) { + dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n"); + err = -EEXIST; + goto spin_unlock; + } + spin_unlock_bh(&adapter->cloud_filter_list_lock); + /* set the mask to all zeroes to begin with */ memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec)); /* start out with flow type and eth type IPv4 to begin with */ @@ -3883,6 +3912,7 @@ static int iavf_configure_clsflower(struct iavf_adapter *adapter, adapter->num_cloud_filters++; filter->add = true; adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; +spin_unlock: spin_unlock_bh(&adapter->cloud_filter_list_lock); err: if (err) @@ -3892,28 +3922,6 @@ err: return err; } -/* iavf_find_cf - Find the cloud filter in the list - * @adapter: Board private structure - * @cookie: filter specific cookie - * - * Returns ptr to the filter object or NULL. Must be called while holding the - * cloud_filter_list_lock. - */ -static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter, - unsigned long *cookie) -{ - struct iavf_cloud_filter *filter = NULL; - - if (!cookie) - return NULL; - - list_for_each_entry(filter, &adapter->cloud_filter_list, list) { - if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) - return filter; - } - return NULL; -} - /** * iavf_delete_clsflower - Remove tc flower filters * @adapter: board private structure diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c index 978f651c6b09..06d18797d25a 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c +++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c @@ -194,7 +194,7 @@ static bool iavf_clean_tx_irq(struct iavf_vsi *vsi, struct iavf_tx_buffer *tx_buf; struct iavf_tx_desc *tx_desc; unsigned int total_bytes = 0, total_packets = 0; - unsigned int budget = vsi->work_limit; + unsigned int budget = IAVF_DEFAULT_IRQ_WORK; tx_buf = &tx_ring->tx_bi[i]; tx_desc = IAVF_TX_DESC(tx_ring, i); @@ -1285,11 +1285,10 @@ static struct iavf_rx_buffer *iavf_get_rx_buffer(struct iavf_ring *rx_ring, { struct iavf_rx_buffer *rx_buffer; - if (!size) - return NULL; - rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean]; prefetchw(rx_buffer->page); + if (!size) + return rx_buffer; /* we are reusing so sync this buffer for CPU use */ dma_sync_single_range_for_cpu(rx_ring->dev, diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index 0d22bba6a5f2..15ee85dc33bd 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -628,6 +628,33 @@ static void iavf_mac_add_reject(struct iavf_adapter *adapter) } /** + * iavf_vlan_add_reject + * @adapter: adapter structure + * + * Remove VLAN filters from list based on PF response. + **/ +static void iavf_vlan_add_reject(struct iavf_adapter *adapter) +{ + struct iavf_vlan_filter *f, *ftmp; + + spin_lock_bh(&adapter->mac_vlan_list_lock); + list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) { + if (f->is_new_vlan) { + if (f->vlan.tpid == ETH_P_8021Q) + clear_bit(f->vlan.vid, + adapter->vsi.active_cvlans); + else + clear_bit(f->vlan.vid, + adapter->vsi.active_svlans); + + list_del(&f->list); + kfree(f); + } + } + spin_unlock_bh(&adapter->mac_vlan_list_lock); +} + +/** * iavf_add_vlans * @adapter: adapter structure * @@ -684,6 +711,7 @@ void iavf_add_vlans(struct iavf_adapter *adapter) vvfl->vlan_id[i] = f->vlan.vid; i++; f->add = false; + f->is_new_vlan = true; if (i == count) break; } @@ -696,10 +724,18 @@ void iavf_add_vlans(struct iavf_adapter *adapter) iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_VLAN, (u8 *)vvfl, len); kfree(vvfl); } else { + u16 max_vlans = adapter->vlan_v2_caps.filtering.max_filters; + u16 current_vlans = iavf_get_num_vlans_added(adapter); struct virtchnl_vlan_filter_list_v2 *vvfl_v2; adapter->current_op = VIRTCHNL_OP_ADD_VLAN_V2; + if ((count + current_vlans) > max_vlans && + current_vlans < max_vlans) { + count = max_vlans - iavf_get_num_vlans_added(adapter); + more = true; + } + len = sizeof(*vvfl_v2) + ((count - 1) * sizeof(struct virtchnl_vlan_filter)); if (len > IAVF_MAX_AQ_BUF_SIZE) { @@ -726,6 +762,9 @@ void iavf_add_vlans(struct iavf_adapter *adapter) &adapter->vlan_v2_caps.filtering.filtering_support; struct virtchnl_vlan *vlan; + if (i == count) + break; + /* give priority over outer if it's enabled */ if (filtering_support->outer) vlan = &vvfl_v2->filters[i].outer; @@ -737,8 +776,7 @@ void iavf_add_vlans(struct iavf_adapter *adapter) i++; f->add = false; - if (i == count) - break; + f->is_new_vlan = true; } } @@ -2082,6 +2120,11 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, */ iavf_netdev_features_vlan_strip_set(netdev, true); break; + case VIRTCHNL_OP_ADD_VLAN_V2: + iavf_vlan_add_reject(adapter); + dev_warn(&adapter->pdev->dev, "Failed to add VLAN filter, error %s\n", + iavf_stat_str(&adapter->hw, v_retval)); + break; default: dev_err(&adapter->pdev->dev, "PF returned error %d (%s) to our request %d\n", v_retval, iavf_stat_str(&adapter->hw, v_retval), @@ -2385,6 +2428,24 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, spin_unlock_bh(&adapter->adv_rss_lock); } break; + case VIRTCHNL_OP_ADD_VLAN_V2: { + struct iavf_vlan_filter *f; + + spin_lock_bh(&adapter->mac_vlan_list_lock); + list_for_each_entry(f, &adapter->vlan_filter_list, list) { + if (f->is_new_vlan) { + f->is_new_vlan = false; + if (f->vlan.tpid == ETH_P_8021Q) + set_bit(f->vlan.vid, + adapter->vsi.active_cvlans); + else + set_bit(f->vlan.vid, + adapter->vsi.active_svlans); + } + } + spin_unlock_bh(&adapter->mac_vlan_list_lock); + } + break; case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING: /* PF enabled vlan strip on this VF. * Update netdev->features if needed to be in sync with ethtool. diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index 60453b3b8d23..1a2e54dbc5a1 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -181,6 +181,7 @@ enum ice_feature { ICE_F_DSCP, + ICE_F_PTP_EXTTS, ICE_F_SMA_CTRL, ICE_F_GNSS, ICE_F_MAX @@ -544,8 +545,8 @@ struct ice_pf { u32 msg_enable; struct ice_ptp ptp; struct tty_driver *ice_gnss_tty_driver; - struct tty_port gnss_tty_port; - struct gnss_serial *gnss_serial; + struct tty_port *gnss_tty_port[ICE_GNSS_TTY_MINOR_DEVICES]; + struct gnss_serial *gnss_serial[ICE_GNSS_TTY_MINOR_DEVICES]; u16 num_rdma_msix; /* Total MSIX vectors for RDMA driver */ u16 rdma_base_vector; diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h index 05cb9dd7035a..9939238573a4 100644 --- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h @@ -1395,7 +1395,7 @@ struct ice_aqc_get_link_topo { u8 rsvd[9]; }; -/* Read I2C (direct, 0x06E2) */ +/* Read/Write I2C (direct, 0x06E2/0x06E3) */ struct ice_aqc_i2c { struct ice_aqc_link_topo_addr topo_addr; __le16 i2c_addr; @@ -1405,7 +1405,7 @@ struct ice_aqc_i2c { u8 rsvd; __le16 i2c_bus_addr; - u8 rsvd2[4]; + u8 i2c_data[4]; /* Used only by write command, reserved in read. */ }; /* Read I2C Response (direct, 0x06E2) */ @@ -2124,7 +2124,7 @@ struct ice_aq_desc { struct ice_aqc_get_link_status get_link_status; struct ice_aqc_event_lan_overflow lan_overflow; struct ice_aqc_get_link_topo get_link_topo; - struct ice_aqc_i2c read_i2c; + struct ice_aqc_i2c read_write_i2c; struct ice_aqc_read_i2c_resp read_i2c_resp; } params; }; @@ -2241,6 +2241,7 @@ enum ice_adminq_opc { ice_aqc_opc_set_mac_lb = 0x0620, ice_aqc_opc_get_link_topo = 0x06E0, ice_aqc_opc_read_i2c = 0x06E2, + ice_aqc_opc_write_i2c = 0x06E3, ice_aqc_opc_set_port_id_led = 0x06E9, ice_aqc_opc_set_gpio = 0x06EC, ice_aqc_opc_get_gpio = 0x06ED, diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index 9619bdb9e49a..27d0cbbd29da 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -4823,7 +4823,7 @@ ice_aq_read_i2c(struct ice_hw *hw, struct ice_aqc_link_topo_addr topo_addr, int status; ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_read_i2c); - cmd = &desc.params.read_i2c; + cmd = &desc.params.read_write_i2c; if (!data) return -EINVAL; @@ -4851,6 +4851,51 @@ ice_aq_read_i2c(struct ice_hw *hw, struct ice_aqc_link_topo_addr topo_addr, } /** + * ice_aq_write_i2c + * @hw: pointer to the hw struct + * @topo_addr: topology address for a device to communicate with + * @bus_addr: 7-bit I2C bus address + * @addr: I2C memory address (I2C offset) with up to 16 bits + * @params: I2C parameters: bit [4] - I2C address type, bits [3:0] - data size to write (0-7 bytes) + * @data: pointer to data (0 to 4 bytes) to be written to the I2C device + * @cd: pointer to command details structure or NULL + * + * Write I2C (0x06E3) + * + * * Return: + * * 0 - Successful write to the i2c device + * * -EINVAL - Data size greater than 4 bytes + * * -EIO - FW error + */ +int +ice_aq_write_i2c(struct ice_hw *hw, struct ice_aqc_link_topo_addr topo_addr, + u16 bus_addr, __le16 addr, u8 params, u8 *data, + struct ice_sq_cd *cd) +{ + struct ice_aq_desc desc = { 0 }; + struct ice_aqc_i2c *cmd; + u8 data_size; + + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_write_i2c); + cmd = &desc.params.read_write_i2c; + + data_size = FIELD_GET(ICE_AQC_I2C_DATA_SIZE_M, params); + + /* data_size limited to 4 */ + if (data_size > 4) + return -EINVAL; + + cmd->i2c_bus_addr = cpu_to_le16(bus_addr); + cmd->topo_addr = topo_addr; + cmd->i2c_params = params; + cmd->i2c_addr = addr; + + memcpy(cmd->i2c_data, data, data_size); + + return ice_aq_send_cmd(hw, &desc, NULL, 0, cd); +} + +/** * ice_aq_set_driver_param - Set driver parameter to share via firmware * @hw: pointer to the HW struct * @idx: parameter index to set diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h index 872ea7d2332d..61b7c60db689 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.h +++ b/drivers/net/ethernet/intel/ice/ice_common.h @@ -214,5 +214,9 @@ int ice_aq_read_i2c(struct ice_hw *hw, struct ice_aqc_link_topo_addr topo_addr, u16 bus_addr, __le16 addr, u8 params, u8 *data, struct ice_sq_cd *cd); +int +ice_aq_write_i2c(struct ice_hw *hw, struct ice_aqc_link_topo_addr topo_addr, + u16 bus_addr, __le16 addr, u8 params, u8 *data, + struct ice_sq_cd *cd); bool ice_fw_supports_report_dflt_cfg(struct ice_hw *hw); #endif /* _ICE_COMMON_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_gnss.c b/drivers/net/ethernet/intel/ice/ice_gnss.c index c6d755f707aa..b5a7f246d230 100644 --- a/drivers/net/ethernet/intel/ice/ice_gnss.c +++ b/drivers/net/ethernet/intel/ice/ice_gnss.c @@ -1,11 +1,104 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2018-2021, Intel Corporation. */ +/* Copyright (C) 2021-2022, Intel Corporation. */ #include "ice.h" #include "ice_lib.h" #include <linux/tty_driver.h> /** + * ice_gnss_do_write - Write data to internal GNSS + * @pf: board private structure + * @buf: command buffer + * @size: command buffer size + * + * Write UBX command data to the GNSS receiver + */ +static unsigned int +ice_gnss_do_write(struct ice_pf *pf, unsigned char *buf, unsigned int size) +{ + struct ice_aqc_link_topo_addr link_topo; + struct ice_hw *hw = &pf->hw; + unsigned int offset = 0; + int err = 0; + + memset(&link_topo, 0, sizeof(struct ice_aqc_link_topo_addr)); + link_topo.topo_params.index = ICE_E810T_GNSS_I2C_BUS; + link_topo.topo_params.node_type_ctx |= + FIELD_PREP(ICE_AQC_LINK_TOPO_NODE_CTX_M, + ICE_AQC_LINK_TOPO_NODE_CTX_OVERRIDE); + + /* It's not possible to write a single byte to u-blox. + * Write all bytes in a loop until there are 6 or less bytes left. If + * there are exactly 6 bytes left, the last write would be only a byte. + * In this case, do 4+2 bytes writes instead of 5+1. Otherwise, do the + * last 2 to 5 bytes write. + */ + while (size - offset > ICE_GNSS_UBX_WRITE_BYTES + 1) { + err = ice_aq_write_i2c(hw, link_topo, ICE_GNSS_UBX_I2C_BUS_ADDR, + cpu_to_le16(buf[offset]), + ICE_MAX_I2C_WRITE_BYTES, + &buf[offset + 1], NULL); + if (err) + goto err_out; + + offset += ICE_GNSS_UBX_WRITE_BYTES; + } + + /* Single byte would be written. Write 4 bytes instead of 5. */ + if (size - offset == ICE_GNSS_UBX_WRITE_BYTES + 1) { + err = ice_aq_write_i2c(hw, link_topo, ICE_GNSS_UBX_I2C_BUS_ADDR, + cpu_to_le16(buf[offset]), + ICE_MAX_I2C_WRITE_BYTES - 1, + &buf[offset + 1], NULL); + if (err) + goto err_out; + + offset += ICE_GNSS_UBX_WRITE_BYTES - 1; + } + + /* Do the last write, 2 to 5 bytes. */ + err = ice_aq_write_i2c(hw, link_topo, ICE_GNSS_UBX_I2C_BUS_ADDR, + cpu_to_le16(buf[offset]), size - offset - 1, + &buf[offset + 1], NULL); + if (err) + goto err_out; + + return size; + +err_out: + dev_err(ice_pf_to_dev(pf), "GNSS failed to write, offset=%u, size=%u, err=%d\n", + offset, size, err); + + return offset; +} + +/** + * ice_gnss_write_pending - Write all pending data to internal GNSS + * @work: GNSS write work structure + */ +static void ice_gnss_write_pending(struct kthread_work *work) +{ + struct gnss_serial *gnss = container_of(work, struct gnss_serial, + write_work); + struct ice_pf *pf = gnss->back; + + if (!list_empty(&gnss->queue)) { + struct gnss_write_buf *write_buf = NULL; + unsigned int bytes; + + write_buf = list_first_entry(&gnss->queue, + struct gnss_write_buf, queue); + + bytes = ice_gnss_do_write(pf, write_buf->buf, write_buf->size); + dev_dbg(ice_pf_to_dev(pf), "%u bytes written to GNSS\n", bytes); + + list_del(&write_buf->queue); + kfree(write_buf->buf); + kfree(write_buf); + } +} + +/** * ice_gnss_read - Read data from internal GNSS module * @work: GNSS read work structure * @@ -104,8 +197,9 @@ exit: /** * ice_gnss_struct_init - Initialize GNSS structure for the TTY * @pf: Board private structure + * @index: TTY device index */ -static struct gnss_serial *ice_gnss_struct_init(struct ice_pf *pf) +static struct gnss_serial *ice_gnss_struct_init(struct ice_pf *pf, int index) { struct device *dev = ice_pf_to_dev(pf); struct kthread_worker *kworker; @@ -118,9 +212,11 @@ static struct gnss_serial *ice_gnss_struct_init(struct ice_pf *pf) mutex_init(&gnss->gnss_mutex); gnss->open_count = 0; gnss->back = pf; - pf->gnss_serial = gnss; + pf->gnss_serial[index] = gnss; kthread_init_delayed_work(&gnss->read_work, ice_gnss_read); + INIT_LIST_HEAD(&gnss->queue); + kthread_init_work(&gnss->write_work, ice_gnss_write_pending); /* Allocate a kworker for handling work required for the GNSS TTY * writes. */ @@ -156,10 +252,10 @@ static int ice_gnss_tty_open(struct tty_struct *tty, struct file *filp) tty->driver_data = NULL; /* Get the serial object associated with this tty pointer */ - gnss = pf->gnss_serial; + gnss = pf->gnss_serial[tty->index]; if (!gnss) { /* Initialize GNSS struct on the first device open */ - gnss = ice_gnss_struct_init(pf); + gnss = ice_gnss_struct_init(pf, tty->index); if (!gnss) return -ENOMEM; } @@ -212,25 +308,100 @@ exit: } /** - * ice_gnss_tty_write - Dummy TTY write function to avoid kernel panic + * ice_gnss_tty_write - Write GNSS data * @tty: pointer to the tty_struct * @buf: pointer to the user data - * @cnt: the number of characters that was able to be sent to the hardware (or - * queued to be sent at a later time) + * @count: the number of characters queued to be sent to the HW + * + * The write function call is called by the user when there is data to be sent + * to the hardware. First the tty core receives the call, and then it passes the + * data on to the tty driver's write function. The tty core also tells the tty + * driver the size of the data being sent. + * If any errors happen during the write call, a negative error value should be + * returned instead of the number of characters queued to be written. */ static int -ice_gnss_tty_write(struct tty_struct *tty, const unsigned char *buf, int cnt) +ice_gnss_tty_write(struct tty_struct *tty, const unsigned char *buf, int count) { - return 0; + struct gnss_write_buf *write_buf; + struct gnss_serial *gnss; + unsigned char *cmd_buf; + struct ice_pf *pf; + int err = count; + + /* We cannot write a single byte using our I2C implementation. */ + if (count <= 1 || count > ICE_GNSS_TTY_WRITE_BUF) + return -EINVAL; + + gnss = tty->driver_data; + if (!gnss) + return -EFAULT; + + pf = (struct ice_pf *)tty->driver->driver_state; + if (!pf) + return -EFAULT; + + /* Only allow to write on TTY 0 */ + if (gnss != pf->gnss_serial[0]) + return -EIO; + + mutex_lock(&gnss->gnss_mutex); + + if (!gnss->open_count) { + err = -EINVAL; + goto exit; + } + + cmd_buf = kcalloc(count, sizeof(*buf), GFP_KERNEL); + if (!cmd_buf) { + err = -ENOMEM; + goto exit; + } + + memcpy(cmd_buf, buf, count); + + /* Send the data out to a hardware port */ + write_buf = kzalloc(sizeof(*write_buf), GFP_KERNEL); + if (!write_buf) { + err = -ENOMEM; + goto exit; + } + + write_buf->buf = cmd_buf; + write_buf->size = count; + INIT_LIST_HEAD(&write_buf->queue); + list_add_tail(&write_buf->queue, &gnss->queue); + kthread_queue_work(gnss->kworker, &gnss->write_work); +exit: + mutex_unlock(&gnss->gnss_mutex); + return err; } /** - * ice_gnss_tty_write_room - Dummy TTY write_room function to avoid kernel panic + * ice_gnss_tty_write_room - Returns the numbers of characters to be written. * @tty: pointer to the tty_struct + * + * This routine returns the numbers of characters the tty driver will accept + * for queuing to be written or 0 if either the TTY is not open or user + * tries to write to the TTY other than the first. */ static unsigned int ice_gnss_tty_write_room(struct tty_struct *tty) { - return 0; + struct gnss_serial *gnss = tty->driver_data; + + /* Only allow to write on TTY 0 */ + if (!gnss || gnss != gnss->back->gnss_serial[0]) + return 0; + + mutex_lock(&gnss->gnss_mutex); + + if (!gnss->open_count) { + mutex_unlock(&gnss->gnss_mutex); + return 0; + } + + mutex_unlock(&gnss->gnss_mutex); + return ICE_GNSS_TTY_WRITE_BUF; } static const struct tty_operations tty_gps_ops = { @@ -250,11 +421,13 @@ static struct tty_driver *ice_gnss_create_tty_driver(struct ice_pf *pf) const int ICE_TTYDRV_NAME_MAX = 14; struct tty_driver *tty_driver; char *ttydrv_name; + unsigned int i; int err; - tty_driver = tty_alloc_driver(1, TTY_DRIVER_REAL_RAW); + tty_driver = tty_alloc_driver(ICE_GNSS_TTY_MINOR_DEVICES, + TTY_DRIVER_REAL_RAW); if (IS_ERR(tty_driver)) { - dev_err(ice_pf_to_dev(pf), "Failed to allocate memory for GNSS TTY\n"); + dev_err(dev, "Failed to allocate memory for GNSS TTY\n"); return NULL; } @@ -284,23 +457,32 @@ static struct tty_driver *ice_gnss_create_tty_driver(struct ice_pf *pf) tty_driver->driver_state = pf; tty_set_operations(tty_driver, &tty_gps_ops); - pf->gnss_serial = NULL; + for (i = 0; i < ICE_GNSS_TTY_MINOR_DEVICES; i++) { + pf->gnss_tty_port[i] = kzalloc(sizeof(*pf->gnss_tty_port[i]), + GFP_KERNEL); + pf->gnss_serial[i] = NULL; - tty_port_init(&pf->gnss_tty_port); - tty_port_link_device(&pf->gnss_tty_port, tty_driver, 0); + tty_port_init(pf->gnss_tty_port[i]); + tty_port_link_device(pf->gnss_tty_port[i], tty_driver, i); + } err = tty_register_driver(tty_driver); if (err) { - dev_err(ice_pf_to_dev(pf), "Failed to register TTY driver err=%d\n", - err); + dev_err(dev, "Failed to register TTY driver err=%d\n", err); - tty_port_destroy(&pf->gnss_tty_port); + for (i = 0; i < ICE_GNSS_TTY_MINOR_DEVICES; i++) { + tty_port_destroy(pf->gnss_tty_port[i]); + kfree(pf->gnss_tty_port[i]); + } kfree(ttydrv_name); tty_driver_kref_put(pf->ice_gnss_tty_driver); return NULL; } + for (i = 0; i < ICE_GNSS_TTY_MINOR_DEVICES; i++) + dev_info(dev, "%s%d registered\n", ttydrv_name, i); + return tty_driver; } @@ -328,17 +510,25 @@ void ice_gnss_init(struct ice_pf *pf) */ void ice_gnss_exit(struct ice_pf *pf) { + unsigned int i; + if (!test_bit(ICE_FLAG_GNSS, pf->flags) || !pf->ice_gnss_tty_driver) return; - tty_port_destroy(&pf->gnss_tty_port); + for (i = 0; i < ICE_GNSS_TTY_MINOR_DEVICES; i++) { + if (pf->gnss_tty_port[i]) { + tty_port_destroy(pf->gnss_tty_port[i]); + kfree(pf->gnss_tty_port[i]); + } - if (pf->gnss_serial) { - struct gnss_serial *gnss = pf->gnss_serial; + if (pf->gnss_serial[i]) { + struct gnss_serial *gnss = pf->gnss_serial[i]; - kthread_cancel_delayed_work_sync(&gnss->read_work); - kfree(gnss); - pf->gnss_serial = NULL; + kthread_cancel_work_sync(&gnss->write_work); + kthread_cancel_delayed_work_sync(&gnss->read_work); + kfree(gnss); + pf->gnss_serial[i] = NULL; + } } tty_unregister_driver(pf->ice_gnss_tty_driver); diff --git a/drivers/net/ethernet/intel/ice/ice_gnss.h b/drivers/net/ethernet/intel/ice/ice_gnss.h index 9211adb2372c..f454dd1d9285 100644 --- a/drivers/net/ethernet/intel/ice/ice_gnss.h +++ b/drivers/net/ethernet/intel/ice/ice_gnss.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2018-2021, Intel Corporation. */ +/* Copyright (C) 2021-2022, Intel Corporation. */ #ifndef _ICE_GNSS_H_ #define _ICE_GNSS_H_ @@ -8,14 +8,34 @@ #include <linux/tty_flip.h> #define ICE_E810T_GNSS_I2C_BUS 0x2 +#define ICE_GNSS_TIMER_DELAY_TIME (HZ / 10) /* 0.1 second per message */ +/* Create 2 minor devices, both using the same GNSS module. First one is RW, + * second one RO. + */ +#define ICE_GNSS_TTY_MINOR_DEVICES 2 +#define ICE_GNSS_TTY_WRITE_BUF 250 +#define ICE_MAX_I2C_DATA_SIZE FIELD_MAX(ICE_AQC_I2C_DATA_SIZE_M) +#define ICE_MAX_I2C_WRITE_BYTES 4 + +/* u-blox ZED-F9T specific definitions */ #define ICE_GNSS_UBX_I2C_BUS_ADDR 0x42 /* Data length register is big endian */ #define ICE_GNSS_UBX_DATA_LEN_H 0xFD #define ICE_GNSS_UBX_DATA_LEN_WIDTH 2 #define ICE_GNSS_UBX_EMPTY_DATA 0xFF -#define ICE_GNSS_TIMER_DELAY_TIME (HZ / 10) /* 0.1 second per message */ -#define ICE_MAX_I2C_DATA_SIZE FIELD_MAX(ICE_AQC_I2C_DATA_SIZE_M) +/* For u-blox writes are performed without address so the first byte to write is + * passed as I2C addr parameter. + */ +#define ICE_GNSS_UBX_WRITE_BYTES (ICE_MAX_I2C_WRITE_BYTES + 1) #define ICE_MAX_UBX_READ_TRIES 255 +#define ICE_MAX_UBX_ACK_READ_TRIES 4095 + +struct gnss_write_buf { + struct list_head queue; + unsigned int size; + unsigned char *buf; +}; + /** * struct gnss_serial - data used to initialize GNSS TTY port @@ -25,6 +45,8 @@ * @gnss_mutex: gnss_mutex used to protect GNSS serial operations * @kworker: kwork thread for handling periodic work * @read_work: read_work function for handling GNSS reads + * @write_work: write_work function for handling GNSS writes + * @queue: write buffers queue */ struct gnss_serial { struct ice_pf *back; @@ -33,6 +55,8 @@ struct gnss_serial { struct mutex gnss_mutex; /* protects GNSS serial structure */ struct kthread_worker *kworker; struct kthread_delayed_work read_work; + struct kthread_work write_work; + struct list_head queue; }; #if IS_ENABLED(CONFIG_TTY) diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index a6c4be5e5566..bc357dfae306 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -4182,6 +4182,7 @@ void ice_init_feature_support(struct ice_pf *pf) case ICE_DEV_ID_E810C_QSFP: case ICE_DEV_ID_E810C_SFP: ice_set_feature_support(pf, ICE_F_DSCP); + ice_set_feature_support(pf, ICE_F_PTP_EXTTS); if (ice_is_e810t(&pf->hw)) { ice_set_feature_support(pf, ICE_F_SMA_CTRL); if (ice_gnss_is_gps_present(&pf->hw)) diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index ff2eac2f8c64..313716615e98 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -5313,12 +5313,6 @@ static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev) result = PCI_ERS_RESULT_DISCONNECT; } - err = pci_aer_clear_nonfatal_status(pdev); - if (err) - dev_dbg(&pdev->dev, "pci_aer_clear_nonfatal_status() failed, error %d\n", - err); - /* non-fatal, continue */ - return result; } diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c index ef9344ef0d8e..29c7a0ccb3c4 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp.c @@ -1900,9 +1900,12 @@ ice_ptp_setup_pins_e810t(struct ice_pf *pf, struct ptp_clock_info *info) } info->n_per_out = N_PER_OUT_E810T; - info->n_ext_ts = N_EXT_TS_E810; - info->n_pins = NUM_PTP_PINS_E810T; - info->verify = ice_verify_pin_e810t; + + if (ice_is_feature_supported(pf, ICE_F_PTP_EXTTS)) { + info->n_ext_ts = N_EXT_TS_E810; + info->n_pins = NUM_PTP_PINS_E810T; + info->verify = ice_verify_pin_e810t; + } /* Complete setup of the SMA pins */ ice_ptp_setup_sma_pins_e810t(pf, info); @@ -1910,11 +1913,16 @@ ice_ptp_setup_pins_e810t(struct ice_pf *pf, struct ptp_clock_info *info) /** * ice_ptp_setup_pins_e810 - Setup PTP pins in sysfs + * @pf: pointer to the PF instance * @info: PTP clock capabilities */ -static void ice_ptp_setup_pins_e810(struct ptp_clock_info *info) +static void ice_ptp_setup_pins_e810(struct ice_pf *pf, struct ptp_clock_info *info) { info->n_per_out = N_PER_OUT_E810; + + if (!ice_is_feature_supported(pf, ICE_F_PTP_EXTTS)) + return; + info->n_ext_ts = N_EXT_TS_E810; } @@ -1956,7 +1964,7 @@ ice_ptp_set_funcs_e810(struct ice_pf *pf, struct ptp_clock_info *info) if (ice_is_e810t(&pf->hw)) ice_ptp_setup_pins_e810t(pf, info); else - ice_ptp_setup_pins_e810(info); + ice_ptp_setup_pins_e810(pf, info); } /** diff --git a/drivers/net/ethernet/intel/igc/igc_hw.h b/drivers/net/ethernet/intel/igc/igc_hw.h index 360644f33d5f..88680e3d613d 100644 --- a/drivers/net/ethernet/intel/igc/igc_hw.h +++ b/drivers/net/ethernet/intel/igc/igc_hw.h @@ -89,8 +89,6 @@ struct igc_mac_info { u32 mta_shadow[MAX_MTA_REG]; u16 rar_entry_count; - u8 forced_speed_duplex; - bool asf_firmware_present; bool arc_subsystem_valid; diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index ae17af44fe02..ebff0e04045d 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -5813,9 +5813,10 @@ static bool validate_schedule(struct igc_adapter *adapter, return false; for (n = 0; n < qopt->num_entries; n++) { - const struct tc_taprio_sched_entry *e; + const struct tc_taprio_sched_entry *e, *prev; int i; + prev = n ? &qopt->entries[n - 1] : NULL; e = &qopt->entries[n]; /* i225 only supports "global" frame preemption @@ -5828,7 +5829,12 @@ static bool validate_schedule(struct igc_adapter *adapter, if (e->gate_mask & BIT(i)) queue_uses[i]++; - if (queue_uses[i] > 1) + /* There are limitations: A single queue cannot be + * opened and closed multiple times per cycle unless the + * gate stays open. Check for it. + */ + if (queue_uses[i] > 1 && + !(prev->gate_mask & BIT(i))) return false; } } @@ -5872,6 +5878,7 @@ static int igc_tsn_clear_schedule(struct igc_adapter *adapter) static int igc_save_qbv_schedule(struct igc_adapter *adapter, struct tc_taprio_qopt_offload *qopt) { + bool queue_configured[IGC_MAX_TX_QUEUES] = { }; u32 start_time = 0, end_time = 0; size_t n; @@ -5887,9 +5894,6 @@ static int igc_save_qbv_schedule(struct igc_adapter *adapter, adapter->cycle_time = qopt->cycle_time; adapter->base_time = qopt->base_time; - /* FIXME: be a little smarter about cases when the gate for a - * queue stays open for more than one entry. - */ for (n = 0; n < qopt->num_entries; n++) { struct tc_taprio_sched_entry *e = &qopt->entries[n]; int i; @@ -5902,8 +5906,15 @@ static int igc_save_qbv_schedule(struct igc_adapter *adapter, if (!(e->gate_mask & BIT(i))) continue; - ring->start_time = start_time; + /* Check whether a queue stays open for more than one + * entry. If so, keep the start and advance the end + * time. + */ + if (!queue_configured[i]) + ring->start_time = start_time; ring->end_time = end_time; + + queue_configured[i] = true; } start_time += e->interval; @@ -6171,6 +6182,9 @@ u32 igc_rd32(struct igc_hw *hw, u32 reg) u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); u32 value = 0; + if (IGC_REMOVED(hw_addr)) + return ~value; + value = readl(&hw_addr[reg]); /* reads should not return all F's */ diff --git a/drivers/net/ethernet/intel/igc/igc_regs.h b/drivers/net/ethernet/intel/igc/igc_regs.h index e197a33d93a0..c0d8214148d1 100644 --- a/drivers/net/ethernet/intel/igc/igc_regs.h +++ b/drivers/net/ethernet/intel/igc/igc_regs.h @@ -59,9 +59,6 @@ #define IGC_IVAR_MISC 0x01740 /* IVAR for "other" causes - RW */ #define IGC_GPIE 0x01514 /* General Purpose Intr Enable - RW */ -/* MSI-X Table Register Descriptions */ -#define IGC_PBACL 0x05B68 /* MSIx PBA Clear - R/W 1 to clear */ - /* RSS registers */ #define IGC_MRQC 0x05818 /* Multiple Receive Control - RW */ @@ -306,7 +303,8 @@ u32 igc_rd32(struct igc_hw *hw, u32 reg); #define wr32(reg, val) \ do { \ u8 __iomem *hw_addr = READ_ONCE((hw)->hw_addr); \ - writel((val), &hw_addr[(reg)]); \ + if (!IGC_REMOVED(hw_addr)) \ + writel((val), &hw_addr[(reg)]); \ } while (0) #define rd32(reg) (igc_rd32(hw, reg)) @@ -318,4 +316,6 @@ do { \ #define array_rd32(reg, offset) (igc_rd32(hw, (reg) + ((offset) << 2))) +#define IGC_REMOVED(h) unlikely(!(h)) + #endif diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index 48444ab9e0b1..5369a97ff5ec 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -813,6 +813,7 @@ struct ixgbe_adapter { #ifdef CONFIG_IXGBE_IPSEC struct ixgbe_ipsec *ipsec; #endif /* CONFIG_IXGBE_IPSEC */ + spinlock_t vfs_lock; }; static inline int ixgbe_determine_xdp_q_idx(int cpu) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 0493326a0d6b..d1e430b8c8aa 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -6445,6 +6445,9 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter, /* n-tuple support exists, always init our spinlock */ spin_lock_init(&adapter->fdir_perfect_lock); + /* init spinlock to avoid concurrency of VF resources */ + spin_lock_init(&adapter->vfs_lock); + #ifdef CONFIG_IXGBE_DCB ixgbe_init_dcb(adapter); #endif diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c index 67e49aac50fe..29cc60988071 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c @@ -205,10 +205,13 @@ void ixgbe_enable_sriov(struct ixgbe_adapter *adapter, unsigned int max_vfs) int ixgbe_disable_sriov(struct ixgbe_adapter *adapter) { unsigned int num_vfs = adapter->num_vfs, vf; + unsigned long flags; int rss; + spin_lock_irqsave(&adapter->vfs_lock, flags); /* set num VFs to 0 to prevent access to vfinfo */ adapter->num_vfs = 0; + spin_unlock_irqrestore(&adapter->vfs_lock, flags); /* put the reference to all of the vf devices */ for (vf = 0; vf < num_vfs; ++vf) { @@ -1355,8 +1358,10 @@ static void ixgbe_rcv_ack_from_vf(struct ixgbe_adapter *adapter, u32 vf) void ixgbe_msg_task(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; + unsigned long flags; u32 vf; + spin_lock_irqsave(&adapter->vfs_lock, flags); for (vf = 0; vf < adapter->num_vfs; vf++) { /* process any reset requests */ if (!ixgbe_check_for_rst(hw, vf)) @@ -1370,6 +1375,7 @@ void ixgbe_msg_task(struct ixgbe_adapter *adapter) if (!ixgbe_check_for_ack(hw, vf)) ixgbe_rcv_ack_from_vf(adapter, vf); } + spin_unlock_irqrestore(&adapter->vfs_lock, flags); } static inline void ixgbe_ping_vf(struct ixgbe_adapter *adapter, int vf) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c index 85eb4a6f4fcf..c8724bfa86b0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c @@ -498,6 +498,32 @@ static u8 cgx_get_lmac_type(void *cgxd, int lmac_id) return (cfg >> CGX_LMAC_TYPE_SHIFT) & CGX_LMAC_TYPE_MASK; } +static u32 cgx_get_lmac_fifo_len(void *cgxd, int lmac_id) +{ + struct cgx *cgx = cgxd; + u8 num_lmacs; + u32 fifo_len; + + fifo_len = cgx->mac_ops->fifo_len; + num_lmacs = cgx->mac_ops->get_nr_lmacs(cgx); + + switch (num_lmacs) { + case 1: + return fifo_len; + case 2: + return fifo_len / 2; + case 3: + /* LMAC0 gets half of the FIFO, reset 1/4th */ + if (lmac_id == 0) + return fifo_len / 2; + return fifo_len / 4; + case 4: + default: + return fifo_len / 4; + } + return 0; +} + /* Configure CGX LMAC in internal loopback mode */ int cgx_lmac_internal_loopback(void *cgxd, int lmac_id, bool enable) { @@ -1704,6 +1730,7 @@ static struct mac_ops cgx_mac_ops = { .tx_stats_cnt = 18, .get_nr_lmacs = cgx_get_nr_lmacs, .get_lmac_type = cgx_get_lmac_type, + .lmac_fifo_len = cgx_get_lmac_fifo_len, .mac_lmac_intl_lbk = cgx_lmac_internal_loopback, .mac_get_rx_stats = cgx_get_rx_stats, .mac_get_tx_stats = cgx_get_tx_stats, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h b/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h index f30581bf0688..52b6016789fa 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h @@ -80,6 +80,7 @@ struct mac_ops { */ int (*get_nr_lmacs)(void *cgx); u8 (*get_lmac_type)(void *cgx, int lmac_id); + u32 (*lmac_fifo_len)(void *cgx, int lmac_id); int (*mac_lmac_intl_lbk)(void *cgx, int lmac_id, bool enable); /* Register Stats related functions */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c index 9025825597a2..ef59de43b11e 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c @@ -22,6 +22,7 @@ static struct mac_ops rpm_mac_ops = { .tx_stats_cnt = 34, .get_nr_lmacs = rpm_get_nr_lmacs, .get_lmac_type = rpm_get_lmac_type, + .lmac_fifo_len = rpm_get_lmac_fifo_len, .mac_lmac_intl_lbk = rpm_lmac_internal_loopback, .mac_get_rx_stats = rpm_get_rx_stats, .mac_get_tx_stats = rpm_get_tx_stats, @@ -350,6 +351,35 @@ u8 rpm_get_lmac_type(void *rpmd, int lmac_id) return err; } +u32 rpm_get_lmac_fifo_len(void *rpmd, int lmac_id) +{ + rpm_t *rpm = rpmd; + u64 hi_perf_lmac; + u8 num_lmacs; + u32 fifo_len; + + fifo_len = rpm->mac_ops->fifo_len; + num_lmacs = rpm->mac_ops->get_nr_lmacs(rpm); + + switch (num_lmacs) { + case 1: + return fifo_len; + case 2: + return fifo_len / 2; + case 3: + /* LMAC marked as hi_perf gets half of the FIFO and rest 1/4th */ + hi_perf_lmac = rpm_read(rpm, 0, CGXX_CMRX_RX_LMACS); + hi_perf_lmac = (hi_perf_lmac >> 4) & 0x3ULL; + if (lmac_id == hi_perf_lmac) + return fifo_len / 2; + return fifo_len / 4; + case 4: + default: + return fifo_len / 4; + } + return 0; +} + int rpm_lmac_internal_loopback(void *rpmd, int lmac_id, bool enable) { rpm_t *rpm = rpmd; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h index 8247db8c1d36..c2bd6e54ea51 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h @@ -75,6 +75,7 @@ /* Function Declarations */ int rpm_get_nr_lmacs(void *rpmd); u8 rpm_get_lmac_type(void *rpmd, int lmac_id); +u32 rpm_get_lmac_fifo_len(void *rpmd, int lmac_id); int rpm_lmac_internal_loopback(void *rpmd, int lmac_id, bool enable); void rpm_lmac_enadis_rx_pause_fwding(void *rpmd, int lmac_id, bool enable); int rpm_lmac_get_pause_frm_status(void *cgxd, int lmac_id, u8 *tx_pause, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index e5fdb7b62651..d15bc443335d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -827,7 +827,7 @@ int rvu_cgx_config_tx(void *cgxd, int lmac_id, bool enable); int rvu_cgx_prio_flow_ctrl_cfg(struct rvu *rvu, u16 pcifunc, u8 tx_pause, u8 rx_pause, u16 pfc_en); int rvu_cgx_cfg_pause_frm(struct rvu *rvu, u16 pcifunc, u8 tx_pause, u8 rx_pause); - +u32 rvu_cgx_get_lmac_fifolen(struct rvu *rvu, int cgx, int lmac); int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, u16 pcifunc, int nixlf, int type); bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c index 41c124b7aea9..addc69f4b65c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c @@ -861,6 +861,22 @@ u32 rvu_cgx_get_fifolen(struct rvu *rvu) return fifo_len; } +u32 rvu_cgx_get_lmac_fifolen(struct rvu *rvu, int cgx, int lmac) +{ + struct mac_ops *mac_ops; + void *cgxd; + + cgxd = rvu_cgx_pdata(cgx, rvu); + if (!cgxd) + return 0; + + mac_ops = get_mac_ops(cgxd); + if (!mac_ops->lmac_fifo_len) + return 0; + + return mac_ops->lmac_fifo_len(cgxd, lmac); +} + static int rvu_cgx_config_intlbk(struct rvu *rvu, u16 pcifunc, bool en) { int pf = rvu_get_pf(pcifunc); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 1d3323da6930..0879a48411f3 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -4010,9 +4010,13 @@ linkcfg: return 0; /* Update transmit credits for CGX links */ - lmac_fifo_len = - rvu_cgx_get_fifolen(rvu) / - cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu)); + lmac_fifo_len = rvu_cgx_get_lmac_fifolen(rvu, cgx, lmac); + if (!lmac_fifo_len) { + dev_err(rvu->dev, + "%s: Failed to get CGX/RPM%d:LMAC%d FIFO size\n", + __func__, cgx, lmac); + return 0; + } return nix_config_link_credits(rvu, blkaddr, link, pcifunc, (lmac_fifo_len - req->maxlen) / 16); } @@ -4064,7 +4068,10 @@ static void nix_link_config(struct rvu *rvu, int blkaddr, struct rvu_hwinfo *hw = rvu->hw; int cgx, lmac_cnt, slink, link; u16 lbk_max_frs, lmac_max_frs; + unsigned long lmac_bmap; u64 tx_credits, cfg; + u64 lmac_fifo_len; + int iter; rvu_get_lbk_link_max_frs(rvu, &lbk_max_frs); rvu_get_lmac_link_max_frs(rvu, &lmac_max_frs); @@ -4098,12 +4105,23 @@ static void nix_link_config(struct rvu *rvu, int blkaddr, /* Skip when cgx is not available or lmac cnt is zero */ if (lmac_cnt <= 0) continue; - tx_credits = ((rvu_cgx_get_fifolen(rvu) / lmac_cnt) - - lmac_max_frs) / 16; - /* Enable credits and set credit pkt count to max allowed */ - cfg = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1); slink = cgx * hw->lmac_per_cgx; - for (link = slink; link < (slink + lmac_cnt); link++) { + + /* Get LMAC id's from bitmap */ + lmac_bmap = cgx_get_lmac_bmap(rvu_cgx_pdata(cgx, rvu)); + for_each_set_bit(iter, &lmac_bmap, MAX_LMAC_PER_CGX) { + lmac_fifo_len = rvu_cgx_get_lmac_fifolen(rvu, cgx, iter); + if (!lmac_fifo_len) { + dev_err(rvu->dev, + "%s: Failed to get CGX/RPM%d:LMAC%d FIFO size\n", + __func__, cgx, iter); + continue; + } + tx_credits = (lmac_fifo_len - lmac_max_frs) / 16; + /* Enable credits and set credit pkt count to max allowed */ + cfg = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1); + + link = iter + slink; nix_hw->tx_credits[link] = tx_credits; rvu_write64(rvu, blkaddr, NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c index 1195b690f483..594029007f85 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c @@ -1199,8 +1199,9 @@ static int rvu_npc_exact_del_table_entry_by_id(struct rvu *rvu, u32 seq_id) struct npc_exact_table_entry *entry = NULL; struct npc_exact_table *table; bool disable_cam = false; - u32 drop_mcam_idx; + u32 drop_mcam_idx = -1; int *cnt; + bool rc; table = rvu->hw->table; @@ -1209,7 +1210,7 @@ static int rvu_npc_exact_del_table_entry_by_id(struct rvu *rvu, u32 seq_id) /* Lookup for entry which needs to be updated */ entry = __rvu_npc_exact_find_entry_by_seq_id(rvu, seq_id); if (!entry) { - dev_dbg(rvu->dev, "%s: failed to find entry for id=0x%x\n", __func__, seq_id); + dev_dbg(rvu->dev, "%s: failed to find entry for id=%d\n", __func__, seq_id); mutex_unlock(&table->lock); return -ENODATA; } @@ -1223,8 +1224,14 @@ static int rvu_npc_exact_del_table_entry_by_id(struct rvu *rvu, u32 seq_id) (*cnt)--; - rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, entry->cgx_id, entry->lmac_id, - &drop_mcam_idx, NULL, NULL, NULL); + rc = rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, entry->cgx_id, + entry->lmac_id, &drop_mcam_idx, NULL, NULL, NULL); + if (!rc) { + dev_dbg(rvu->dev, "%s: failed to retrieve drop info for id=0x%x\n", + __func__, seq_id); + mutex_unlock(&table->lock); + return -ENODATA; + } if (entry->cmd) __rvu_npc_exact_cmd_rules_cnt_update(rvu, drop_mcam_idx, -1, &disable_cam); @@ -1276,6 +1283,7 @@ static int rvu_npc_exact_add_table_entry(struct rvu *rvu, u8 cgx_id, u8 lmac_id, u32 drop_mcam_idx; u32 index; u64 mdata; + bool rc; int err; u8 ways; @@ -1304,8 +1312,15 @@ static int rvu_npc_exact_add_table_entry(struct rvu *rvu, u8 cgx_id, u8 lmac_id, return err; } - rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, cgx_id, lmac_id, - &drop_mcam_idx, NULL, NULL, NULL); + rc = rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, cgx_id, lmac_id, + &drop_mcam_idx, NULL, NULL, NULL); + if (!rc) { + rvu_npc_exact_dealloc_table_entry(rvu, opc_type, ways, index); + dev_dbg(rvu->dev, "%s: failed to get drop rule info cgx=%d lmac=%d\n", + __func__, cgx_id, lmac_id); + return -EINVAL; + } + if (cmd) __rvu_npc_exact_cmd_rules_cnt_update(rvu, drop_mcam_idx, 1, &enable_cam); @@ -1388,7 +1403,7 @@ static int rvu_npc_exact_update_table_entry(struct rvu *rvu, u8 cgx_id, u8 lmac_ dev_dbg(rvu->dev, "%s: Successfully updated entry (index=%d, dmac=%pM, ways=%d opc_type=%d\n", - __func__, hash_index, entry->mac, entry->ways, entry->opc_type); + __func__, entry->index, entry->mac, entry->ways, entry->opc_type); dev_dbg(rvu->dev, "%s: Successfully updated entry (old mac=%pM new_mac=%pM\n", __func__, old_mac, new_mac); @@ -1414,13 +1429,19 @@ int rvu_npc_exact_promisc_disable(struct rvu *rvu, u16 pcifunc) u8 cgx_id, lmac_id; u32 drop_mcam_idx; bool *promisc; + bool rc; u32 cnt; table = rvu->hw->table; rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); - rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, cgx_id, lmac_id, - &drop_mcam_idx, NULL, NULL, NULL); + rc = rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, cgx_id, lmac_id, + &drop_mcam_idx, NULL, NULL, NULL); + if (!rc) { + dev_dbg(rvu->dev, "%s: failed to get drop rule info cgx=%d lmac=%d\n", + __func__, cgx_id, lmac_id); + return -EINVAL; + } mutex_lock(&table->lock); promisc = &table->promisc_mode[drop_mcam_idx]; @@ -1459,13 +1480,19 @@ int rvu_npc_exact_promisc_enable(struct rvu *rvu, u16 pcifunc) u8 cgx_id, lmac_id; u32 drop_mcam_idx; bool *promisc; + bool rc; u32 cnt; table = rvu->hw->table; rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); - rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, cgx_id, lmac_id, - &drop_mcam_idx, NULL, NULL, NULL); + rc = rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, cgx_id, lmac_id, + &drop_mcam_idx, NULL, NULL, NULL); + if (!rc) { + dev_dbg(rvu->dev, "%s: failed to get drop rule info cgx=%d lmac=%d\n", + __func__, cgx_id, lmac_id); + return -EINVAL; + } mutex_lock(&table->lock); promisc = &table->promisc_mode[drop_mcam_idx]; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dmac_flt.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dmac_flt.c index 846a0294a215..80d853b343f9 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dmac_flt.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dmac_flt.c @@ -54,12 +54,19 @@ static int otx2_dmacflt_add_pfmac(struct otx2_nic *pf, u32 *dmac_index) ether_addr_copy(req->mac_addr, pf->netdev->dev_addr); err = otx2_sync_mbox_msg(&pf->mbox); - if (!err) { - rsp = (struct cgx_mac_addr_set_or_get *) - otx2_mbox_get_rsp(&pf->mbox.mbox, 0, &req->hdr); - *dmac_index = rsp->index; + if (err) + goto out; + + rsp = (struct cgx_mac_addr_set_or_get *) + otx2_mbox_get_rsp(&pf->mbox.mbox, 0, &req->hdr); + + if (IS_ERR_OR_NULL(rsp)) { + err = -EINVAL; + goto out; } + *dmac_index = rsp->index; +out: mutex_unlock(&pf->mbox.lock); return err; } @@ -154,6 +161,12 @@ int otx2_dmacflt_get_max_cnt(struct otx2_nic *pf) rsp = (struct cgx_max_dmac_entries_get_rsp *) otx2_mbox_get_rsp(&pf->mbox.mbox, 0, &msg->hdr); + + if (IS_ERR_OR_NULL(rsp)) { + err = -EINVAL; + goto out; + } + pf->flow_cfg->dmacflt_max_flows = rsp->max_dmac_filters; out: diff --git a/drivers/net/ethernet/marvell/prestera/Kconfig b/drivers/net/ethernet/marvell/prestera/Kconfig index b6f20e2034c6..f2f7663c3d10 100644 --- a/drivers/net/ethernet/marvell/prestera/Kconfig +++ b/drivers/net/ethernet/marvell/prestera/Kconfig @@ -8,6 +8,7 @@ config PRESTERA depends on NET_SWITCHDEV && VLAN_8021Q depends on BRIDGE || BRIDGE=n select NET_DEVLINK + select PHYLINK help This driver supports Marvell Prestera Switch ASICs family. diff --git a/drivers/net/ethernet/marvell/prestera/prestera.h b/drivers/net/ethernet/marvell/prestera/prestera.h index bff9651f0a89..2f84d0fb4094 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera.h +++ b/drivers/net/ethernet/marvell/prestera/prestera.h @@ -7,6 +7,7 @@ #include <linux/notifier.h> #include <linux/skbuff.h> #include <linux/workqueue.h> +#include <linux/phylink.h> #include <net/devlink.h> #include <uapi/linux/if_ether.h> @@ -92,6 +93,7 @@ struct prestera_lag { struct prestera_flow_block; struct prestera_port_mac_state { + bool valid; u32 mode; u32 speed; bool oper; @@ -151,6 +153,13 @@ struct prestera_port { struct prestera_port_phy_config cfg_phy; struct prestera_port_mac_state state_mac; struct prestera_port_phy_state state_phy; + + struct phylink_config phy_config; + struct phylink *phy_link; + struct phylink_pcs phylink_pcs; + + /* protects state_mac */ + spinlock_t state_mac_lock; }; struct prestera_device { @@ -291,6 +300,7 @@ struct prestera_switch { u32 mtu_min; u32 mtu_max; u8 id; + struct device_node *np; struct prestera_router *router; struct prestera_lag *lags; struct prestera_counter *counter; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c index 40d5b89573bb..1da7ff889417 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c @@ -521,6 +521,9 @@ prestera_ethtool_get_link_ksettings(struct net_device *dev, ecmd->base.speed = SPEED_UNKNOWN; ecmd->base.duplex = DUPLEX_UNKNOWN; + if (port->phy_link) + return phylink_ethtool_ksettings_get(port->phy_link, ecmd); + ecmd->base.autoneg = port->autoneg ? AUTONEG_ENABLE : AUTONEG_DISABLE; if (port->caps.type == PRESTERA_PORT_TYPE_TP) { @@ -648,6 +651,9 @@ prestera_ethtool_set_link_ksettings(struct net_device *dev, u8 adver_fec; int err; + if (port->phy_link) + return phylink_ethtool_ksettings_set(port->phy_link, ecmd); + err = prestera_port_type_set(ecmd, port); if (err) return err; @@ -782,28 +788,6 @@ static int prestera_ethtool_nway_reset(struct net_device *dev) return -EINVAL; } -void prestera_ethtool_port_state_changed(struct prestera_port *port, - struct prestera_port_event *evt) -{ - struct prestera_port_mac_state *smac = &port->state_mac; - - smac->oper = evt->data.mac.oper; - - if (smac->oper) { - smac->mode = evt->data.mac.mode; - smac->speed = evt->data.mac.speed; - smac->duplex = evt->data.mac.duplex; - smac->fc = evt->data.mac.fc; - smac->fec = evt->data.mac.fec; - } else { - smac->mode = PRESTERA_MAC_MODE_MAX; - smac->speed = SPEED_UNKNOWN; - smac->duplex = DUPLEX_UNKNOWN; - smac->fc = 0; - smac->fec = 0; - } -} - const struct ethtool_ops prestera_ethtool_ops = { .get_drvinfo = prestera_ethtool_get_drvinfo, .get_link_ksettings = prestera_ethtool_get_link_ksettings, diff --git a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.h b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.h index 9eb18e99dea6..bd5600886bc6 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.h @@ -11,7 +11,4 @@ struct prestera_port; extern const struct ethtool_ops prestera_ethtool_ops; -void prestera_ethtool_port_state_changed(struct prestera_port *port, - struct prestera_port_event *evt); - #endif /* _PRESTERA_ETHTOOL_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.c b/drivers/net/ethernet/marvell/prestera/prestera_flower.c index 652aa95e65ac..19d3b55c578e 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.c @@ -116,7 +116,7 @@ static int prestera_flower_parse_actions(struct prestera_flow_block *block, rule->re_arg.police.rate = act->police.rate_bytes_ps; rule->re_arg.police.burst = act->police.burst; - rule->re_arg.police.ingress = true; + rule->re_arg.police.ingress = block->ingress; break; case FLOW_ACTION_GOTO: err = prestera_flower_parse_goto_action(block, rule, @@ -138,7 +138,8 @@ static int prestera_flower_parse_actions(struct prestera_flow_block *block, static int prestera_flower_parse_meta(struct prestera_acl_rule *rule, struct flow_cls_offload *f, struct prestera_flow_block *block) -{ struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); +{ + struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); struct prestera_acl_match *r_match = &rule->re_key.match; struct prestera_port *port; struct net_device *ingress_dev; @@ -167,24 +168,24 @@ static int prestera_flower_parse_meta(struct prestera_acl_rule *rule, } port = netdev_priv(ingress_dev); - mask = htons(0x1FFF); - key = htons(port->hw_id); + mask = htons(0x1FFF << 3); + key = htons(port->hw_id << 3); rule_match_set(r_match->key, SYS_PORT, key); rule_match_set(r_match->mask, SYS_PORT, mask); - mask = htons(0x1FF); + mask = htons(0x3FF); key = htons(port->dev_id); rule_match_set(r_match->key, SYS_DEV, key); rule_match_set(r_match->mask, SYS_DEV, mask); return 0; - } static int prestera_flower_parse(struct prestera_flow_block *block, struct prestera_acl_rule *rule, struct flow_cls_offload *f) -{ struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); +{ + struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); struct flow_dissector *dissector = f_rule->match.dissector; struct prestera_acl_match *r_match = &rule->re_key.match; __be16 n_proto_mask = 0; @@ -421,7 +422,6 @@ void prestera_flower_destroy(struct prestera_flow_block *block, prestera_acl_rule_destroy(rule); } prestera_acl_ruleset_put(ruleset); - } int prestera_flower_tmplt_create(struct prestera_flow_block *block, diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c index ea5bd5069826..c267ca1ccdba 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_main.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c @@ -9,6 +9,7 @@ #include <linux/of.h> #include <linux/of_net.h> #include <linux/if_vlan.h> +#include <linux/phylink.h> #include "prestera.h" #include "prestera_hw.h" @@ -142,18 +143,24 @@ static int prestera_port_open(struct net_device *dev) struct prestera_port_mac_config cfg_mac; int err = 0; - if (port->caps.transceiver == PRESTERA_PORT_TCVR_SFP) { - err = prestera_port_cfg_mac_read(port, &cfg_mac); - if (!err) { - cfg_mac.admin = true; - err = prestera_port_cfg_mac_write(port, &cfg_mac); - } + if (port->phy_link) { + phylink_start(port->phy_link); } else { - port->cfg_phy.admin = true; - err = prestera_hw_port_phy_mode_set(port, true, port->autoneg, - port->cfg_phy.mode, - port->adver_link_modes, - port->cfg_phy.mdix); + if (port->caps.transceiver == PRESTERA_PORT_TCVR_SFP) { + err = prestera_port_cfg_mac_read(port, &cfg_mac); + if (!err) { + cfg_mac.admin = true; + err = prestera_port_cfg_mac_write(port, + &cfg_mac); + } + } else { + port->cfg_phy.admin = true; + err = prestera_hw_port_phy_mode_set(port, true, + port->autoneg, + port->cfg_phy.mode, + port->adver_link_modes, + port->cfg_phy.mdix); + } } netif_start_queue(dev); @@ -169,23 +176,259 @@ static int prestera_port_close(struct net_device *dev) netif_stop_queue(dev); - if (port->caps.transceiver == PRESTERA_PORT_TCVR_SFP) { + if (port->phy_link) { + phylink_stop(port->phy_link); + phylink_disconnect_phy(port->phy_link); err = prestera_port_cfg_mac_read(port, &cfg_mac); if (!err) { cfg_mac.admin = false; prestera_port_cfg_mac_write(port, &cfg_mac); } } else { - port->cfg_phy.admin = false; - err = prestera_hw_port_phy_mode_set(port, false, port->autoneg, - port->cfg_phy.mode, - port->adver_link_modes, - port->cfg_phy.mdix); + if (port->caps.transceiver == PRESTERA_PORT_TCVR_SFP) { + err = prestera_port_cfg_mac_read(port, &cfg_mac); + if (!err) { + cfg_mac.admin = false; + prestera_port_cfg_mac_write(port, &cfg_mac); + } + } else { + port->cfg_phy.admin = false; + err = prestera_hw_port_phy_mode_set(port, false, port->autoneg, + port->cfg_phy.mode, + port->adver_link_modes, + port->cfg_phy.mdix); + } + } + + return err; +} + +static void +prestera_port_mac_state_cache_read(struct prestera_port *port, + struct prestera_port_mac_state *state) +{ + spin_lock(&port->state_mac_lock); + *state = port->state_mac; + spin_unlock(&port->state_mac_lock); +} + +static void +prestera_port_mac_state_cache_write(struct prestera_port *port, + struct prestera_port_mac_state *state) +{ + spin_lock(&port->state_mac_lock); + port->state_mac = *state; + spin_unlock(&port->state_mac_lock); +} + +static struct prestera_port *prestera_pcs_to_port(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct prestera_port, phylink_pcs); +} + +static void prestera_mac_config(struct phylink_config *config, + unsigned int an_mode, + const struct phylink_link_state *state) +{ +} + +static void prestera_mac_link_down(struct phylink_config *config, + unsigned int mode, phy_interface_t interface) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct prestera_port *port = netdev_priv(ndev); + struct prestera_port_mac_state state_mac; + + /* Invalidate. Parameters will update on next link event. */ + memset(&state_mac, 0, sizeof(state_mac)); + state_mac.valid = false; + prestera_port_mac_state_cache_write(port, &state_mac); +} + +static void prestera_mac_link_up(struct phylink_config *config, + struct phy_device *phy, + unsigned int mode, phy_interface_t interface, + int speed, int duplex, + bool tx_pause, bool rx_pause) +{ +} + +static struct phylink_pcs * +prestera_mac_select_pcs(struct phylink_config *config, + phy_interface_t interface) +{ + struct net_device *dev = to_net_dev(config->dev); + struct prestera_port *port = netdev_priv(dev); + + return &port->phylink_pcs; +} + +static void prestera_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + struct prestera_port *port = container_of(pcs, struct prestera_port, + phylink_pcs); + struct prestera_port_mac_state smac; + + prestera_port_mac_state_cache_read(port, &smac); + + if (smac.valid) { + state->link = smac.oper ? 1 : 0; + /* AN is completed, when port is up */ + state->an_complete = (smac.oper && port->autoneg) ? 1 : 0; + state->speed = smac.speed; + state->duplex = smac.duplex; + } else { + state->link = 0; + state->an_complete = 0; + } +} + +static int prestera_pcs_config(struct phylink_pcs *pcs, + unsigned int mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + struct prestera_port *port = port = prestera_pcs_to_port(pcs); + struct prestera_port_mac_config cfg_mac; + int err; + + err = prestera_port_cfg_mac_read(port, &cfg_mac); + if (err) + return err; + + cfg_mac.admin = true; + cfg_mac.fec = PRESTERA_PORT_FEC_OFF; + + switch (interface) { + case PHY_INTERFACE_MODE_10GBASER: + cfg_mac.speed = SPEED_10000; + cfg_mac.inband = 0; + cfg_mac.mode = PRESTERA_MAC_MODE_SR_LR; + break; + case PHY_INTERFACE_MODE_2500BASEX: + cfg_mac.speed = SPEED_2500; + cfg_mac.duplex = DUPLEX_FULL; + cfg_mac.inband = test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + advertising); + cfg_mac.mode = PRESTERA_MAC_MODE_SGMII; + break; + case PHY_INTERFACE_MODE_SGMII: + cfg_mac.inband = 1; + cfg_mac.mode = PRESTERA_MAC_MODE_SGMII; + break; + case PHY_INTERFACE_MODE_1000BASEX: + default: + cfg_mac.speed = SPEED_1000; + cfg_mac.duplex = DUPLEX_FULL; + cfg_mac.inband = test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + advertising); + cfg_mac.mode = PRESTERA_MAC_MODE_1000BASE_X; + break; } + err = prestera_port_cfg_mac_write(port, &cfg_mac); + if (err) + return err; + + return 0; +} + +static void prestera_pcs_an_restart(struct phylink_pcs *pcs) +{ + /* TODO: add 1000basex AN restart support + * (Currently FW has no support for 1000baseX AN restart, but it will in the future, + * so as for now the function would stay empty.) + */ +} + +static const struct phylink_mac_ops prestera_mac_ops = { + .validate = phylink_generic_validate, + .mac_select_pcs = prestera_mac_select_pcs, + .mac_config = prestera_mac_config, + .mac_link_down = prestera_mac_link_down, + .mac_link_up = prestera_mac_link_up, +}; + +static const struct phylink_pcs_ops prestera_pcs_ops = { + .pcs_get_state = prestera_pcs_get_state, + .pcs_config = prestera_pcs_config, + .pcs_an_restart = prestera_pcs_an_restart, +}; + +static int prestera_port_sfp_bind(struct prestera_port *port) +{ + struct prestera_switch *sw = port->sw; + struct device_node *ports, *node; + struct fwnode_handle *fwnode; + struct phylink *phy_link; + int err; + + if (!sw->np) + return 0; + + ports = of_find_node_by_name(sw->np, "ports"); + + for_each_child_of_node(ports, node) { + int num; + + err = of_property_read_u32(node, "prestera,port-num", &num); + if (err) { + dev_err(sw->dev->dev, + "device node %pOF has no valid reg property: %d\n", + node, err); + goto out; + } + + if (port->fp_id != num) + continue; + + port->phylink_pcs.ops = &prestera_pcs_ops; + + port->phy_config.dev = &port->dev->dev; + port->phy_config.type = PHYLINK_NETDEV; + + fwnode = of_fwnode_handle(node); + + __set_bit(PHY_INTERFACE_MODE_10GBASER, + port->phy_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_2500BASEX, + port->phy_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_SGMII, + port->phy_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, + port->phy_config.supported_interfaces); + + port->phy_config.mac_capabilities = + MAC_1000 | MAC_2500FD | MAC_10000FD; + + phy_link = phylink_create(&port->phy_config, fwnode, + PHY_INTERFACE_MODE_INTERNAL, + &prestera_mac_ops); + if (IS_ERR(phy_link)) { + netdev_err(port->dev, "failed to create phylink\n"); + err = PTR_ERR(phy_link); + goto out; + } + + port->phy_link = phy_link; + break; + } + +out: + of_node_put(ports); return err; } +static int prestera_port_sfp_unbind(struct prestera_port *port) +{ + if (port->phy_link) + phylink_destroy(port->phy_link); + + return 0; +} + static netdev_tx_t prestera_port_xmit(struct sk_buff *skb, struct net_device *dev) { @@ -366,6 +609,8 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) port->id = id; port->sw = sw; + spin_lock_init(&port->state_mac_lock); + err = prestera_hw_port_info_get(port, &port->dev_id, &port->hw_id, &port->fp_id); if (err) { @@ -380,8 +625,10 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_HW_TC; dev->netdev_ops = &prestera_netdev_ops; dev->ethtool_ops = &prestera_ethtool_ops; + SET_NETDEV_DEV(dev, sw->dev->dev); - netif_carrier_off(dev); + if (port->caps.transceiver != PRESTERA_PORT_TCVR_SFP) + netif_carrier_off(dev); dev->mtu = min_t(unsigned int, sw->mtu_max, PRESTERA_MTU_DEFAULT); dev->min_mtu = sw->mtu_min; @@ -432,7 +679,7 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) cfg_mac.admin = false; cfg_mac.mode = PRESTERA_MAC_MODE_MAX; } - cfg_mac.inband = false; + cfg_mac.inband = 0; cfg_mac.speed = 0; cfg_mac.duplex = DUPLEX_UNKNOWN; cfg_mac.fec = PRESTERA_PORT_FEC_OFF; @@ -474,8 +721,13 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) prestera_devlink_port_set(port); + err = prestera_port_sfp_bind(port); + if (err) + goto err_sfp_bind; + return 0; +err_sfp_bind: err_register_netdev: prestera_port_list_del(port); err_port_init: @@ -521,8 +773,10 @@ static int prestera_create_ports(struct prestera_switch *sw) return 0; err_port_create: - list_for_each_entry_safe(port, tmp, &sw->port_list, list) + list_for_each_entry_safe(port, tmp, &sw->port_list, list) { + prestera_port_sfp_unbind(port); prestera_port_destroy(port); + } return err; } @@ -530,25 +784,47 @@ err_port_create: static void prestera_port_handle_event(struct prestera_switch *sw, struct prestera_event *evt, void *arg) { + struct prestera_port_mac_state smac; + struct prestera_port_event *pevt; struct delayed_work *caching_dw; struct prestera_port *port; - port = prestera_find_port(sw, evt->port_evt.port_id); - if (!port || !port->dev) - return; - - caching_dw = &port->cached_hw_stats.caching_dw; - - prestera_ethtool_port_state_changed(port, &evt->port_evt); - if (evt->id == PRESTERA_PORT_EVENT_MAC_STATE_CHANGED) { + pevt = &evt->port_evt; + port = prestera_find_port(sw, pevt->port_id); + if (!port || !port->dev) + return; + + caching_dw = &port->cached_hw_stats.caching_dw; + + if (port->phy_link) { + memset(&smac, 0, sizeof(smac)); + smac.valid = true; + smac.oper = pevt->data.mac.oper; + if (smac.oper) { + smac.mode = pevt->data.mac.mode; + smac.speed = pevt->data.mac.speed; + smac.duplex = pevt->data.mac.duplex; + smac.fc = pevt->data.mac.fc; + smac.fec = pevt->data.mac.fec; + phylink_mac_change(port->phy_link, true); + } else { + phylink_mac_change(port->phy_link, false); + } + prestera_port_mac_state_cache_write(port, &smac); + } + if (port->state_mac.oper) { - netif_carrier_on(port->dev); + if (!port->phy_link) + netif_carrier_on(port->dev); + if (!delayed_work_pending(caching_dw)) queue_delayed_work(prestera_wq, caching_dw, 0); } else if (netif_running(port->dev) && netif_carrier_ok(port->dev)) { - netif_carrier_off(port->dev); + if (!port->phy_link) + netif_carrier_off(port->dev); + if (delayed_work_pending(caching_dw)) cancel_delayed_work(caching_dw); } @@ -571,19 +847,20 @@ static void prestera_event_handlers_unregister(struct prestera_switch *sw) static int prestera_switch_set_base_mac_addr(struct prestera_switch *sw) { struct device_node *base_mac_np; - struct device_node *np; int ret; - np = of_find_compatible_node(NULL, NULL, "marvell,prestera"); - base_mac_np = of_parse_phandle(np, "base-mac-provider", 0); + if (sw->np) { + base_mac_np = of_parse_phandle(sw->np, "base-mac-provider", 0); + if (base_mac_np) { + ret = of_get_mac_address(base_mac_np, sw->base_mac); + of_node_put(base_mac_np); + } + } - ret = of_get_mac_address(base_mac_np, sw->base_mac); - if (ret) { + if (!is_valid_ether_addr(sw->base_mac) || ret) { eth_random_addr(sw->base_mac); dev_info(prestera_dev(sw), "using random base mac address\n"); } - of_node_put(base_mac_np); - of_node_put(np); return prestera_hw_switch_mac_set(sw, sw->base_mac); } @@ -1083,6 +1360,8 @@ static int prestera_switch_init(struct prestera_switch *sw) { int err; + sw->np = of_find_compatible_node(NULL, NULL, "marvell,prestera"); + err = prestera_hw_switch_init(sw); if (err) { dev_err(prestera_dev(sw), "Failed to init Switch device\n"); @@ -1183,6 +1462,7 @@ static void prestera_switch_fini(struct prestera_switch *sw) prestera_router_fini(sw); prestera_netdev_event_handler_unregister(sw); prestera_hw_switch_fini(sw); + of_node_put(sw->np); } int prestera_device_register(struct prestera_device *dev) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router.c b/drivers/net/ethernet/marvell/prestera/prestera_router.c index 3c8116f16b4d..58f4e44d5ad7 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_router.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_router.c @@ -389,8 +389,8 @@ static int __prestera_inetaddr_event(struct prestera_switch *sw, unsigned long event, struct netlink_ext_ack *extack) { - if (!prestera_netdev_check(dev) || netif_is_bridge_port(dev) || - netif_is_lag_port(dev) || netif_is_ovs_port(dev)) + if (!prestera_netdev_check(dev) || netif_is_any_bridge_port(dev) || + netif_is_lag_port(dev)) return 0; return __prestera_inetaddr_port_event(dev, event, extack); diff --git a/drivers/net/ethernet/mediatek/Kconfig b/drivers/net/ethernet/mediatek/Kconfig index da4ec235d146..97374fb3ee79 100644 --- a/drivers/net/ethernet/mediatek/Kconfig +++ b/drivers/net/ethernet/mediatek/Kconfig @@ -17,6 +17,8 @@ config NET_MEDIATEK_SOC select PINCTRL select PHYLINK select DIMLIB + select PAGE_POOL + select PAGE_POOL_STATS help This driver supports the gigabit ethernet MACs in the MediaTek SoC family. diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 6beb3d4873a3..c370d6589596 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -34,6 +34,10 @@ MODULE_PARM_DESC(msg_level, "Message level (-1=defaults,0=none,...,16=all)"); #define MTK_ETHTOOL_STAT(x) { #x, \ offsetof(struct mtk_hw_stats, x) / sizeof(u64) } +#define MTK_ETHTOOL_XDP_STAT(x) { #x, \ + offsetof(struct mtk_hw_stats, xdp_stats.x) / \ + sizeof(u64) } + static const struct mtk_reg_map mtk_reg_map = { .tx_irq_mask = 0x1a1c, .tx_irq_status = 0x1a18, @@ -141,6 +145,13 @@ static const struct mtk_ethtool_stats { MTK_ETHTOOL_STAT(rx_long_errors), MTK_ETHTOOL_STAT(rx_checksum_errors), MTK_ETHTOOL_STAT(rx_flow_control_packets), + MTK_ETHTOOL_XDP_STAT(rx_xdp_redirect), + MTK_ETHTOOL_XDP_STAT(rx_xdp_pass), + MTK_ETHTOOL_XDP_STAT(rx_xdp_drop), + MTK_ETHTOOL_XDP_STAT(rx_xdp_tx), + MTK_ETHTOOL_XDP_STAT(rx_xdp_tx_errors), + MTK_ETHTOOL_XDP_STAT(tx_xdp_xmit), + MTK_ETHTOOL_XDP_STAT(tx_xdp_xmit_errors), }; static const char * const mtk_clks_source_name[] = { @@ -1020,15 +1031,26 @@ static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf, } } - tx_buf->flags = 0; - if (tx_buf->skb && - (tx_buf->skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC)) { - if (napi) - napi_consume_skb(tx_buf->skb, napi); + if (tx_buf->type == MTK_TYPE_SKB) { + if (tx_buf->data && + tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) { + struct sk_buff *skb = tx_buf->data; + + if (napi) + napi_consume_skb(skb, napi); + else + dev_kfree_skb_any(skb); + } + } else if (tx_buf->data) { + struct xdp_frame *xdpf = tx_buf->data; + + if (napi && tx_buf->type == MTK_TYPE_XDP_TX) + xdp_return_frame_rx_napi(xdpf); else - dev_kfree_skb_any(tx_buf->skb); + xdp_return_frame(xdpf); } - tx_buf->skb = NULL; + tx_buf->flags = 0; + tx_buf->data = NULL; } static void setup_tx_buf(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf, @@ -1045,7 +1067,7 @@ static void setup_tx_buf(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf, dma_unmap_addr_set(tx_buf, dma_addr1, mapped_addr); dma_unmap_len_set(tx_buf, dma_len1, size); } else { - tx_buf->skb = (struct sk_buff *)MTK_DMA_DUMMY_DESC; + tx_buf->data = (void *)MTK_DMA_DUMMY_DESC; txd->txd1 = mapped_addr; txd->txd2 = TX_DMA_PLEN0(size); dma_unmap_addr_set(tx_buf, dma_addr0, mapped_addr); @@ -1221,7 +1243,7 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev, soc->txrx.txd_size); if (new_desc) memset(tx_buf, 0, sizeof(*tx_buf)); - tx_buf->skb = (struct sk_buff *)MTK_DMA_DUMMY_DESC; + tx_buf->data = (void *)MTK_DMA_DUMMY_DESC; tx_buf->flags |= MTK_TX_FLAGS_PAGE0; tx_buf->flags |= (!mac->id) ? MTK_TX_FLAGS_FPORT0 : MTK_TX_FLAGS_FPORT1; @@ -1235,7 +1257,8 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev, } /* store skb to cleanup */ - itx_buf->skb = skb; + itx_buf->type = MTK_TYPE_SKB; + itx_buf->data = skb; if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) { if (k & 0x1) @@ -1432,11 +1455,259 @@ static void mtk_update_rx_cpu_idx(struct mtk_eth *eth) } } +static bool mtk_page_pool_enabled(struct mtk_eth *eth) +{ + return !eth->hwlro; +} + +static struct page_pool *mtk_create_page_pool(struct mtk_eth *eth, + struct xdp_rxq_info *xdp_q, + int id, int size) +{ + struct page_pool_params pp_params = { + .order = 0, + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + .pool_size = size, + .nid = NUMA_NO_NODE, + .dev = eth->dma_dev, + .offset = MTK_PP_HEADROOM, + .max_len = MTK_PP_MAX_BUF_SIZE, + }; + struct page_pool *pp; + int err; + + pp_params.dma_dir = rcu_access_pointer(eth->prog) ? DMA_BIDIRECTIONAL + : DMA_FROM_DEVICE; + pp = page_pool_create(&pp_params); + if (IS_ERR(pp)) + return pp; + + err = __xdp_rxq_info_reg(xdp_q, ð->dummy_dev, eth->rx_napi.napi_id, + id, PAGE_SIZE); + if (err < 0) + goto err_free_pp; + + err = xdp_rxq_info_reg_mem_model(xdp_q, MEM_TYPE_PAGE_POOL, pp); + if (err) + goto err_unregister_rxq; + + return pp; + +err_unregister_rxq: + xdp_rxq_info_unreg(xdp_q); +err_free_pp: + page_pool_destroy(pp); + + return ERR_PTR(err); +} + +static void *mtk_page_pool_get_buff(struct page_pool *pp, dma_addr_t *dma_addr, + gfp_t gfp_mask) +{ + struct page *page; + + page = page_pool_alloc_pages(pp, gfp_mask | __GFP_NOWARN); + if (!page) + return NULL; + + *dma_addr = page_pool_get_dma_addr(page) + MTK_PP_HEADROOM; + return page_address(page); +} + +static void mtk_rx_put_buff(struct mtk_rx_ring *ring, void *data, bool napi) +{ + if (ring->page_pool) + page_pool_put_full_page(ring->page_pool, + virt_to_head_page(data), napi); + else + skb_free_frag(data); +} + +static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf, + struct net_device *dev, bool dma_map) +{ + const struct mtk_soc_data *soc = eth->soc; + struct mtk_tx_ring *ring = ð->tx_ring; + struct mtk_tx_dma_desc_info txd_info = { + .size = xdpf->len, + .first = true, + .last = true, + }; + struct mtk_mac *mac = netdev_priv(dev); + struct mtk_tx_dma *txd, *txd_pdma; + int err = 0, index = 0, n_desc = 1; + struct mtk_tx_buf *tx_buf; + + if (unlikely(test_bit(MTK_RESETTING, ð->state))) + return -EBUSY; + + if (unlikely(atomic_read(&ring->free_count) <= 1)) + return -EBUSY; + + spin_lock(ð->page_lock); + + txd = ring->next_free; + if (txd == ring->last_free) { + err = -ENOMEM; + goto out; + } + + tx_buf = mtk_desc_to_tx_buf(ring, txd, soc->txrx.txd_size); + memset(tx_buf, 0, sizeof(*tx_buf)); + + if (dma_map) { /* ndo_xdp_xmit */ + txd_info.addr = dma_map_single(eth->dma_dev, xdpf->data, + txd_info.size, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(eth->dma_dev, txd_info.addr))) { + err = -ENOMEM; + goto out; + } + tx_buf->flags |= MTK_TX_FLAGS_SINGLE0; + } else { + struct page *page = virt_to_head_page(xdpf->data); + + txd_info.addr = page_pool_get_dma_addr(page) + + sizeof(*xdpf) + xdpf->headroom; + dma_sync_single_for_device(eth->dma_dev, txd_info.addr, + txd_info.size, + DMA_BIDIRECTIONAL); + } + mtk_tx_set_dma_desc(dev, txd, &txd_info); + + tx_buf->flags |= !mac->id ? MTK_TX_FLAGS_FPORT0 : MTK_TX_FLAGS_FPORT1; + + txd_pdma = qdma_to_pdma(ring, txd); + setup_tx_buf(eth, tx_buf, txd_pdma, txd_info.addr, txd_info.size, + index++); + + /* store xdpf for cleanup */ + tx_buf->type = dma_map ? MTK_TYPE_XDP_NDO : MTK_TYPE_XDP_TX; + tx_buf->data = xdpf; + + if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) { + if (index & 1) + txd_pdma->txd2 |= TX_DMA_LS0; + else + txd_pdma->txd2 |= TX_DMA_LS1; + } + + ring->next_free = mtk_qdma_phys_to_virt(ring, txd->txd2); + atomic_sub(n_desc, &ring->free_count); + + /* make sure that all changes to the dma ring are flushed before we + * continue + */ + wmb(); + + if (MTK_HAS_CAPS(soc->caps, MTK_QDMA)) { + mtk_w32(eth, txd->txd2, soc->reg_map->qdma.ctx_ptr); + } else { + int idx; + + idx = txd_to_idx(ring, txd, soc->txrx.txd_size); + mtk_w32(eth, NEXT_DESP_IDX(idx, ring->dma_size), + MT7628_TX_CTX_IDX0); + } +out: + spin_unlock(ð->page_lock); + + return err; +} + +static int mtk_xdp_xmit(struct net_device *dev, int num_frame, + struct xdp_frame **frames, u32 flags) +{ + struct mtk_mac *mac = netdev_priv(dev); + struct mtk_hw_stats *hw_stats = mac->hw_stats; + struct mtk_eth *eth = mac->hw; + int i, nxmit = 0; + + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) + return -EINVAL; + + for (i = 0; i < num_frame; i++) { + if (mtk_xdp_submit_frame(eth, frames[i], dev, true)) + break; + nxmit++; + } + + u64_stats_update_begin(&hw_stats->syncp); + hw_stats->xdp_stats.tx_xdp_xmit += nxmit; + hw_stats->xdp_stats.tx_xdp_xmit_errors += num_frame - nxmit; + u64_stats_update_end(&hw_stats->syncp); + + return nxmit; +} + +static u32 mtk_xdp_run(struct mtk_eth *eth, struct mtk_rx_ring *ring, + struct xdp_buff *xdp, struct net_device *dev) +{ + struct mtk_mac *mac = netdev_priv(dev); + struct mtk_hw_stats *hw_stats = mac->hw_stats; + u64 *count = &hw_stats->xdp_stats.rx_xdp_drop; + struct bpf_prog *prog; + u32 act = XDP_PASS; + + rcu_read_lock(); + + prog = rcu_dereference(eth->prog); + if (!prog) + goto out; + + act = bpf_prog_run_xdp(prog, xdp); + switch (act) { + case XDP_PASS: + count = &hw_stats->xdp_stats.rx_xdp_pass; + goto update_stats; + case XDP_REDIRECT: + if (unlikely(xdp_do_redirect(dev, xdp, prog))) { + act = XDP_DROP; + break; + } + + count = &hw_stats->xdp_stats.rx_xdp_redirect; + goto update_stats; + case XDP_TX: { + struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); + + if (mtk_xdp_submit_frame(eth, xdpf, dev, false)) { + count = &hw_stats->xdp_stats.rx_xdp_tx_errors; + act = XDP_DROP; + break; + } + + count = &hw_stats->xdp_stats.rx_xdp_tx; + goto update_stats; + } + default: + bpf_warn_invalid_xdp_action(dev, prog, act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(dev, prog, act); + fallthrough; + case XDP_DROP: + break; + } + + page_pool_put_full_page(ring->page_pool, + virt_to_head_page(xdp->data), true); + +update_stats: + u64_stats_update_begin(&hw_stats->syncp); + *count = *count + 1; + u64_stats_update_end(&hw_stats->syncp); +out: + rcu_read_unlock(); + + return act; +} + static int mtk_poll_rx(struct napi_struct *napi, int budget, struct mtk_eth *eth) { struct dim_sample dim_sample = {}; struct mtk_rx_ring *ring; + bool xdp_flush = false; int idx; struct sk_buff *skb; u8 *data, *new_data; @@ -1477,41 +1748,86 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, if (unlikely(test_bit(MTK_RESETTING, ð->state))) goto release_desc; + pktlen = RX_DMA_GET_PLEN0(trxd.rxd2); + /* alloc new buffer */ - if (ring->frag_size <= PAGE_SIZE) - new_data = napi_alloc_frag(ring->frag_size); - else - new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC); - if (unlikely(!new_data)) { - netdev->stats.rx_dropped++; - goto release_desc; - } - dma_addr = dma_map_single(eth->dma_dev, - new_data + NET_SKB_PAD + - eth->ip_align, - ring->buf_size, - DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr))) { - skb_free_frag(new_data); - netdev->stats.rx_dropped++; - goto release_desc; - } + if (ring->page_pool) { + struct page *page = virt_to_head_page(data); + struct xdp_buff xdp; + u32 ret; + + new_data = mtk_page_pool_get_buff(ring->page_pool, + &dma_addr, + GFP_ATOMIC); + if (unlikely(!new_data)) { + netdev->stats.rx_dropped++; + goto release_desc; + } - dma_unmap_single(eth->dma_dev, trxd.rxd1, - ring->buf_size, DMA_FROM_DEVICE); + dma_sync_single_for_cpu(eth->dma_dev, + page_pool_get_dma_addr(page) + MTK_PP_HEADROOM, + pktlen, page_pool_get_dma_dir(ring->page_pool)); - /* receive data */ - skb = build_skb(data, ring->frag_size); - if (unlikely(!skb)) { - skb_free_frag(data); - netdev->stats.rx_dropped++; - goto skip_rx; + xdp_init_buff(&xdp, PAGE_SIZE, &ring->xdp_q); + xdp_prepare_buff(&xdp, data, MTK_PP_HEADROOM, pktlen, + false); + xdp_buff_clear_frags_flag(&xdp); + + ret = mtk_xdp_run(eth, ring, &xdp, netdev); + if (ret == XDP_REDIRECT) + xdp_flush = true; + + if (ret != XDP_PASS) + goto skip_rx; + + skb = build_skb(data, PAGE_SIZE); + if (unlikely(!skb)) { + page_pool_put_full_page(ring->page_pool, + page, true); + netdev->stats.rx_dropped++; + goto skip_rx; + } + + skb_reserve(skb, xdp.data - xdp.data_hard_start); + skb_put(skb, xdp.data_end - xdp.data); + skb_mark_for_recycle(skb); + } else { + if (ring->frag_size <= PAGE_SIZE) + new_data = napi_alloc_frag(ring->frag_size); + else + new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC); + + if (unlikely(!new_data)) { + netdev->stats.rx_dropped++; + goto release_desc; + } + + dma_addr = dma_map_single(eth->dma_dev, + new_data + NET_SKB_PAD + eth->ip_align, + ring->buf_size, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(eth->dma_dev, + dma_addr))) { + skb_free_frag(new_data); + netdev->stats.rx_dropped++; + goto release_desc; + } + + dma_unmap_single(eth->dma_dev, trxd.rxd1, + ring->buf_size, DMA_FROM_DEVICE); + + skb = build_skb(data, ring->frag_size); + if (unlikely(!skb)) { + netdev->stats.rx_dropped++; + skb_free_frag(data); + goto skip_rx; + } + + skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); + skb_put(skb, pktlen); } - skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); - pktlen = RX_DMA_GET_PLEN0(trxd.rxd2); skb->dev = netdev; - skb_put(skb, pktlen); + bytes += skb->len; if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) rxdcsum = &trxd.rxd3; @@ -1523,7 +1839,6 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, else skb_checksum_none_assert(skb); skb->protocol = eth_type_trans(skb, netdev); - bytes += pktlen; hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY; if (hash != MTK_RXD4_FOE_ENTRY) { @@ -1561,7 +1876,6 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, skip_rx: ring->data[idx] = new_data; rxd->rxd1 = (unsigned int)dma_addr; - release_desc: if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) rxd->rxd2 = RX_DMA_LSO; @@ -1569,7 +1883,6 @@ release_desc: rxd->rxd2 = RX_DMA_PREP_PLEN0(ring->buf_size); ring->calc_idx = idx; - done++; } @@ -1588,6 +1901,9 @@ rx_done: &dim_sample); net_dim(ð->rx_dim, dim_sample); + if (xdp_flush) + xdp_do_flush_map(); + return done; } @@ -1596,9 +1912,8 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget, { const struct mtk_reg_map *reg_map = eth->soc->reg_map; struct mtk_tx_ring *ring = ð->tx_ring; - struct mtk_tx_dma *desc; - struct sk_buff *skb; struct mtk_tx_buf *tx_buf; + struct mtk_tx_dma *desc; u32 cpu, dma; cpu = ring->last_free_ptr; @@ -1619,15 +1934,21 @@ static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget, if (tx_buf->flags & MTK_TX_FLAGS_FPORT1) mac = 1; - skb = tx_buf->skb; - if (!skb) + if (!tx_buf->data) break; - if (skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC) { + if (tx_buf->type == MTK_TYPE_SKB && + tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) { + struct sk_buff *skb = tx_buf->data; + bytes[mac] += skb->len; done[mac]++; budget--; + } else if (tx_buf->type == MTK_TYPE_XDP_TX || + tx_buf->type == MTK_TYPE_XDP_NDO) { + budget--; } + mtk_tx_unmap(eth, tx_buf, true); ring->last_free = desc; @@ -1646,9 +1967,8 @@ static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget, unsigned int *done, unsigned int *bytes) { struct mtk_tx_ring *ring = ð->tx_ring; - struct mtk_tx_dma *desc; - struct sk_buff *skb; struct mtk_tx_buf *tx_buf; + struct mtk_tx_dma *desc; u32 cpu, dma; cpu = ring->cpu_idx; @@ -1656,14 +1976,18 @@ static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget, while ((cpu != dma) && budget) { tx_buf = &ring->buf[cpu]; - skb = tx_buf->skb; - if (!skb) + if (!tx_buf->data) break; - if (skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC) { + if (tx_buf->type == MTK_TYPE_SKB && + tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) { + struct sk_buff *skb = tx_buf->data; bytes[0] += skb->len; done[0]++; budget--; + } else if (tx_buf->type == MTK_TYPE_XDP_TX || + tx_buf->type == MTK_TYPE_XDP_NDO) { + budget--; } mtk_tx_unmap(eth, tx_buf, true); @@ -1933,13 +2257,15 @@ static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag) if (!ring->data) return -ENOMEM; - for (i = 0; i < rx_dma_size; i++) { - if (ring->frag_size <= PAGE_SIZE) - ring->data[i] = netdev_alloc_frag(ring->frag_size); - else - ring->data[i] = mtk_max_lro_buf_alloc(GFP_KERNEL); - if (!ring->data[i]) - return -ENOMEM; + if (mtk_page_pool_enabled(eth)) { + struct page_pool *pp; + + pp = mtk_create_page_pool(eth, &ring->xdp_q, ring_no, + rx_dma_size); + if (IS_ERR(pp)) + return PTR_ERR(pp); + + ring->page_pool = pp; } ring->dma = dma_alloc_coherent(eth->dma_dev, @@ -1950,16 +2276,33 @@ static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag) for (i = 0; i < rx_dma_size; i++) { struct mtk_rx_dma_v2 *rxd; - - dma_addr_t dma_addr = dma_map_single(eth->dma_dev, - ring->data[i] + NET_SKB_PAD + eth->ip_align, - ring->buf_size, - DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr))) - return -ENOMEM; + dma_addr_t dma_addr; + void *data; rxd = ring->dma + i * eth->soc->txrx.rxd_size; + if (ring->page_pool) { + data = mtk_page_pool_get_buff(ring->page_pool, + &dma_addr, GFP_KERNEL); + if (!data) + return -ENOMEM; + } else { + if (ring->frag_size <= PAGE_SIZE) + data = netdev_alloc_frag(ring->frag_size); + else + data = mtk_max_lro_buf_alloc(GFP_KERNEL); + + if (!data) + return -ENOMEM; + + dma_addr = dma_map_single(eth->dma_dev, + data + NET_SKB_PAD + eth->ip_align, + ring->buf_size, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(eth->dma_dev, + dma_addr))) + return -ENOMEM; + } rxd->rxd1 = (unsigned int)dma_addr; + ring->data[i] = data; if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) rxd->rxd2 = RX_DMA_LSO; @@ -1975,6 +2318,7 @@ static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag) rxd->rxd8 = 0; } } + ring->dma_size = rx_dma_size; ring->calc_idx_update = false; ring->calc_idx = rx_dma_size - 1; @@ -2026,7 +2370,7 @@ static void mtk_rx_clean(struct mtk_eth *eth, struct mtk_rx_ring *ring) dma_unmap_single(eth->dma_dev, rxd->rxd1, ring->buf_size, DMA_FROM_DEVICE); - skb_free_frag(ring->data[i]); + mtk_rx_put_buff(ring, ring->data[i], false); } kfree(ring->data); ring->data = NULL; @@ -2038,6 +2382,13 @@ static void mtk_rx_clean(struct mtk_eth *eth, struct mtk_rx_ring *ring) ring->dma, ring->phys); ring->dma = NULL; } + + if (ring->page_pool) { + if (xdp_rxq_info_is_reg(&ring->xdp_q)) + xdp_rxq_info_unreg(&ring->xdp_q); + page_pool_destroy(ring->page_pool); + ring->page_pool = NULL; + } } static int mtk_hwlro_rx_init(struct mtk_eth *eth) @@ -2645,6 +2996,48 @@ static int mtk_stop(struct net_device *dev) return 0; } +static int mtk_xdp_setup(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) +{ + struct mtk_mac *mac = netdev_priv(dev); + struct mtk_eth *eth = mac->hw; + struct bpf_prog *old_prog; + bool need_update; + + if (eth->hwlro) { + NL_SET_ERR_MSG_MOD(extack, "XDP not supported with HWLRO"); + return -EOPNOTSUPP; + } + + if (dev->mtu > MTK_PP_MAX_BUF_SIZE) { + NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP"); + return -EOPNOTSUPP; + } + + need_update = !!eth->prog != !!prog; + if (netif_running(dev) && need_update) + mtk_stop(dev); + + old_prog = rcu_replace_pointer(eth->prog, prog, lockdep_rtnl_is_held()); + if (old_prog) + bpf_prog_put(old_prog); + + if (netif_running(dev) && need_update) + return mtk_open(dev); + + return 0; +} + +static int mtk_xdp(struct net_device *dev, struct netdev_bpf *xdp) +{ + switch (xdp->command) { + case XDP_SETUP_PROG: + return mtk_xdp_setup(dev, xdp->prog, xdp->extack); + default: + return -EINVAL; + } +} + static void ethsys_reset(struct mtk_eth *eth, u32 reset_bits) { regmap_update_bits(eth->ethsys, ETHSYS_RSTCTRL, @@ -2940,6 +3333,12 @@ static int mtk_change_mtu(struct net_device *dev, int new_mtu) struct mtk_eth *eth = mac->hw; u32 mcr_cur, mcr_new; + if (rcu_access_pointer(eth->prog) && + length > MTK_PP_MAX_BUF_SIZE) { + netdev_err(dev, "Invalid MTU for XDP mode\n"); + return -EINVAL; + } + if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) { mcr_cur = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id)); mcr_new = mcr_cur & ~MAC_MCR_MAX_RX_MASK; @@ -3129,11 +3528,18 @@ static void mtk_get_strings(struct net_device *dev, u32 stringset, u8 *data) int i; switch (stringset) { - case ETH_SS_STATS: + case ETH_SS_STATS: { + struct mtk_mac *mac = netdev_priv(dev); + for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++) { memcpy(data, mtk_ethtool_stats[i].str, ETH_GSTRING_LEN); data += ETH_GSTRING_LEN; } + if (mtk_page_pool_enabled(mac->hw)) + page_pool_ethtool_stats_get_strings(data); + break; + } + default: break; } } @@ -3141,13 +3547,35 @@ static void mtk_get_strings(struct net_device *dev, u32 stringset, u8 *data) static int mtk_get_sset_count(struct net_device *dev, int sset) { switch (sset) { - case ETH_SS_STATS: - return ARRAY_SIZE(mtk_ethtool_stats); + case ETH_SS_STATS: { + int count = ARRAY_SIZE(mtk_ethtool_stats); + struct mtk_mac *mac = netdev_priv(dev); + + if (mtk_page_pool_enabled(mac->hw)) + count += page_pool_ethtool_stats_get_count(); + return count; + } default: return -EOPNOTSUPP; } } +static void mtk_ethtool_pp_stats(struct mtk_eth *eth, u64 *data) +{ + struct page_pool_stats stats = {}; + int i; + + for (i = 0; i < ARRAY_SIZE(eth->rx_ring); i++) { + struct mtk_rx_ring *ring = ð->rx_ring[i]; + + if (!ring->page_pool) + continue; + + page_pool_get_stats(ring->page_pool, &stats); + } + page_pool_ethtool_stats_get(data, &stats); +} + static void mtk_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *stats, u64 *data) { @@ -3175,6 +3603,8 @@ static void mtk_get_ethtool_stats(struct net_device *dev, for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++) *data_dst++ = *(data_src + mtk_ethtool_stats[i].offset); + if (mtk_page_pool_enabled(mac->hw)) + mtk_ethtool_pp_stats(mac->hw, data_dst); } while (u64_stats_fetch_retry_irq(&hwstats->syncp, start)); } @@ -3267,6 +3697,8 @@ static const struct net_device_ops mtk_netdev_ops = { .ndo_poll_controller = mtk_poll_controller, #endif .ndo_setup_tc = mtk_eth_setup_tc, + .ndo_bpf = mtk_xdp, + .ndo_xdp_xmit = mtk_xdp_xmit, }; static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h index 0a632896451a..7405c97cda66 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h @@ -18,6 +18,8 @@ #include <linux/rhashtable.h> #include <linux/dim.h> #include <linux/bitfield.h> +#include <net/page_pool.h> +#include <linux/bpf_trace.h> #include "mtk_ppe.h" #define MTK_QDMA_PAGE_SIZE 2048 @@ -49,6 +51,11 @@ #define MTK_HW_FEATURES_MT7628 (NETIF_F_SG | NETIF_F_RXCSUM) #define NEXT_DESP_IDX(X, Y) (((X) + 1) & ((Y) - 1)) +#define MTK_PP_HEADROOM XDP_PACKET_HEADROOM +#define MTK_PP_PAD (MTK_PP_HEADROOM + \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) +#define MTK_PP_MAX_BUF_SIZE (PAGE_SIZE - MTK_PP_PAD) + #define MTK_QRX_OFFSET 0x10 #define MTK_MAX_RX_RING_NUM 4 @@ -563,6 +570,16 @@ struct mtk_tx_dma_v2 { struct mtk_eth; struct mtk_mac; +struct mtk_xdp_stats { + u64 rx_xdp_redirect; + u64 rx_xdp_pass; + u64 rx_xdp_drop; + u64 rx_xdp_tx; + u64 rx_xdp_tx_errors; + u64 tx_xdp_xmit; + u64 tx_xdp_xmit_errors; +}; + /* struct mtk_hw_stats - the structure that holds the traffic statistics. * @stats_lock: make sure that stats operations are atomic * @reg_offset: the status register offset of the SoC @@ -586,6 +603,8 @@ struct mtk_hw_stats { u64 rx_checksum_errors; u64 rx_flow_control_packets; + struct mtk_xdp_stats xdp_stats; + spinlock_t stats_lock; u32 reg_offset; struct u64_stats_sync syncp; @@ -677,6 +696,12 @@ enum mtk_dev_state { MTK_RESETTING }; +enum mtk_tx_buf_type { + MTK_TYPE_SKB, + MTK_TYPE_XDP_TX, + MTK_TYPE_XDP_NDO, +}; + /* struct mtk_tx_buf - This struct holds the pointers to the memory pointed at * by the TX descriptor s * @skb: The SKB pointer of the packet being sent @@ -686,7 +711,9 @@ enum mtk_dev_state { * @dma_len1: The length of the second segment */ struct mtk_tx_buf { - struct sk_buff *skb; + enum mtk_tx_buf_type type; + void *data; + u32 flags; DEFINE_DMA_UNMAP_ADDR(dma_addr0); DEFINE_DMA_UNMAP_LEN(dma_len0); @@ -745,6 +772,9 @@ struct mtk_rx_ring { bool calc_idx_update; u16 calc_idx; u32 crx_idx_reg; + /* page_pool */ + struct page_pool *page_pool; + struct xdp_rxq_info xdp_q; }; enum mkt_eth_capabilities { @@ -1078,6 +1108,8 @@ struct mtk_eth { struct mtk_ppe *ppe; struct rhashtable flow_table; + + struct bpf_prog __rcu *prog; }; /* struct mtk_mac - the structure that holds the info about the MACs of the diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c index 90e7dfd011c9..25dc3c3aa31d 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c @@ -88,29 +88,28 @@ mtk_flow_offload_mangle_eth(const struct flow_action_entry *act, void *eth) static int mtk_flow_get_wdma_info(struct net_device *dev, const u8 *addr, struct mtk_wdma_info *info) { - struct net_device_path_ctx ctx = { - .dev = dev, - }; - struct net_device_path path = {}; + struct net_device_path_stack stack; + struct net_device_path *path; + int err; - memcpy(ctx.daddr, addr, sizeof(ctx.daddr)); + if (!dev) + return -ENODEV; if (!IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED)) return -1; - if (!dev->netdev_ops->ndo_fill_forward_path) - return -1; - - if (dev->netdev_ops->ndo_fill_forward_path(&ctx, &path)) - return -1; + err = dev_fill_forward_path(dev, addr, &stack); + if (err) + return err; - if (path.type != DEV_PATH_MTK_WDMA) + path = &stack.path[stack.num_paths - 1]; + if (path->type != DEV_PATH_MTK_WDMA) return -1; - info->wdma_idx = path.mtk_wdma.wdma_idx; - info->queue = path.mtk_wdma.queue; - info->bss = path.mtk_wdma.bss; - info->wcid = path.mtk_wdma.wcid; + info->wdma_idx = path->mtk_wdma.wdma_idx; + info->queue = path->mtk_wdma.queue; + info->bss = path->mtk_wdma.bss; + info->wcid = path->mtk_wdma.wcid; return 0; } diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c index 8f0cd3196aac..29be2fcafea3 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed.c +++ b/drivers/net/ethernet/mediatek/mtk_wed.c @@ -651,7 +651,7 @@ mtk_wed_tx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs) * WDMA RX. */ - BUG_ON(idx > ARRAY_SIZE(dev->tx_ring)); + BUG_ON(idx >= ARRAY_SIZE(dev->tx_ring)); if (mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE)) return -ENOMEM; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 7ab432cc522f..a3773a8177ed 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -28,7 +28,8 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en/rqt.o en/tir.o en/rss.o en/rx_res.o \ en_selftest.o en/port.o en/monitor_stats.o en/health.o \ en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \ en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o \ - en/qos.o en/trap.o en/fs_tt_redirect.o en/selq.o lib/crypto.o + en/qos.o en/htb.o en/trap.o en/fs_tt_redirect.o en/selq.o \ + lib/crypto.o # # Netdev extra diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index b6c15efe92ad..b07228f69b91 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -321,7 +321,8 @@ struct mlx5e_params { u8 num_tc; struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE]; struct { - struct mlx5e_mqprio_rl *rl; + u64 max_rate[TC_MAX_QUEUE]; + u32 hw_id[TC_MAX_QUEUE]; } channel; } mqprio; bool rx_cqe_compress_def; @@ -898,16 +899,8 @@ struct mlx5e_scratchpad { cpumask_var_t cpumask; }; -struct mlx5e_htb { - DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES)); - DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES); - struct mlx5e_sq_stats **qos_sq_stats; - u16 max_qos_sqs; - u16 maj_id; - u16 defcls; -}; - struct mlx5e_trap; +struct mlx5e_htb; struct mlx5e_priv { /* priv data path fields - start */ @@ -945,6 +938,8 @@ struct mlx5e_priv { struct mlx5e_channel_stats **channel_stats; struct mlx5e_channel_stats trap_stats; struct mlx5e_ptp_stats ptp_stats; + struct mlx5e_sq_stats **htb_qos_sq_stats; + u16 htb_max_qos_sqs; u16 stats_nch; u16 max_nch; u8 max_opened_tc; @@ -976,7 +971,7 @@ struct mlx5e_priv { struct mlx5e_hv_vhca_stats_agent stats_agent; #endif struct mlx5e_scratchpad scratchpad; - struct mlx5e_htb htb; + struct mlx5e_htb *htb; struct mlx5e_mqprio_rl *mqprio_rl; }; @@ -1181,7 +1176,8 @@ int mlx5e_ethtool_get_sset_count(struct mlx5e_priv *priv, int sset); void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv, struct ethtool_stats *stats, u64 *data); void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, - struct ethtool_ringparam *param); + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param); int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv, struct ethtool_ringparam *param); void mlx5e_ethtool_get_channels(struct mlx5e_priv *priv, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/htb.c b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.c new file mode 100644 index 000000000000..6dac76fa58a3 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.c @@ -0,0 +1,722 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include <net/pkt_cls.h> +#include "htb.h" +#include "en.h" +#include "../qos.h" + +struct mlx5e_qos_node { + struct hlist_node hnode; + struct mlx5e_qos_node *parent; + u64 rate; + u32 bw_share; + u32 max_average_bw; + u32 hw_id; + u32 classid; /* 16-bit, except root. */ + u16 qid; +}; + +struct mlx5e_htb { + DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES)); + DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES); + struct mlx5_core_dev *mdev; + struct net_device *netdev; + struct mlx5e_priv *priv; + struct mlx5e_selq *selq; +}; + +#define MLX5E_QOS_QID_INNER 0xffff +#define MLX5E_HTB_CLASSID_ROOT 0xffffffff + +/* Software representation of the QoS tree */ + +int mlx5e_htb_enumerate_leaves(struct mlx5e_htb *htb, mlx5e_fp_htb_enumerate callback, void *data) +{ + struct mlx5e_qos_node *node = NULL; + int bkt, err; + + hash_for_each(htb->qos_tc2node, bkt, node, hnode) { + if (node->qid == MLX5E_QOS_QID_INNER) + continue; + err = callback(data, node->qid, node->hw_id); + if (err) + return err; + } + return 0; +} + +int mlx5e_htb_cur_leaf_nodes(struct mlx5e_htb *htb) +{ + int last; + + last = find_last_bit(htb->qos_used_qids, mlx5e_qos_max_leaf_nodes(htb->mdev)); + return last == mlx5e_qos_max_leaf_nodes(htb->mdev) ? 0 : last + 1; +} + +static int mlx5e_htb_find_unused_qos_qid(struct mlx5e_htb *htb) +{ + int size = mlx5e_qos_max_leaf_nodes(htb->mdev); + struct mlx5e_priv *priv = htb->priv; + int res; + + WARN_ONCE(!mutex_is_locked(&priv->state_lock), "%s: state_lock is not held\n", __func__); + res = find_first_zero_bit(htb->qos_used_qids, size); + + return res == size ? -ENOSPC : res; +} + +static struct mlx5e_qos_node * +mlx5e_htb_node_create_leaf(struct mlx5e_htb *htb, u16 classid, u16 qid, + struct mlx5e_qos_node *parent) +{ + struct mlx5e_qos_node *node; + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return ERR_PTR(-ENOMEM); + + node->parent = parent; + + node->qid = qid; + __set_bit(qid, htb->qos_used_qids); + + node->classid = classid; + hash_add_rcu(htb->qos_tc2node, &node->hnode, classid); + + mlx5e_update_tx_netdev_queues(htb->priv); + + return node; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_create_root(struct mlx5e_htb *htb) +{ + struct mlx5e_qos_node *node; + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return ERR_PTR(-ENOMEM); + + node->qid = MLX5E_QOS_QID_INNER; + node->classid = MLX5E_HTB_CLASSID_ROOT; + hash_add_rcu(htb->qos_tc2node, &node->hnode, node->classid); + + return node; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_find(struct mlx5e_htb *htb, u32 classid) +{ + struct mlx5e_qos_node *node = NULL; + + hash_for_each_possible(htb->qos_tc2node, node, hnode, classid) { + if (node->classid == classid) + break; + } + + return node; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_find_rcu(struct mlx5e_htb *htb, u32 classid) +{ + struct mlx5e_qos_node *node = NULL; + + hash_for_each_possible_rcu(htb->qos_tc2node, node, hnode, classid) { + if (node->classid == classid) + break; + } + + return node; +} + +static void mlx5e_htb_node_delete(struct mlx5e_htb *htb, struct mlx5e_qos_node *node) +{ + hash_del_rcu(&node->hnode); + if (node->qid != MLX5E_QOS_QID_INNER) { + __clear_bit(node->qid, htb->qos_used_qids); + mlx5e_update_tx_netdev_queues(htb->priv); + } + /* Make sure this qid is no longer selected by mlx5e_select_queue, so + * that mlx5e_reactivate_qos_sq can safely restart the netdev TX queue. + */ + synchronize_net(); + kfree(node); +} + +/* TX datapath API */ + +int mlx5e_htb_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid) +{ + struct mlx5e_qos_node *node; + u16 qid; + int res; + + rcu_read_lock(); + + node = mlx5e_htb_node_find_rcu(htb, classid); + if (!node) { + res = -ENOENT; + goto out; + } + qid = READ_ONCE(node->qid); + if (qid == MLX5E_QOS_QID_INNER) { + res = -EINVAL; + goto out; + } + res = mlx5e_qid_from_qos(&htb->priv->channels, qid); + +out: + rcu_read_unlock(); + return res; +} + +/* HTB TC handlers */ + +static int +mlx5e_htb_root_add(struct mlx5e_htb *htb, u16 htb_maj_id, u16 htb_defcls, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = htb->priv; + struct mlx5e_qos_node *root; + bool opened; + int err; + + qos_dbg(htb->mdev, "TC_HTB_CREATE handle %04x:, default :%04x\n", htb_maj_id, htb_defcls); + + mlx5e_selq_prepare_htb(htb->selq, htb_maj_id, htb_defcls); + + opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + if (opened) { + err = mlx5e_qos_alloc_queues(priv, &priv->channels); + if (err) + goto err_cancel_selq; + } + + root = mlx5e_htb_node_create_root(htb); + if (IS_ERR(root)) { + err = PTR_ERR(root); + goto err_free_queues; + } + + err = mlx5_qos_create_root_node(htb->mdev, &root->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error. Try upgrading firmware."); + goto err_sw_node_delete; + } + + mlx5e_selq_apply(htb->selq); + + return 0; + +err_sw_node_delete: + mlx5e_htb_node_delete(htb, root); + +err_free_queues: + if (opened) + mlx5e_qos_close_all_queues(&priv->channels); +err_cancel_selq: + mlx5e_selq_cancel(htb->selq); + return err; +} + +static int mlx5e_htb_root_del(struct mlx5e_htb *htb) +{ + struct mlx5e_priv *priv = htb->priv; + struct mlx5e_qos_node *root; + int err; + + qos_dbg(htb->mdev, "TC_HTB_DESTROY\n"); + + /* Wait until real_num_tx_queues is updated for mlx5e_select_queue, + * so that we can safely switch to its non-HTB non-PTP fastpath. + */ + synchronize_net(); + + mlx5e_selq_prepare_htb(htb->selq, 0, 0); + mlx5e_selq_apply(htb->selq); + + root = mlx5e_htb_node_find(htb, MLX5E_HTB_CLASSID_ROOT); + if (!root) { + qos_err(htb->mdev, "Failed to find the root node in the QoS tree\n"); + return -ENOENT; + } + err = mlx5_qos_destroy_node(htb->mdev, root->hw_id); + if (err) + qos_err(htb->mdev, "Failed to destroy root node %u, err = %d\n", + root->hw_id, err); + mlx5e_htb_node_delete(htb, root); + + mlx5e_qos_deactivate_all_queues(&priv->channels); + mlx5e_qos_close_all_queues(&priv->channels); + + return err; +} + +static int mlx5e_htb_convert_rate(struct mlx5e_htb *htb, u64 rate, + struct mlx5e_qos_node *parent, u32 *bw_share) +{ + u64 share = 0; + + while (parent->classid != MLX5E_HTB_CLASSID_ROOT && !parent->max_average_bw) + parent = parent->parent; + + if (parent->max_average_bw) + share = div64_u64(div_u64(rate * 100, BYTES_IN_MBIT), + parent->max_average_bw); + else + share = 101; + + *bw_share = share == 0 ? 1 : share > 100 ? 0 : share; + + qos_dbg(htb->mdev, "Convert: rate %llu, parent ceil %llu -> bw_share %u\n", + rate, (u64)parent->max_average_bw * BYTES_IN_MBIT, *bw_share); + + return 0; +} + +static void mlx5e_htb_convert_ceil(struct mlx5e_htb *htb, u64 ceil, u32 *max_average_bw) +{ + /* Hardware treats 0 as "unlimited", set at least 1. */ + *max_average_bw = max_t(u32, div_u64(ceil, BYTES_IN_MBIT), 1); + + qos_dbg(htb->mdev, "Convert: ceil %llu -> max_average_bw %u\n", + ceil, *max_average_bw); +} + +int +mlx5e_htb_leaf_alloc_queue(struct mlx5e_htb *htb, u16 classid, + u32 parent_classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *parent; + struct mlx5e_priv *priv = htb->priv; + int qid; + int err; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_ALLOC_QUEUE classid %04x, parent %04x, rate %llu, ceil %llu\n", + classid, parent_classid, rate, ceil); + + qid = mlx5e_htb_find_unused_qos_qid(htb); + if (qid < 0) { + NL_SET_ERR_MSG_MOD(extack, "Maximum amount of leaf classes is reached."); + return qid; + } + + parent = mlx5e_htb_node_find(htb, parent_classid); + if (!parent) + return -EINVAL; + + node = mlx5e_htb_node_create_leaf(htb, classid, qid, parent); + if (IS_ERR(node)) + return PTR_ERR(node); + + node->rate = rate; + mlx5e_htb_convert_rate(htb, rate, node->parent, &node->bw_share); + mlx5e_htb_convert_ceil(htb, ceil, &node->max_average_bw); + + err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->hw_id, + node->bw_share, node->max_average_bw, + &node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + mlx5e_htb_node_delete(htb, node); + return err; + } + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, node->qid, node->hw_id); + } + } + + return mlx5e_qid_from_qos(&priv->channels, node->qid); +} + +int +mlx5e_htb_leaf_to_inner(struct mlx5e_htb *htb, u16 classid, u16 child_classid, + u64 rate, u64 ceil, struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *child; + struct mlx5e_priv *priv = htb->priv; + int err, tmp_err; + u32 new_hw_id; + u16 qid; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_TO_INNER classid %04x, upcoming child %04x, rate %llu, ceil %llu\n", + classid, child_classid, rate, ceil); + + node = mlx5e_htb_node_find(htb, classid); + if (!node) + return -ENOENT; + + err = mlx5_qos_create_inner_node(htb->mdev, node->parent->hw_id, + node->bw_share, node->max_average_bw, + &new_hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating an inner node."); + qos_err(htb->mdev, "Failed to create an inner node (class %04x), err = %d\n", + classid, err); + return err; + } + + /* Intentionally reuse the qid for the upcoming first child. */ + child = mlx5e_htb_node_create_leaf(htb, child_classid, node->qid, node); + if (IS_ERR(child)) { + err = PTR_ERR(child); + goto err_destroy_hw_node; + } + + child->rate = rate; + mlx5e_htb_convert_rate(htb, rate, node, &child->bw_share); + mlx5e_htb_convert_ceil(htb, ceil, &child->max_average_bw); + + err = mlx5_qos_create_leaf_node(htb->mdev, new_hw_id, child->bw_share, + child->max_average_bw, &child->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + goto err_delete_sw_node; + } + + /* No fail point. */ + + qid = node->qid; + /* Pairs with mlx5e_htb_get_txq_by_classid. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + node->hw_id = new_hw_id; + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, child->qid, child->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, child->qid, child->hw_id); + } + } + + return 0; + +err_delete_sw_node: + child->qid = MLX5E_QOS_QID_INNER; + mlx5e_htb_node_delete(htb, child); + +err_destroy_hw_node: + tmp_err = mlx5_qos_destroy_node(htb->mdev, new_hw_id); + if (tmp_err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to roll back creation of an inner node %u (class %04x), err = %d\n", + new_hw_id, classid, tmp_err); + return err; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_find_by_qid(struct mlx5e_htb *htb, u16 qid) +{ + struct mlx5e_qos_node *node = NULL; + int bkt; + + hash_for_each(htb->qos_tc2node, bkt, node, hnode) + if (node->qid == qid) + break; + + return node; +} + +int mlx5e_htb_leaf_del(struct mlx5e_htb *htb, u16 *classid, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = htb->priv; + struct mlx5e_qos_node *node; + struct netdev_queue *txq; + u16 qid, moved_qid; + bool opened; + int err; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL classid %04x\n", *classid); + + node = mlx5e_htb_node_find(htb, *classid); + if (!node) + return -ENOENT; + + /* Store qid for reuse. */ + qid = node->qid; + + opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + if (opened) { + txq = netdev_get_tx_queue(htb->netdev, + mlx5e_qid_from_qos(&priv->channels, qid)); + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, *classid, err); + + mlx5e_htb_node_delete(htb, node); + + moved_qid = mlx5e_htb_cur_leaf_nodes(htb); + + if (moved_qid == 0) { + /* The last QoS SQ was just destroyed. */ + if (opened) + mlx5e_reactivate_qos_sq(priv, qid, txq); + return 0; + } + moved_qid--; + + if (moved_qid < qid) { + /* The highest QoS SQ was just destroyed. */ + WARN(moved_qid != qid - 1, "Gaps in queue numeration: destroyed queue %u, the highest queue is %u", + qid, moved_qid); + if (opened) + mlx5e_reactivate_qos_sq(priv, qid, txq); + return 0; + } + + WARN(moved_qid == qid, "Can't move node with qid %u to itself", qid); + qos_dbg(htb->mdev, "Moving QoS SQ %u to %u\n", moved_qid, qid); + + node = mlx5e_htb_node_find_by_qid(htb, moved_qid); + WARN(!node, "Could not find a node with qid %u to move to queue %u", + moved_qid, qid); + + /* Stop traffic to the old queue. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + __clear_bit(moved_qid, priv->htb->qos_used_qids); + + if (opened) { + txq = netdev_get_tx_queue(htb->netdev, + mlx5e_qid_from_qos(&priv->channels, moved_qid)); + mlx5e_deactivate_qos_sq(priv, moved_qid); + mlx5e_close_qos_sq(priv, moved_qid); + } + + /* Prevent packets from the old class from getting into the new one. */ + mlx5e_reset_qdisc(htb->netdev, moved_qid); + + __set_bit(qid, htb->qos_used_qids); + WRITE_ONCE(node->qid, qid); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x) while moving qid %u to %u, err = %d\n", + node->classid, moved_qid, qid, err); + } else { + mlx5e_activate_qos_sq(priv, node->qid, node->hw_id); + } + } + + mlx5e_update_tx_netdev_queues(priv); + if (opened) + mlx5e_reactivate_qos_sq(priv, moved_qid, txq); + + *classid = node->classid; + return 0; +} + +int +mlx5e_htb_leaf_del_last(struct mlx5e_htb *htb, u16 classid, bool force, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *parent; + struct mlx5e_priv *priv = htb->priv; + u32 old_hw_id, new_hw_id; + int err, saved_err = 0; + u16 qid; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL_LAST%s classid %04x\n", + force ? "_FORCE" : "", classid); + + node = mlx5e_htb_node_find(htb, classid); + if (!node) + return -ENOENT; + + err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->parent->hw_id, + node->parent->bw_share, + node->parent->max_average_bw, + &new_hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + if (!force) + return err; + saved_err = err; + } + + /* Store qid for reuse and prevent clearing the bit. */ + qid = node->qid; + /* Pairs with mlx5e_htb_get_txq_by_classid. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + /* Prevent packets from the old class from getting into the new one. */ + mlx5e_reset_qdisc(htb->netdev, qid); + + err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + parent = node->parent; + mlx5e_htb_node_delete(htb, node); + + node = parent; + WRITE_ONCE(node->qid, qid); + + /* Early return on error in force mode. Parent will still be an inner + * node to be deleted by a following delete operation. + */ + if (saved_err) + return saved_err; + + old_hw_id = node->hw_id; + node->hw_id = new_hw_id; + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, node->qid, node->hw_id); + } + } + + err = mlx5_qos_destroy_node(htb->mdev, old_hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + return 0; +} + +static int +mlx5e_htb_update_children(struct mlx5e_htb *htb, struct mlx5e_qos_node *node, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *child; + int err = 0; + int bkt; + + hash_for_each(htb->qos_tc2node, bkt, child, hnode) { + u32 old_bw_share = child->bw_share; + int err_one; + + if (child->parent != node) + continue; + + mlx5e_htb_convert_rate(htb, child->rate, node, &child->bw_share); + if (child->bw_share == old_bw_share) + continue; + + err_one = mlx5_qos_update_node(htb->mdev, child->hw_id, child->bw_share, + child->max_average_bw, child->hw_id); + if (!err && err_one) { + err = err_one; + + NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a child node."); + qos_err(htb->mdev, "Failed to modify a child node (class %04x), err = %d\n", + node->classid, err); + } + } + + return err; +} + +int +mlx5e_htb_node_modify(struct mlx5e_htb *htb, u16 classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack) +{ + u32 bw_share, max_average_bw; + struct mlx5e_qos_node *node; + bool ceil_changed = false; + int err; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_MODIFY classid %04x, rate %llu, ceil %llu\n", + classid, rate, ceil); + + node = mlx5e_htb_node_find(htb, classid); + if (!node) + return -ENOENT; + + node->rate = rate; + mlx5e_htb_convert_rate(htb, rate, node->parent, &bw_share); + mlx5e_htb_convert_ceil(htb, ceil, &max_average_bw); + + err = mlx5_qos_update_node(htb->mdev, node->parent->hw_id, bw_share, + max_average_bw, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a node."); + qos_err(htb->mdev, "Failed to modify a node (class %04x), err = %d\n", + classid, err); + return err; + } + + if (max_average_bw != node->max_average_bw) + ceil_changed = true; + + node->bw_share = bw_share; + node->max_average_bw = max_average_bw; + + if (ceil_changed) + err = mlx5e_htb_update_children(htb, node, extack); + + return err; +} + +struct mlx5e_htb *mlx5e_htb_alloc(void) +{ + return kvzalloc(sizeof(struct mlx5e_htb), GFP_KERNEL); +} + +void mlx5e_htb_free(struct mlx5e_htb *htb) +{ + kvfree(htb); +} + +int mlx5e_htb_init(struct mlx5e_htb *htb, struct tc_htb_qopt_offload *htb_qopt, + struct net_device *netdev, struct mlx5_core_dev *mdev, + struct mlx5e_selq *selq, struct mlx5e_priv *priv) +{ + htb->mdev = mdev; + htb->netdev = netdev; + htb->selq = selq; + htb->priv = priv; + hash_init(htb->qos_tc2node); + return mlx5e_htb_root_add(htb, htb_qopt->parent_classid, htb_qopt->classid, + htb_qopt->extack); +} + +void mlx5e_htb_cleanup(struct mlx5e_htb *htb) +{ + mlx5e_htb_root_del(htb); +} + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/htb.h b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.h new file mode 100644 index 000000000000..8386f1ea4559 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5E_EN_HTB_H_ +#define __MLX5E_EN_HTB_H_ + +#include "qos.h" + +#define MLX5E_QOS_MAX_LEAF_NODES 256 + +struct mlx5e_selq; +struct mlx5e_htb; + +typedef int (*mlx5e_fp_htb_enumerate)(void *data, u16 qid, u32 hw_id); +int mlx5e_htb_enumerate_leaves(struct mlx5e_htb *htb, mlx5e_fp_htb_enumerate callback, void *data); + +int mlx5e_htb_cur_leaf_nodes(struct mlx5e_htb *htb); + +/* TX datapath API */ +int mlx5e_htb_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid); + +/* HTB TC handlers */ + +int +mlx5e_htb_leaf_alloc_queue(struct mlx5e_htb *htb, u16 classid, + u32 parent_classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack); +int +mlx5e_htb_leaf_to_inner(struct mlx5e_htb *htb, u16 classid, u16 child_classid, + u64 rate, u64 ceil, struct netlink_ext_ack *extack); +int mlx5e_htb_leaf_del(struct mlx5e_htb *htb, u16 *classid, + struct netlink_ext_ack *extack); +int +mlx5e_htb_leaf_del_last(struct mlx5e_htb *htb, u16 classid, bool force, + struct netlink_ext_ack *extack); +int +mlx5e_htb_node_modify(struct mlx5e_htb *htb, u16 classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack); +struct mlx5e_htb *mlx5e_htb_alloc(void); +void mlx5e_htb_free(struct mlx5e_htb *htb); +int mlx5e_htb_init(struct mlx5e_htb *htb, struct tc_htb_qopt_offload *htb_qopt, + struct net_device *netdev, struct mlx5_core_dev *mdev, + struct mlx5e_selq *selq, struct mlx5e_priv *priv); +void mlx5e_htb_cleanup(struct mlx5e_htb *htb); +#endif + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 047f88f09203..78ad96cf4222 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -79,19 +79,49 @@ void mlx5e_skb_cb_hwtstamp_handler(struct sk_buff *skb, int hwtstamp_type, memset(skb->cb, 0, sizeof(struct mlx5e_skb_cb_hwtstamp)); } +#define PTP_WQE_CTR2IDX(val) ((val) & ptpsq->ts_cqe_ctr_mask) + +static bool mlx5e_ptp_ts_cqe_drop(struct mlx5e_ptpsq *ptpsq, u16 skb_cc, u16 skb_id) +{ + return (ptpsq->ts_cqe_ctr_mask && (skb_cc != skb_id)); +} + +static void mlx5e_ptp_skb_fifo_ts_cqe_resync(struct mlx5e_ptpsq *ptpsq, u16 skb_cc, u16 skb_id) +{ + struct skb_shared_hwtstamps hwts = {}; + struct sk_buff *skb; + + ptpsq->cq_stats->resync_event++; + + while (skb_cc != skb_id) { + skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); + hwts.hwtstamp = mlx5e_skb_cb_get_hwts(skb)->cqe_hwtstamp; + skb_tstamp_tx(skb, &hwts); + ptpsq->cq_stats->resync_cqe++; + skb_cc = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc); + } +} + static void mlx5e_ptp_handle_ts_cqe(struct mlx5e_ptpsq *ptpsq, struct mlx5_cqe64 *cqe, int budget) { - struct sk_buff *skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); + u16 skb_id = PTP_WQE_CTR2IDX(be16_to_cpu(cqe->wqe_counter)); + u16 skb_cc = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc); struct mlx5e_txqsq *sq = &ptpsq->txqsq; + struct sk_buff *skb; ktime_t hwtstamp; if (unlikely(MLX5E_RX_ERR_CQE(cqe))) { + skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); ptpsq->cq_stats->err_cqe++; goto out; } + if (mlx5e_ptp_ts_cqe_drop(ptpsq, skb_cc, skb_id)) + mlx5e_ptp_skb_fifo_ts_cqe_resync(ptpsq, skb_cc, skb_id); + + skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); hwtstamp = mlx5e_cqe_ts_to_ns(sq->ptp_cyc2time, sq->clock, get_cqe_ts(cqe)); mlx5e_skb_cb_hwtstamp_handler(skb, MLX5E_SKB_CB_PORT_HWTSTAMP, hwtstamp, ptpsq->cq_stats); @@ -241,6 +271,7 @@ static void mlx5e_ptp_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) static int mlx5e_ptp_alloc_traffic_db(struct mlx5e_ptpsq *ptpsq, int numa) { int wq_sz = mlx5_wq_cyc_get_size(&ptpsq->txqsq.wq); + struct mlx5_core_dev *mdev = ptpsq->txqsq.mdev; ptpsq->skb_fifo.fifo = kvzalloc_node(array_size(wq_sz, sizeof(*ptpsq->skb_fifo.fifo)), GFP_KERNEL, numa); @@ -250,7 +281,9 @@ static int mlx5e_ptp_alloc_traffic_db(struct mlx5e_ptpsq *ptpsq, int numa) ptpsq->skb_fifo.pc = &ptpsq->skb_fifo_pc; ptpsq->skb_fifo.cc = &ptpsq->skb_fifo_cc; ptpsq->skb_fifo.mask = wq_sz - 1; - + if (MLX5_CAP_GEN_2(mdev, ts_cqe_metadata_size2wqe_counter)) + ptpsq->ts_cqe_ctr_mask = + (1 << MLX5_CAP_GEN_2(mdev, ts_cqe_metadata_size2wqe_counter)) - 1; return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h index a71a32e00ebb..92dbbec472ec 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h @@ -17,6 +17,7 @@ struct mlx5e_ptpsq { u16 skb_fifo_pc; struct mlx5e_skb_fifo skb_fifo; struct mlx5e_ptp_cq_stats *cq_stats; + u16 ts_cqe_ctr_mask; }; enum { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c index 9db677e9ca9c..2842195ee548 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -2,11 +2,16 @@ /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ #include <net/sch_generic.h> +#include <net/pkt_cls.h> #include "en.h" #include "params.h" #include "../qos.h" +#include "en/htb.h" -#define BYTES_IN_MBIT 125000 +struct qos_sq_callback_params { + struct mlx5e_priv *priv; + struct mlx5e_channels *chs; +}; int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes) { @@ -28,124 +33,14 @@ int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev) return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev)); } -int mlx5e_qos_cur_leaf_nodes(struct mlx5e_priv *priv) -{ - int last = find_last_bit(priv->htb.qos_used_qids, mlx5e_qos_max_leaf_nodes(priv->mdev)); - - return last == mlx5e_qos_max_leaf_nodes(priv->mdev) ? 0 : last + 1; -} - -/* Software representation of the QoS tree (internal to this file) */ - -static int mlx5e_find_unused_qos_qid(struct mlx5e_priv *priv) -{ - int size = mlx5e_qos_max_leaf_nodes(priv->mdev); - int res; - - WARN_ONCE(!mutex_is_locked(&priv->state_lock), "%s: state_lock is not held\n", __func__); - res = find_first_zero_bit(priv->htb.qos_used_qids, size); - - return res == size ? -ENOSPC : res; -} - -struct mlx5e_qos_node { - struct hlist_node hnode; - struct mlx5e_qos_node *parent; - u64 rate; - u32 bw_share; - u32 max_average_bw; - u32 hw_id; - u32 classid; /* 16-bit, except root. */ - u16 qid; -}; - -#define MLX5E_QOS_QID_INNER 0xffff -#define MLX5E_HTB_CLASSID_ROOT 0xffffffff - -static struct mlx5e_qos_node * -mlx5e_sw_node_create_leaf(struct mlx5e_priv *priv, u16 classid, u16 qid, - struct mlx5e_qos_node *parent) -{ - struct mlx5e_qos_node *node; - - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return ERR_PTR(-ENOMEM); - - node->parent = parent; - - node->qid = qid; - __set_bit(qid, priv->htb.qos_used_qids); - - node->classid = classid; - hash_add_rcu(priv->htb.qos_tc2node, &node->hnode, classid); - - mlx5e_update_tx_netdev_queues(priv); - - return node; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_create_root(struct mlx5e_priv *priv) -{ - struct mlx5e_qos_node *node; - - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return ERR_PTR(-ENOMEM); - - node->qid = MLX5E_QOS_QID_INNER; - node->classid = MLX5E_HTB_CLASSID_ROOT; - hash_add_rcu(priv->htb.qos_tc2node, &node->hnode, node->classid); - - return node; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_find(struct mlx5e_priv *priv, u32 classid) -{ - struct mlx5e_qos_node *node = NULL; - - hash_for_each_possible(priv->htb.qos_tc2node, node, hnode, classid) { - if (node->classid == classid) - break; - } - - return node; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_find_rcu(struct mlx5e_priv *priv, u32 classid) -{ - struct mlx5e_qos_node *node = NULL; - - hash_for_each_possible_rcu(priv->htb.qos_tc2node, node, hnode, classid) { - if (node->classid == classid) - break; - } - - return node; -} - -static void mlx5e_sw_node_delete(struct mlx5e_priv *priv, struct mlx5e_qos_node *node) -{ - hash_del_rcu(&node->hnode); - if (node->qid != MLX5E_QOS_QID_INNER) { - __clear_bit(node->qid, priv->htb.qos_used_qids); - mlx5e_update_tx_netdev_queues(priv); - } - /* Make sure this qid is no longer selected by mlx5e_select_queue, so - * that mlx5e_reactivate_qos_sq can safely restart the netdev TX queue. - */ - synchronize_net(); - kfree(node); -} - /* TX datapath API */ -static u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) +u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) { /* These channel params are safe to access from the datapath, because: - * 1. This function is called only after checking priv->htb.maj_id != 0, + * 1. This function is called only after checking selq->htb_maj_id != 0, * and the number of queues can't change while HTB offload is active. - * 2. When priv->htb.maj_id becomes 0, synchronize_rcu waits for + * 2. When selq->htb_maj_id becomes 0, synchronize_rcu waits for * mlx5e_select_queue to finish while holding priv->state_lock, * preventing other code from changing the number of queues. */ @@ -154,30 +49,7 @@ static u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) return (chs->params.num_channels + is_ptp) * mlx5e_get_dcb_num_tc(&chs->params) + qid; } -int mlx5e_get_txq_by_classid(struct mlx5e_priv *priv, u16 classid) -{ - struct mlx5e_qos_node *node; - u16 qid; - int res; - - rcu_read_lock(); - - node = mlx5e_sw_node_find_rcu(priv, classid); - if (!node) { - res = -ENOENT; - goto out; - } - qid = READ_ONCE(node->qid); - if (qid == MLX5E_QOS_QID_INNER) { - res = -EINVAL; - goto out; - } - res = mlx5e_qid_from_qos(&priv->channels, qid); - -out: - rcu_read_unlock(); - return res; -} +/* SQ lifecycle */ static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid) { @@ -194,10 +66,8 @@ static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid) return mlx5e_state_dereference(priv, qos_sqs[qid]); } -/* SQ lifecycle */ - -static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, - struct mlx5e_qos_node *node) +int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, + u16 node_qid, u32 hw_id) { struct mlx5e_create_cq_param ccp = {}; struct mlx5e_txqsq __rcu **qos_sqs; @@ -210,13 +80,13 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs params = &chs->params; - txq_ix = mlx5e_qid_from_qos(chs, node->qid); + txq_ix = mlx5e_qid_from_qos(chs, node_qid); - WARN_ON(node->qid > priv->htb.max_qos_sqs); - if (node->qid == priv->htb.max_qos_sqs) { + WARN_ON(node_qid > priv->htb_max_qos_sqs); + if (node_qid == priv->htb_max_qos_sqs) { struct mlx5e_sq_stats *stats, **stats_list = NULL; - if (priv->htb.max_qos_sqs == 0) { + if (priv->htb_max_qos_sqs == 0) { stats_list = kvcalloc(mlx5e_qos_max_leaf_nodes(priv->mdev), sizeof(*stats_list), GFP_KERNEL); @@ -229,16 +99,16 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs return -ENOMEM; } if (stats_list) - WRITE_ONCE(priv->htb.qos_sq_stats, stats_list); - WRITE_ONCE(priv->htb.qos_sq_stats[node->qid], stats); - /* Order max_qos_sqs increment after writing the array pointer. + WRITE_ONCE(priv->htb_qos_sq_stats, stats_list); + WRITE_ONCE(priv->htb_qos_sq_stats[node_qid], stats); + /* Order htb_max_qos_sqs increment after writing the array pointer. * Pairs with smp_load_acquire in en_stats.c. */ - smp_store_release(&priv->htb.max_qos_sqs, priv->htb.max_qos_sqs + 1); + smp_store_release(&priv->htb_max_qos_sqs, priv->htb_max_qos_sqs + 1); } - ix = node->qid % params->num_channels; - qid = node->qid / params->num_channels; + ix = node_qid % params->num_channels; + qid = node_qid / params->num_channels; c = chs->c[ix]; qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); @@ -257,8 +127,8 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs if (err) goto err_free_sq; err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params, - ¶m_sq, sq, 0, node->hw_id, - priv->htb.qos_sq_stats[node->qid]); + ¶m_sq, sq, 0, hw_id, + priv->htb_qos_sq_stats[node_qid]); if (err) goto err_close_cq; @@ -273,14 +143,22 @@ err_free_sq: return err; } -static void mlx5e_activate_qos_sq(struct mlx5e_priv *priv, struct mlx5e_qos_node *node) +static int mlx5e_open_qos_sq_cb_wrapper(void *data, u16 node_qid, u32 hw_id) +{ + struct qos_sq_callback_params *cb_params = data; + + return mlx5e_open_qos_sq(cb_params->priv, cb_params->chs, node_qid, hw_id); +} + +int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id) { + struct mlx5e_priv *priv = data; struct mlx5e_txqsq *sq; u16 qid; - sq = mlx5e_get_qos_sq(priv, node->qid); + sq = mlx5e_get_qos_sq(priv, node_qid); - qid = mlx5e_qid_from_qos(&priv->channels, node->qid); + qid = mlx5e_qid_from_qos(&priv->channels, node_qid); /* If it's a new queue, it will be marked as started at this point. * Stop it before updating txq2sq. @@ -295,11 +173,13 @@ static void mlx5e_activate_qos_sq(struct mlx5e_priv *priv, struct mlx5e_qos_node */ smp_wmb(); - qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node->qid); + qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid); mlx5e_activate_txqsq(sq); + + return 0; } -static void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) +void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) { struct mlx5e_txqsq *sq; @@ -319,7 +199,7 @@ static void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) smp_wmb(); } -static void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid) +void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid) { struct mlx5e_txqsq __rcu **qos_sqs; struct mlx5e_params *params; @@ -369,7 +249,7 @@ void mlx5e_qos_close_queues(struct mlx5e_channel *c) kvfree(qos_sqs); } -static void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) +void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) { int i; @@ -377,7 +257,7 @@ static void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) mlx5e_qos_close_queues(chs->c[i]); } -static int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) +int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) { u16 qos_sqs_size; int i; @@ -413,24 +293,20 @@ err_free: int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) { - struct mlx5e_qos_node *node = NULL; - int bkt, err; - - if (!priv->htb.maj_id) - return 0; + struct qos_sq_callback_params callback_params; + int err; err = mlx5e_qos_alloc_queues(priv, chs); if (err) return err; - hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) { - if (node->qid == MLX5E_QOS_QID_INNER) - continue; - err = mlx5e_open_qos_sq(priv, chs, node); - if (err) { - mlx5e_qos_close_all_queues(chs); - return err; - } + callback_params.priv = priv; + callback_params.chs = chs; + + err = mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_open_qos_sq_cb_wrapper, &callback_params); + if (err) { + mlx5e_qos_close_all_queues(chs); + return err; } return 0; @@ -438,14 +314,7 @@ int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) void mlx5e_qos_activate_queues(struct mlx5e_priv *priv) { - struct mlx5e_qos_node *node = NULL; - int bkt; - - hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) { - if (node->qid == MLX5E_QOS_QID_INNER) - continue; - mlx5e_activate_qos_sq(priv, node); - } + mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_activate_qos_sq, priv); } void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c) @@ -474,7 +343,7 @@ void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c) } } -static void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) +void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) { int i; @@ -482,293 +351,14 @@ static void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) mlx5e_qos_deactivate_queues(chs->c[i]); } -/* HTB API */ - -int mlx5e_htb_root_add(struct mlx5e_priv *priv, u16 htb_maj_id, u16 htb_defcls, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *root; - bool opened; - int err; - - qos_dbg(priv->mdev, "TC_HTB_CREATE handle %04x:, default :%04x\n", htb_maj_id, htb_defcls); - - if (!mlx5_qos_is_supported(priv->mdev)) { - NL_SET_ERR_MSG_MOD(extack, - "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); - return -EOPNOTSUPP; - } - - opened = test_bit(MLX5E_STATE_OPENED, &priv->state); - if (opened) { - mlx5e_selq_prepare(&priv->selq, &priv->channels.params, true); - - err = mlx5e_qos_alloc_queues(priv, &priv->channels); - if (err) - goto err_cancel_selq; - } - - root = mlx5e_sw_node_create_root(priv); - if (IS_ERR(root)) { - err = PTR_ERR(root); - goto err_free_queues; - } - - err = mlx5_qos_create_root_node(priv->mdev, &root->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error. Try upgrading firmware."); - goto err_sw_node_delete; - } - - WRITE_ONCE(priv->htb.defcls, htb_defcls); - /* Order maj_id after defcls - pairs with - * mlx5e_select_queue/mlx5e_select_htb_queues. - */ - smp_store_release(&priv->htb.maj_id, htb_maj_id); - - if (opened) - mlx5e_selq_apply(&priv->selq); - - return 0; - -err_sw_node_delete: - mlx5e_sw_node_delete(priv, root); - -err_free_queues: - if (opened) - mlx5e_qos_close_all_queues(&priv->channels); -err_cancel_selq: - mlx5e_selq_cancel(&priv->selq); - return err; -} - -int mlx5e_htb_root_del(struct mlx5e_priv *priv) -{ - struct mlx5e_qos_node *root; - int err; - - qos_dbg(priv->mdev, "TC_HTB_DESTROY\n"); - - /* Wait until real_num_tx_queues is updated for mlx5e_select_queue, - * so that we can safely switch to its non-HTB non-PTP fastpath. - */ - synchronize_net(); - - mlx5e_selq_prepare(&priv->selq, &priv->channels.params, false); - mlx5e_selq_apply(&priv->selq); - - WRITE_ONCE(priv->htb.maj_id, 0); - - root = mlx5e_sw_node_find(priv, MLX5E_HTB_CLASSID_ROOT); - if (!root) { - qos_err(priv->mdev, "Failed to find the root node in the QoS tree\n"); - return -ENOENT; - } - err = mlx5_qos_destroy_node(priv->mdev, root->hw_id); - if (err) - qos_err(priv->mdev, "Failed to destroy root node %u, err = %d\n", - root->hw_id, err); - mlx5e_sw_node_delete(priv, root); - - mlx5e_qos_deactivate_all_queues(&priv->channels); - mlx5e_qos_close_all_queues(&priv->channels); - - return err; -} - -static int mlx5e_htb_convert_rate(struct mlx5e_priv *priv, u64 rate, - struct mlx5e_qos_node *parent, u32 *bw_share) -{ - u64 share = 0; - - while (parent->classid != MLX5E_HTB_CLASSID_ROOT && !parent->max_average_bw) - parent = parent->parent; - - if (parent->max_average_bw) - share = div64_u64(div_u64(rate * 100, BYTES_IN_MBIT), - parent->max_average_bw); - else - share = 101; - - *bw_share = share == 0 ? 1 : share > 100 ? 0 : share; - - qos_dbg(priv->mdev, "Convert: rate %llu, parent ceil %llu -> bw_share %u\n", - rate, (u64)parent->max_average_bw * BYTES_IN_MBIT, *bw_share); - - return 0; -} - -static void mlx5e_htb_convert_ceil(struct mlx5e_priv *priv, u64 ceil, u32 *max_average_bw) -{ - /* Hardware treats 0 as "unlimited", set at least 1. */ - *max_average_bw = max_t(u32, div_u64(ceil, BYTES_IN_MBIT), 1); - - qos_dbg(priv->mdev, "Convert: ceil %llu -> max_average_bw %u\n", - ceil, *max_average_bw); -} - -int mlx5e_htb_leaf_alloc_queue(struct mlx5e_priv *priv, u16 classid, - u32 parent_classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *node, *parent; - int qid; - int err; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_ALLOC_QUEUE classid %04x, parent %04x, rate %llu, ceil %llu\n", - classid, parent_classid, rate, ceil); - - qid = mlx5e_find_unused_qos_qid(priv); - if (qid < 0) { - NL_SET_ERR_MSG_MOD(extack, "Maximum amount of leaf classes is reached."); - return qid; - } - - parent = mlx5e_sw_node_find(priv, parent_classid); - if (!parent) - return -EINVAL; - - node = mlx5e_sw_node_create_leaf(priv, classid, qid, parent); - if (IS_ERR(node)) - return PTR_ERR(node); - - node->rate = rate; - mlx5e_htb_convert_rate(priv, rate, node->parent, &node->bw_share); - mlx5e_htb_convert_ceil(priv, ceil, &node->max_average_bw); - - err = mlx5_qos_create_leaf_node(priv->mdev, node->parent->hw_id, - node->bw_share, node->max_average_bw, - &node->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); - qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", - classid, err); - mlx5e_sw_node_delete(priv, node); - return err; - } - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, node); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", - classid, err); - } else { - mlx5e_activate_qos_sq(priv, node); - } - } - - return mlx5e_qid_from_qos(&priv->channels, node->qid); -} - -int mlx5e_htb_leaf_to_inner(struct mlx5e_priv *priv, u16 classid, u16 child_classid, - u64 rate, u64 ceil, struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *node, *child; - int err, tmp_err; - u32 new_hw_id; - u16 qid; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_TO_INNER classid %04x, upcoming child %04x, rate %llu, ceil %llu\n", - classid, child_classid, rate, ceil); - - node = mlx5e_sw_node_find(priv, classid); - if (!node) - return -ENOENT; - - err = mlx5_qos_create_inner_node(priv->mdev, node->parent->hw_id, - node->bw_share, node->max_average_bw, - &new_hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating an inner node."); - qos_err(priv->mdev, "Failed to create an inner node (class %04x), err = %d\n", - classid, err); - return err; - } - - /* Intentionally reuse the qid for the upcoming first child. */ - child = mlx5e_sw_node_create_leaf(priv, child_classid, node->qid, node); - if (IS_ERR(child)) { - err = PTR_ERR(child); - goto err_destroy_hw_node; - } - - child->rate = rate; - mlx5e_htb_convert_rate(priv, rate, node, &child->bw_share); - mlx5e_htb_convert_ceil(priv, ceil, &child->max_average_bw); - - err = mlx5_qos_create_leaf_node(priv->mdev, new_hw_id, child->bw_share, - child->max_average_bw, &child->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); - qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", - classid, err); - goto err_delete_sw_node; - } - - /* No fail point. */ - - qid = node->qid; - /* Pairs with mlx5e_get_txq_by_classid. */ - WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - mlx5e_deactivate_qos_sq(priv, qid); - mlx5e_close_qos_sq(priv, qid); - } - - err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, classid, err); - - node->hw_id = new_hw_id; - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, child); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", - classid, err); - } else { - mlx5e_activate_qos_sq(priv, child); - } - } - - return 0; - -err_delete_sw_node: - child->qid = MLX5E_QOS_QID_INNER; - mlx5e_sw_node_delete(priv, child); - -err_destroy_hw_node: - tmp_err = mlx5_qos_destroy_node(priv->mdev, new_hw_id); - if (tmp_err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to roll back creation of an inner node %u (class %04x), err = %d\n", - new_hw_id, classid, tmp_err); - return err; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_find_by_qid(struct mlx5e_priv *priv, u16 qid) -{ - struct mlx5e_qos_node *node = NULL; - int bkt; - - hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) - if (node->qid == qid) - break; - - return node; -} - -static void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq) +void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq) { qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid); netdev_tx_reset_queue(txq); netif_tx_start_queue(txq); } -static void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) +void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) { struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid); struct Qdisc *qdisc = dev_queue->qdisc_sleeping; @@ -781,251 +371,65 @@ static void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) spin_unlock_bh(qdisc_lock(qdisc)); } -int mlx5e_htb_leaf_del(struct mlx5e_priv *priv, u16 *classid, - struct netlink_ext_ack *extack) +int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_qopt) { - struct mlx5e_qos_node *node; - struct netdev_queue *txq; - u16 qid, moved_qid; - bool opened; - int err; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_DEL classid %04x\n", *classid); - - node = mlx5e_sw_node_find(priv, *classid); - if (!node) - return -ENOENT; - - /* Store qid for reuse. */ - qid = node->qid; - - opened = test_bit(MLX5E_STATE_OPENED, &priv->state); - if (opened) { - txq = netdev_get_tx_queue(priv->netdev, - mlx5e_qid_from_qos(&priv->channels, qid)); - mlx5e_deactivate_qos_sq(priv, qid); - mlx5e_close_qos_sq(priv, qid); - } - - err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, *classid, err); - - mlx5e_sw_node_delete(priv, node); - - moved_qid = mlx5e_qos_cur_leaf_nodes(priv); - - if (moved_qid == 0) { - /* The last QoS SQ was just destroyed. */ - if (opened) - mlx5e_reactivate_qos_sq(priv, qid, txq); - return 0; - } - moved_qid--; - - if (moved_qid < qid) { - /* The highest QoS SQ was just destroyed. */ - WARN(moved_qid != qid - 1, "Gaps in queue numeration: destroyed queue %u, the highest queue is %u", - qid, moved_qid); - if (opened) - mlx5e_reactivate_qos_sq(priv, qid, txq); - return 0; - } - - WARN(moved_qid == qid, "Can't move node with qid %u to itself", qid); - qos_dbg(priv->mdev, "Moving QoS SQ %u to %u\n", moved_qid, qid); - - node = mlx5e_sw_node_find_by_qid(priv, moved_qid); - WARN(!node, "Could not find a node with qid %u to move to queue %u", - moved_qid, qid); - - /* Stop traffic to the old queue. */ - WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); - __clear_bit(moved_qid, priv->htb.qos_used_qids); - - if (opened) { - txq = netdev_get_tx_queue(priv->netdev, - mlx5e_qid_from_qos(&priv->channels, moved_qid)); - mlx5e_deactivate_qos_sq(priv, moved_qid); - mlx5e_close_qos_sq(priv, moved_qid); - } - - /* Prevent packets from the old class from getting into the new one. */ - mlx5e_reset_qdisc(priv->netdev, moved_qid); - - __set_bit(qid, priv->htb.qos_used_qids); - WRITE_ONCE(node->qid, qid); - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, node); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x) while moving qid %u to %u, err = %d\n", - node->classid, moved_qid, qid, err); - } else { - mlx5e_activate_qos_sq(priv, node); - } - } - - mlx5e_update_tx_netdev_queues(priv); - if (opened) - mlx5e_reactivate_qos_sq(priv, moved_qid, txq); - - *classid = node->classid; - return 0; -} - -int mlx5e_htb_leaf_del_last(struct mlx5e_priv *priv, u16 classid, bool force, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *node, *parent; - u32 old_hw_id, new_hw_id; - int err, saved_err = 0; - u16 qid; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_DEL_LAST%s classid %04x\n", - force ? "_FORCE" : "", classid); - - node = mlx5e_sw_node_find(priv, classid); - if (!node) - return -ENOENT; - - err = mlx5_qos_create_leaf_node(priv->mdev, node->parent->parent->hw_id, - node->parent->bw_share, - node->parent->max_average_bw, - &new_hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); - qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", - classid, err); - if (!force) - return err; - saved_err = err; - } - - /* Store qid for reuse and prevent clearing the bit. */ - qid = node->qid; - /* Pairs with mlx5e_get_txq_by_classid. */ - WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - mlx5e_deactivate_qos_sq(priv, qid); - mlx5e_close_qos_sq(priv, qid); - } - - /* Prevent packets from the old class from getting into the new one. */ - mlx5e_reset_qdisc(priv->netdev, qid); - - err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, classid, err); - - parent = node->parent; - mlx5e_sw_node_delete(priv, node); + struct mlx5e_htb *htb = priv->htb; + int res; - node = parent; - WRITE_ONCE(node->qid, qid); + if (!htb && htb_qopt->command != TC_HTB_CREATE) + return -EINVAL; - /* Early return on error in force mode. Parent will still be an inner - * node to be deleted by a following delete operation. - */ - if (saved_err) - return saved_err; - - old_hw_id = node->hw_id; - node->hw_id = new_hw_id; - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, node); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", - classid, err); - } else { - mlx5e_activate_qos_sq(priv, node); + switch (htb_qopt->command) { + case TC_HTB_CREATE: + if (!mlx5_qos_is_supported(priv->mdev)) { + NL_SET_ERR_MSG_MOD(htb_qopt->extack, + "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); + return -EOPNOTSUPP; } - } - - err = mlx5_qos_destroy_node(priv->mdev, old_hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, classid, err); - - return 0; -} - -static int mlx5e_qos_update_children(struct mlx5e_priv *priv, struct mlx5e_qos_node *node, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *child; - int err = 0; - int bkt; - - hash_for_each(priv->htb.qos_tc2node, bkt, child, hnode) { - u32 old_bw_share = child->bw_share; - int err_one; - - if (child->parent != node) - continue; - - mlx5e_htb_convert_rate(priv, child->rate, node, &child->bw_share); - if (child->bw_share == old_bw_share) - continue; - - err_one = mlx5_qos_update_node(priv->mdev, child->hw_id, child->bw_share, - child->max_average_bw, child->hw_id); - if (!err && err_one) { - err = err_one; - - NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a child node."); - qos_err(priv->mdev, "Failed to modify a child node (class %04x), err = %d\n", - node->classid, err); + priv->htb = mlx5e_htb_alloc(); + htb = priv->htb; + if (!htb) + return -ENOMEM; + res = mlx5e_htb_init(htb, htb_qopt, priv->netdev, priv->mdev, &priv->selq, priv); + if (res) { + mlx5e_htb_free(htb); + priv->htb = NULL; } + return res; + case TC_HTB_DESTROY: + mlx5e_htb_cleanup(htb); + mlx5e_htb_free(htb); + priv->htb = NULL; + return 0; + case TC_HTB_LEAF_ALLOC_QUEUE: + res = mlx5e_htb_leaf_alloc_queue(htb, htb_qopt->classid, htb_qopt->parent_classid, + htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); + if (res < 0) + return res; + htb_qopt->qid = res; + return 0; + case TC_HTB_LEAF_TO_INNER: + return mlx5e_htb_leaf_to_inner(htb, htb_qopt->parent_classid, htb_qopt->classid, + htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); + case TC_HTB_LEAF_DEL: + return mlx5e_htb_leaf_del(htb, &htb_qopt->classid, htb_qopt->extack); + case TC_HTB_LEAF_DEL_LAST: + case TC_HTB_LEAF_DEL_LAST_FORCE: + return mlx5e_htb_leaf_del_last(htb, htb_qopt->classid, + htb_qopt->command == TC_HTB_LEAF_DEL_LAST_FORCE, + htb_qopt->extack); + case TC_HTB_NODE_MODIFY: + return mlx5e_htb_node_modify(htb, htb_qopt->classid, htb_qopt->rate, htb_qopt->ceil, + htb_qopt->extack); + case TC_HTB_LEAF_QUERY_QUEUE: + res = mlx5e_htb_get_txq_by_classid(htb, htb_qopt->classid); + if (res < 0) + return res; + htb_qopt->qid = res; + return 0; + default: + return -EOPNOTSUPP; } - - return err; -} - -int mlx5e_htb_node_modify(struct mlx5e_priv *priv, u16 classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack) -{ - u32 bw_share, max_average_bw; - struct mlx5e_qos_node *node; - bool ceil_changed = false; - int err; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_MODIFY classid %04x, rate %llu, ceil %llu\n", - classid, rate, ceil); - - node = mlx5e_sw_node_find(priv, classid); - if (!node) - return -ENOENT; - - node->rate = rate; - mlx5e_htb_convert_rate(priv, rate, node->parent, &bw_share); - mlx5e_htb_convert_ceil(priv, ceil, &max_average_bw); - - err = mlx5_qos_update_node(priv->mdev, node->parent->hw_id, bw_share, - max_average_bw, node->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a node."); - qos_err(priv->mdev, "Failed to modify a node (class %04x), err = %d\n", - classid, err); - return err; - } - - if (max_average_bw != node->max_average_bw) - ceil_changed = true; - - node->bw_share = bw_share; - node->max_average_bw = max_average_bw; - - if (ceil_changed) - err = mlx5e_qos_update_children(priv, node, extack); - - return err; } struct mlx5e_mqprio_rl { @@ -1111,3 +515,4 @@ int mlx5e_mqprio_rl_get_node_hw_id(struct mlx5e_mqprio_rl *rl, int tc, u32 *hw_i *hw_id = rl->leaves_id[tc]; return 0; } + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h index 5d9bd91d86c2..4947afa23b73 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h @@ -6,40 +6,39 @@ #include <linux/mlx5/driver.h> -#define MLX5E_QOS_MAX_LEAF_NODES 256 +#define BYTES_IN_MBIT 125000 struct mlx5e_priv; +struct mlx5e_htb; struct mlx5e_channels; struct mlx5e_channel; +struct tc_htb_qopt_offload; int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes); int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev); -int mlx5e_qos_cur_leaf_nodes(struct mlx5e_priv *priv); - -/* TX datapath API */ -int mlx5e_get_txq_by_classid(struct mlx5e_priv *priv, u16 classid); /* SQ lifecycle */ +int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, + u16 node_qid, u32 hw_id); +int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id); +void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid); +void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid); +void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq); +void mlx5e_reset_qdisc(struct net_device *dev, u16 qid); + int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs); void mlx5e_qos_activate_queues(struct mlx5e_priv *priv); void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c); +void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs); void mlx5e_qos_close_queues(struct mlx5e_channel *c); +void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs); +int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs); + +/* TX datapath API */ +u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid); /* HTB API */ -int mlx5e_htb_root_add(struct mlx5e_priv *priv, u16 htb_maj_id, u16 htb_defcls, - struct netlink_ext_ack *extack); -int mlx5e_htb_root_del(struct mlx5e_priv *priv); -int mlx5e_htb_leaf_alloc_queue(struct mlx5e_priv *priv, u16 classid, - u32 parent_classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack); -int mlx5e_htb_leaf_to_inner(struct mlx5e_priv *priv, u16 classid, u16 child_classid, - u64 rate, u64 ceil, struct netlink_ext_ack *extack); -int mlx5e_htb_leaf_del(struct mlx5e_priv *priv, u16 *classid, - struct netlink_ext_ack *extack); -int mlx5e_htb_leaf_del_last(struct mlx5e_priv *priv, u16 classid, bool force, - struct netlink_ext_ack *extack); -int mlx5e_htb_node_modify(struct mlx5e_priv *priv, u16 classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack); +int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb); /* MQPRIO TX rate limit */ struct mlx5e_mqprio_rl; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c index d98a277eb7f8..f675b1926340 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c @@ -7,6 +7,7 @@ #include <linux/rcupdate.h> #include "en.h" #include "en/ptp.h" +#include "en/htb.h" struct mlx5e_selq_params { unsigned int num_regular_queues; @@ -19,6 +20,8 @@ struct mlx5e_selq_params { bool is_ptp : 1; }; }; + u16 htb_maj_id; + u16 htb_defcls; }; int mlx5e_selq_init(struct mlx5e_selq *selq, struct mutex *state_lock) @@ -44,6 +47,8 @@ int mlx5e_selq_init(struct mlx5e_selq *selq, struct mutex *state_lock) .num_tcs = 1, .is_htb = false, .is_ptp = false, + .htb_maj_id = 0, + .htb_defcls = 0, }; rcu_assign_pointer(selq->active, init_params); @@ -64,21 +69,50 @@ void mlx5e_selq_cleanup(struct mlx5e_selq *selq) selq->standby = NULL; } -void mlx5e_selq_prepare(struct mlx5e_selq *selq, struct mlx5e_params *params, bool htb) +void mlx5e_selq_prepare_params(struct mlx5e_selq *selq, struct mlx5e_params *params) { + struct mlx5e_selq_params *selq_active; + lockdep_assert_held(selq->state_lock); WARN_ON_ONCE(selq->is_prepared); selq->is_prepared = true; + selq_active = rcu_dereference_protected(selq->active, + lockdep_is_held(selq->state_lock)); + *selq->standby = *selq_active; selq->standby->num_channels = params->num_channels; selq->standby->num_tcs = mlx5e_get_dcb_num_tc(params); selq->standby->num_regular_queues = selq->standby->num_channels * selq->standby->num_tcs; - selq->standby->is_htb = htb; selq->standby->is_ptp = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_TX_PORT_TS); } +bool mlx5e_selq_is_htb_enabled(struct mlx5e_selq *selq) +{ + struct mlx5e_selq_params *selq_active = + rcu_dereference_protected(selq->active, lockdep_is_held(selq->state_lock)); + + return selq_active->htb_maj_id; +} + +void mlx5e_selq_prepare_htb(struct mlx5e_selq *selq, u16 htb_maj_id, u16 htb_defcls) +{ + struct mlx5e_selq_params *selq_active; + + lockdep_assert_held(selq->state_lock); + WARN_ON_ONCE(selq->is_prepared); + + selq->is_prepared = true; + + selq_active = rcu_dereference_protected(selq->active, + lockdep_is_held(selq->state_lock)); + *selq->standby = *selq_active; + selq->standby->is_htb = htb_maj_id; + selq->standby->htb_maj_id = htb_maj_id; + selq->standby->htb_defcls = htb_defcls; +} + void mlx5e_selq_apply(struct mlx5e_selq *selq) { struct mlx5e_selq_params *old_params; @@ -137,20 +171,21 @@ static u16 mlx5e_select_ptpsq(struct net_device *dev, struct sk_buff *skb, return selq->num_regular_queues + up; } -static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb) +static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb, + struct mlx5e_selq_params *selq) { u16 classid; /* Order maj_id before defcls - pairs with mlx5e_htb_root_add. */ - if ((TC_H_MAJ(skb->priority) >> 16) == smp_load_acquire(&priv->htb.maj_id)) + if ((TC_H_MAJ(skb->priority) >> 16) == selq->htb_maj_id) classid = TC_H_MIN(skb->priority); else - classid = READ_ONCE(priv->htb.defcls); + classid = selq->htb_defcls; if (!classid) return 0; - return mlx5e_get_txq_by_classid(priv, classid); + return mlx5e_htb_get_txq_by_classid(priv->htb, classid); } u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, @@ -187,10 +222,10 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, up * selq->num_channels; } - if (unlikely(selq->is_htb)) { + if (unlikely(selq->htb_maj_id)) { /* num_tcs == 1, shortcut for PTP */ - txq_ix = mlx5e_select_htb_queue(priv, skb); + txq_ix = mlx5e_select_htb_queue(priv, skb, selq); if (txq_ix > 0) return txq_ix; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h index 6c070141d8f1..fd590f80e4d1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h @@ -21,7 +21,9 @@ struct sk_buff; int mlx5e_selq_init(struct mlx5e_selq *selq, struct mutex *state_lock); void mlx5e_selq_cleanup(struct mlx5e_selq *selq); -void mlx5e_selq_prepare(struct mlx5e_selq *selq, struct mlx5e_params *params, bool htb); +void mlx5e_selq_prepare_params(struct mlx5e_selq *selq, struct mlx5e_params *params); +void mlx5e_selq_prepare_htb(struct mlx5e_selq *selq, u16 htb_maj_id, u16 htb_defcls); +bool mlx5e_selq_is_htb_enabled(struct mlx5e_selq *selq); void mlx5e_selq_apply(struct mlx5e_selq *selq); void mlx5e_selq_cancel(struct mlx5e_selq *selq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c index 2093cc2b0d48..33c1411ed8db 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c @@ -36,7 +36,7 @@ mlx5e_tc_post_act_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains, int err; if (!MLX5_CAP_FLOWTABLE_TYPE(priv->mdev, ignore_flow_level, table_type)) { - if (priv->mdev->coredev_type != MLX5_COREDEV_VF) + if (priv->mdev->coredev_type == MLX5_COREDEV_PF) mlx5_core_warn(priv->mdev, "firmware level support is missing\n"); err = -EOPNOTSUPP; goto err_check; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index a6d84ff3a0e7..864ce0c393e6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -2062,7 +2062,7 @@ mlx5_tc_ct_init_check_support(struct mlx5e_priv *priv, /* Ignore_flow_level support isn't supported by default for VFs and so post_act * won't be supported. Skip showing error msg. */ - if (priv->mdev->coredev_type != MLX5_COREDEV_VF) + if (priv->mdev->coredev_type == MLX5_COREDEV_PF) err_msg = "post action is missing"; err = -EOPNOTSUPP; goto out_err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 6e80585d731f..b811207fe5ed 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -30,6 +30,8 @@ * SOFTWARE. */ +#include <linux/ethtool_netlink.h> + #include "en.h" #include "en/port.h" #include "en/params.h" @@ -305,12 +307,18 @@ static void mlx5e_get_ethtool_stats(struct net_device *dev, } void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param) { param->rx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE; param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE; param->rx_pending = 1 << priv->channels.params.log_rq_mtu_frames; param->tx_pending = 1 << priv->channels.params.log_sq_size; + + kernel_param->tcp_data_split = + (priv->channels.params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) ? + ETHTOOL_TCP_DATA_SPLIT_ENABLED : + ETHTOOL_TCP_DATA_SPLIT_DISABLED; } static void mlx5e_get_ringparam(struct net_device *dev, @@ -320,7 +328,7 @@ static void mlx5e_get_ringparam(struct net_device *dev, { struct mlx5e_priv *priv = netdev_priv(dev); - mlx5e_ethtool_get_ringparam(priv, param); + mlx5e_ethtool_get_ringparam(priv, param, kernel_param); } int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv, @@ -451,7 +459,7 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, * because the numeration of the QoS SQs will change, while per-queue * qdiscs are attached. */ - if (priv->htb.maj_id) { + if (mlx5e_selq_is_htb_enabled(&priv->selq)) { err = -EINVAL; netdev_err(priv->netdev, "%s: HTB offload is active, cannot change the number of channels\n", __func__); @@ -2067,7 +2075,7 @@ static int set_pflag_tx_port_ts(struct net_device *netdev, bool enable) * the numeration of the QoS SQs will change, while per-queue qdiscs are * attached. */ - if (priv->htb.maj_id) { + if (mlx5e_selq_is_htb_enabled(&priv->selq)) { netdev_err(priv->netdev, "%s: HTB offload is active, cannot change the PTP state\n", __func__); return -EINVAL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index cac4022ba7f3..180b2f418339 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -31,7 +31,6 @@ */ #include <net/tc_act/tc_gact.h> -#include <net/pkt_cls.h> #include <linux/mlx5/fs.h> #include <net/vxlan.h> #include <net/geneve.h> @@ -64,6 +63,7 @@ #include "en/devlink.h" #include "lib/mlx5.h" #include "en/ptp.h" +#include "en/htb.h" #include "qos.h" #include "en/trap.h" @@ -1912,8 +1912,7 @@ static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, { int tc; - if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL || - !params->mqprio.channel.rl) { + if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) { *hw_id = 0; return 0; } @@ -1922,7 +1921,14 @@ static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, if (tc < 0) return tc; - return mlx5e_mqprio_rl_get_node_hw_id(params->mqprio.channel.rl, tc, hw_id); + if (tc >= params->mqprio.num_tc) { + WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u", + tc, params->mqprio.num_tc); + return -EINVAL; + } + + *hw_id = params->mqprio.channel.hw_id[tc]; + return 0; } static int mlx5e_open_sqs(struct mlx5e_channel *c, @@ -2383,9 +2389,11 @@ int mlx5e_open_channels(struct mlx5e_priv *priv, goto err_close_channels; } - err = mlx5e_qos_open_queues(priv, chs); - if (err) - goto err_close_ptp; + if (priv->htb) { + err = mlx5e_qos_open_queues(priv, chs); + if (err) + goto err_close_ptp; + } mlx5e_health_channels_update(priv); kvfree(cparam); @@ -2567,9 +2575,11 @@ static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc, int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv) { - int qos_queues, nch, ntc, num_txqs, err; + int nch, ntc, num_txqs, err; + int qos_queues = 0; - qos_queues = mlx5e_qos_cur_leaf_nodes(priv); + if (priv->htb) + qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb); nch = priv->channels.params.num_channels; ntc = mlx5e_get_dcb_num_tc(&priv->channels.params); @@ -2615,13 +2625,6 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err); goto err_txqs; } - if (priv->mqprio_rl != priv->channels.params.mqprio.channel.rl) { - if (priv->mqprio_rl) { - mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); - mlx5e_mqprio_rl_free(priv->mqprio_rl); - } - priv->mqprio_rl = priv->channels.params.mqprio.channel.rl; - } return 0; @@ -2724,7 +2727,8 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) { mlx5e_build_txq_maps(priv); mlx5e_activate_channels(&priv->channels); - mlx5e_qos_activate_queues(priv); + if (priv->htb) + mlx5e_qos_activate_queues(priv); mlx5e_xdp_tx_enable(priv); /* dev_watchdog() wants all TX queues to be started when the carrier is @@ -2841,7 +2845,7 @@ int mlx5e_safe_switch_params(struct mlx5e_priv *priv, new_chs.params = *params; - mlx5e_selq_prepare(&priv->selq, &new_chs.params, !!priv->htb.maj_id); + mlx5e_selq_prepare_params(&priv->selq, &new_chs.params); err = mlx5e_open_channels(priv, &new_chs); if (err) @@ -2897,7 +2901,7 @@ int mlx5e_open_locked(struct net_device *netdev) struct mlx5e_priv *priv = netdev_priv(netdev); int err; - mlx5e_selq_prepare(&priv->selq, &priv->channels.params, !!priv->htb.maj_id); + mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params); set_bit(MLX5E_STATE_OPENED, &priv->state); @@ -3135,6 +3139,11 @@ err_close_tises: static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) { + if (priv->mqprio_rl) { + mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); + mlx5e_mqprio_rl_free(priv->mqprio_rl); + priv->mqprio_rl = NULL; + } mlx5e_destroy_tises(priv); } @@ -3203,19 +3212,38 @@ static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc) { params->mqprio.mode = TC_MQPRIO_MODE_DCB; params->mqprio.num_tc = num_tc; - params->mqprio.channel.rl = NULL; mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc, params->num_channels); } +static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params, + struct mlx5e_mqprio_rl *rl) +{ + int tc; + + for (tc = 0; tc < TC_MAX_QUEUE; tc++) { + u32 hw_id = 0; + + if (rl) + mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id); + params->mqprio.channel.hw_id[tc] = hw_id; + } +} + static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params, - struct tc_mqprio_qopt *qopt, + struct tc_mqprio_qopt_offload *mqprio, struct mlx5e_mqprio_rl *rl) { + int tc; + params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL; - params->mqprio.num_tc = qopt->num_tc; - params->mqprio.channel.rl = rl; - mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, qopt); + params->mqprio.num_tc = mqprio->qopt.num_tc; + + for (tc = 0; tc < TC_MAX_QUEUE; tc++) + params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc]; + + mlx5e_mqprio_rl_update_params(params, rl); + mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt); } static void mlx5e_params_mqprio_reset(struct mlx5e_params *params) @@ -3241,6 +3269,12 @@ static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv, err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_num_channels_changed_ctx, NULL, true); + if (!err && priv->mqprio_rl) { + mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); + mlx5e_mqprio_rl_free(priv->mqprio_rl); + priv->mqprio_rl = NULL; + } + priv->max_opened_tc = max_t(u8, priv->max_opened_tc, mlx5e_get_dcb_num_tc(&priv->channels.params)); return err; @@ -3299,16 +3333,38 @@ static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv, return 0; } -static bool mlx5e_mqprio_rate_limit(struct tc_mqprio_qopt_offload *mqprio) +static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[]) { int tc; - for (tc = 0; tc < mqprio->qopt.num_tc; tc++) - if (mqprio->max_rate[tc]) + for (tc = 0; tc < num_tc; tc++) + if (max_rate[tc]) return true; return false; } +static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev, + u8 num_tc, u64 max_rate[]) +{ + struct mlx5e_mqprio_rl *rl; + int err; + + if (!mlx5e_mqprio_rate_limit(num_tc, max_rate)) + return NULL; + + rl = mlx5e_mqprio_rl_alloc(); + if (!rl) + return ERR_PTR(-ENOMEM); + + err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate); + if (err) { + mlx5e_mqprio_rl_free(rl); + return ERR_PTR(err); + } + + return rl; +} + static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, struct tc_mqprio_qopt_offload *mqprio) { @@ -3322,32 +3378,32 @@ static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, if (err) return err; - rl = NULL; - if (mlx5e_mqprio_rate_limit(mqprio)) { - rl = mlx5e_mqprio_rl_alloc(); - if (!rl) - return -ENOMEM; - err = mlx5e_mqprio_rl_init(rl, priv->mdev, mqprio->qopt.num_tc, - mqprio->max_rate); - if (err) { - mlx5e_mqprio_rl_free(rl); - return err; - } - } + rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate); + if (IS_ERR(rl)) + return PTR_ERR(rl); new_params = priv->channels.params; - mlx5e_params_mqprio_channel_set(&new_params, &mqprio->qopt, rl); + mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl); nch_changed = mlx5e_get_dcb_num_tc(&priv->channels.params) > 1; preactivate = nch_changed ? mlx5e_num_channels_changed_ctx : mlx5e_update_netdev_queues_ctx; err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, true); - if (err && rl) { - mlx5e_mqprio_rl_cleanup(rl); - mlx5e_mqprio_rl_free(rl); + if (err) { + if (rl) { + mlx5e_mqprio_rl_cleanup(rl); + mlx5e_mqprio_rl_free(rl); + } + return err; } - return err; + if (priv->mqprio_rl) { + mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); + mlx5e_mqprio_rl_free(priv->mqprio_rl); + } + priv->mqprio_rl = rl; + + return 0; } static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, @@ -3356,7 +3412,7 @@ static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, /* MQPRIO is another toplevel qdisc that can't be attached * simultaneously with the offloaded HTB. */ - if (WARN_ON(priv->htb.maj_id)) + if (WARN_ON(mlx5e_selq_is_htb_enabled(&priv->selq))) return -EINVAL; switch (mqprio->mode) { @@ -3369,47 +3425,6 @@ static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, } } -static int mlx5e_setup_tc_htb(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb) -{ - int res; - - switch (htb->command) { - case TC_HTB_CREATE: - return mlx5e_htb_root_add(priv, htb->parent_classid, htb->classid, - htb->extack); - case TC_HTB_DESTROY: - return mlx5e_htb_root_del(priv); - case TC_HTB_LEAF_ALLOC_QUEUE: - res = mlx5e_htb_leaf_alloc_queue(priv, htb->classid, htb->parent_classid, - htb->rate, htb->ceil, htb->extack); - if (res < 0) - return res; - htb->qid = res; - return 0; - case TC_HTB_LEAF_TO_INNER: - return mlx5e_htb_leaf_to_inner(priv, htb->parent_classid, htb->classid, - htb->rate, htb->ceil, htb->extack); - case TC_HTB_LEAF_DEL: - return mlx5e_htb_leaf_del(priv, &htb->classid, htb->extack); - case TC_HTB_LEAF_DEL_LAST: - case TC_HTB_LEAF_DEL_LAST_FORCE: - return mlx5e_htb_leaf_del_last(priv, htb->classid, - htb->command == TC_HTB_LEAF_DEL_LAST_FORCE, - htb->extack); - case TC_HTB_NODE_MODIFY: - return mlx5e_htb_node_modify(priv, htb->classid, htb->rate, htb->ceil, - htb->extack); - case TC_HTB_LEAF_QUERY_QUEUE: - res = mlx5e_get_txq_by_classid(priv, htb->classid); - if (res < 0) - return res; - htb->qid = res; - return 0; - default: - return -EOPNOTSUPP; - } -} - static LIST_HEAD(mlx5e_block_cb_list); static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, @@ -3443,7 +3458,7 @@ static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, return err; case TC_SETUP_QDISC_HTB: mutex_lock(&priv->state_lock); - err = mlx5e_setup_tc_htb(priv, type_data); + err = mlx5e_htb_setup_tc(priv, type_data); mutex_unlock(&priv->state_lock); return err; default: @@ -3663,6 +3678,7 @@ static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) static int set_feature_hw_tc(struct net_device *netdev, bool enable) { struct mlx5e_priv *priv = netdev_priv(netdev); + int err = 0; #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) if (!enable && mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD))) { @@ -3672,12 +3688,14 @@ static int set_feature_hw_tc(struct net_device *netdev, bool enable) } #endif - if (!enable && priv->htb.maj_id) { + mutex_lock(&priv->state_lock); + if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) { netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n"); - return -EINVAL; + err = -EINVAL; } + mutex_unlock(&priv->state_lock); - return 0; + return err; } static int set_feature_rx_all(struct net_device *netdev, bool enable) @@ -5102,6 +5120,23 @@ static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) priv->rx_res = NULL; } +static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv) +{ + struct mlx5e_params *params; + struct mlx5e_mqprio_rl *rl; + + params = &priv->channels.params; + if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) + return; + + rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc, + params->mqprio.channel.max_rate); + if (IS_ERR(rl)) + rl = NULL; + priv->mqprio_rl = rl; + mlx5e_mqprio_rl_update_params(params, rl); +} + static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) { int err; @@ -5112,6 +5147,7 @@ static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) return err; } + mlx5e_set_mqprio_rl(priv); mlx5e_dcbnl_initialize(priv); return 0; } @@ -5285,7 +5321,6 @@ int mlx5e_priv_init(struct mlx5e_priv *priv, if (err) goto err_free_cpumask; - hash_init(priv->htb.qos_tc2node); INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); @@ -5342,14 +5377,9 @@ void mlx5e_priv_cleanup(struct mlx5e_priv *priv) mutex_unlock(&priv->state_lock); free_cpumask_var(priv->scratchpad.cpumask); - for (i = 0; i < priv->htb.max_qos_sqs; i++) - kfree(priv->htb.qos_sq_stats[i]); - kvfree(priv->htb.qos_sq_stats); - - if (priv->mqprio_rl) { - mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); - mlx5e_mqprio_rl_free(priv->mqprio_rl); - } + for (i = 0; i < priv->htb_max_qos_sqs; i++) + kfree(priv->htb_qos_sq_stats[i]); + kvfree(priv->htb_qos_sq_stats); memset(priv, 0, sizeof(*priv)); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index f797fd97d305..ae90b06d21e2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -229,7 +229,7 @@ mlx5e_rep_get_ringparam(struct net_device *dev, { struct mlx5e_priv *priv = netdev_priv(dev); - mlx5e_ethtool_get_ringparam(priv, param); + mlx5e_ethtool_get_ringparam(priv, param, kernel_param); } static int diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 1e87bb2b7541..7409829d1201 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -474,8 +474,8 @@ static void mlx5e_stats_grp_sw_update_stats_qos(struct mlx5e_priv *priv, int i; /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); - stats = READ_ONCE(priv->htb.qos_sq_stats); + max_qos_sqs = smp_load_acquire(&priv->htb_max_qos_sqs); + stats = READ_ONCE(priv->htb_qos_sq_stats); for (i = 0; i < max_qos_sqs; i++) { mlx5e_stats_grp_sw_update_stats_sq(s, READ_ONCE(stats[i])); @@ -2100,6 +2100,8 @@ static const struct counter_desc ptp_cq_stats_desc[] = { { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, err_cqe) }, { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, abort) }, { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, abort_abs_diff_ns) }, + { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, resync_cqe) }, + { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, resync_event) }, }; static const struct counter_desc ptp_rq_stats_desc[] = { @@ -2184,13 +2186,13 @@ static const struct counter_desc qos_sq_stats_desc[] = { static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(qos) { /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - return NUM_QOS_SQ_STATS * smp_load_acquire(&priv->htb.max_qos_sqs); + return NUM_QOS_SQ_STATS * smp_load_acquire(&priv->htb_max_qos_sqs); } static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(qos) { /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - u16 max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); + u16 max_qos_sqs = smp_load_acquire(&priv->htb_max_qos_sqs); int i, qid; for (qid = 0; qid < max_qos_sqs; qid++) @@ -2208,8 +2210,8 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(qos) int i, qid; /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); - stats = READ_ONCE(priv->htb.qos_sq_stats); + max_qos_sqs = smp_load_acquire(&priv->htb_max_qos_sqs); + stats = READ_ONCE(priv->htb_qos_sq_stats); for (qid = 0; qid < max_qos_sqs; qid++) { struct mlx5e_sq_stats *s = READ_ONCE(stats[qid]); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index e48b15b55b6f..ed4fc940e4ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -453,6 +453,8 @@ struct mlx5e_ptp_cq_stats { u64 err_cqe; u64 abort; u64 abort_abs_diff_ns; + u64 resync_cqe; + u64 resync_event; }; struct mlx5e_stats { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 699d3a9886bd..dc1e01e93d5a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -631,12 +631,22 @@ void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq) mlx5e_tx_mpwqe_session_complete(sq); } +static void mlx5e_cqe_ts_id_eseg(struct mlx5e_ptpsq *ptpsq, struct sk_buff *skb, + struct mlx5_wqe_eth_seg *eseg) +{ + if (ptpsq->ts_cqe_ctr_mask && unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) + eseg->flow_table_metadata = cpu_to_be32(ptpsq->skb_fifo_pc & + ptpsq->ts_cqe_ctr_mask); +} + static void mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5e_accel_tx_state *accel, struct mlx5_wqe_eth_seg *eseg, u16 ihs) { mlx5e_accel_tx_eseg(priv, skb, eseg, ihs); mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg); + if (unlikely(sq->ptpsq)) + mlx5e_cqe_ts_id_eseg(sq->ptpsq, skb, eseg); } netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c index 8da73ef5680f..ac3757beaea2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c @@ -83,7 +83,7 @@ static void mlx5i_get_ringparam(struct net_device *dev, { struct mlx5e_priv *priv = mlx5i_epriv(dev); - mlx5e_ethtool_get_ringparam(priv, param); + mlx5e_ethtool_get_ringparam(priv, param, kernel_param); } static int mlx5i_set_channels(struct net_device *dev, diff --git a/drivers/net/ethernet/mellanox/mlxsw/cmd.h b/drivers/net/ethernet/mellanox/mlxsw/cmd.h index 666d6b6e4dbf..60232fb8ccd7 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/cmd.h +++ b/drivers/net/ethernet/mellanox/mlxsw/cmd.h @@ -329,6 +329,32 @@ MLXSW_ITEM64(cmd_mbox, query_fw, free_running_clock_offset, 0x50, 0, 64); */ MLXSW_ITEM32(cmd_mbox, query_fw, fr_rn_clk_bar, 0x58, 30, 2); +/* cmd_mbox_query_fw_utc_sec_offset + * The offset of the UTC_Sec page + */ +MLXSW_ITEM64(cmd_mbox, query_fw, utc_sec_offset, 0x70, 0, 64); + +/* cmd_mbox_query_fw_utc_sec_bar + * PCI base address register (BAR) of the UTC_Sec page + * 0: BAR 0 + * 1: 64 bit BAR + * Reserved on SwitchX/-2, Switch-IB/2, Spectrum-1 + */ +MLXSW_ITEM32(cmd_mbox, query_fw, utc_sec_bar, 0x78, 30, 2); + +/* cmd_mbox_query_fw_utc_nsec_offset + * The offset of the UTC_nSec page + */ +MLXSW_ITEM64(cmd_mbox, query_fw, utc_nsec_offset, 0x80, 0, 64); + +/* cmd_mbox_query_fw_utc_nsec_bar + * PCI base address register (BAR) of the UTC_nSec page + * 0: BAR 0 + * 1: 64 bit BAR + * Reserved on SwitchX/-2, Switch-IB/2, Spectrum-1 + */ +MLXSW_ITEM32(cmd_mbox, query_fw, utc_nsec_bar, 0x88, 30, 2); + /* QUERY_BOARDINFO - Query Board Information * ----------------------------------------- * OpMod == 0 (N/A), INMmod == 0 (N/A) @@ -663,6 +689,12 @@ MLXSW_ITEM32(cmd_mbox, config_profile, set_kvd_hash_double_size, 0x0C, 26, 1); */ MLXSW_ITEM32(cmd_mbox, config_profile, set_cqe_version, 0x08, 0, 1); +/* cmd_mbox_config_set_cqe_time_stamp_type + * Capability bit. Setting a bit to 1 configures the profile + * according to the mailbox contents. + */ +MLXSW_ITEM32(cmd_mbox, config_profile, set_cqe_time_stamp_type, 0x08, 2, 1); + /* cmd_mbox_config_profile_max_vepa_channels * Maximum number of VEPA channels per port (0 through 16) * 0 - multi-channel VEPA is disabled @@ -858,6 +890,26 @@ MLXSW_ITEM32_INDEXED(cmd_mbox, config_profile, swid_config_type, MLXSW_ITEM32_INDEXED(cmd_mbox, config_profile, swid_config_properties, 0x60, 0, 8, 0x08, 0x00, false); +enum mlxsw_cmd_mbox_config_profile_cqe_time_stamp_type { + /* uSec - 1.024uSec (default). Only bits 15:0 are valid. */ + MLXSW_CMD_MBOX_CONFIG_PROFILE_CQE_TIME_STAMP_TYPE_USEC, + /* FRC - Free Running Clock, units of 1nSec. + * Reserved when SwitchX/-2, Switch-IB/2 and Spectrum-1. + */ + MLXSW_CMD_MBOX_CONFIG_PROFILE_CQE_TIME_STAMP_TYPE_FRC, + /* UTC. time_stamp[37:30] = Sec, time_stamp[29:0] = nSec. + * Reserved when SwitchX/2, Switch-IB/2 and Spectrum-1. + */ + MLXSW_CMD_MBOX_CONFIG_PROFILE_CQE_TIME_STAMP_TYPE_UTC, +}; + +/* cmd_mbox_config_profile_cqe_time_stamp_type + * CQE time_stamp_type for non-mirror-packets. + * Configured if set_cqe_time_stamp_type is set. + * Reserved when SwitchX/-2, Switch-IB/2 and Spectrum-1. + */ +MLXSW_ITEM32(cmd_mbox, config_profile, cqe_time_stamp_type, 0xB0, 8, 2); + /* cmd_mbox_config_profile_cqe_version * CQE version: * 0: CQE version is 0 diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index ab1cebf227fb..1b61bc8f59a2 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -127,11 +127,11 @@ static int mlxsw_core_resources_ports_register(struct mlxsw_core *mlxsw_core) max_ports, 1, DEVLINK_RESOURCE_UNIT_ENTRY); - return devlink_resource_register(devlink, - DEVLINK_RESOURCE_GENERIC_NAME_PORTS, - max_ports, MLXSW_CORE_RESOURCE_PORTS, - DEVLINK_RESOURCE_ID_PARENT_TOP, - &ports_num_params); + return devl_resource_register(devlink, + DEVLINK_RESOURCE_GENERIC_NAME_PORTS, + max_ports, MLXSW_CORE_RESOURCE_PORTS, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &ports_num_params); } static int mlxsw_ports_init(struct mlxsw_core *mlxsw_core, bool reload) @@ -157,8 +157,8 @@ static int mlxsw_ports_init(struct mlxsw_core *mlxsw_core, bool reload) goto err_resources_ports_register; } atomic_set(&mlxsw_core->active_ports_count, 0); - devlink_resource_occ_get_register(devlink, MLXSW_CORE_RESOURCE_PORTS, - mlxsw_ports_occ_get, mlxsw_core); + devl_resource_occ_get_register(devlink, MLXSW_CORE_RESOURCE_PORTS, + mlxsw_ports_occ_get, mlxsw_core); return 0; @@ -171,9 +171,9 @@ static void mlxsw_ports_fini(struct mlxsw_core *mlxsw_core, bool reload) { struct devlink *devlink = priv_to_devlink(mlxsw_core); - devlink_resource_occ_get_unregister(devlink, MLXSW_CORE_RESOURCE_PORTS); + devl_resource_occ_get_unregister(devlink, MLXSW_CORE_RESOURCE_PORTS); if (!reload) - devlink_resources_unregister(priv_to_devlink(mlxsw_core)); + devl_resources_unregister(priv_to_devlink(mlxsw_core)); kfree(mlxsw_core->ports); } @@ -1488,7 +1488,9 @@ mlxsw_devlink_core_bus_device_reload_down(struct devlink *devlink, if (!(mlxsw_core->bus->features & MLXSW_BUS_F_RESET)) return -EOPNOTSUPP; + devl_lock(devlink); mlxsw_core_bus_device_unregister(mlxsw_core, true); + devl_unlock(devlink); return 0; } @@ -1498,13 +1500,17 @@ mlxsw_devlink_core_bus_device_reload_up(struct devlink *devlink, enum devlink_re struct netlink_ext_ack *extack) { struct mlxsw_core *mlxsw_core = devlink_priv(devlink); + int err; *actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) | BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE); - return mlxsw_core_bus_device_register(mlxsw_core->bus_info, - mlxsw_core->bus, - mlxsw_core->bus_priv, true, - devlink, extack); + devl_lock(devlink); + err = mlxsw_core_bus_device_register(mlxsw_core->bus_info, + mlxsw_core->bus, + mlxsw_core->bus_priv, true, + devlink, extack); + devl_unlock(devlink); + return err; } static int mlxsw_devlink_flash_update(struct devlink *devlink, @@ -2102,6 +2108,7 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, err = -ENOMEM; goto err_devlink_alloc; } + devl_lock(devlink); } mlxsw_core = devlink_priv(devlink); @@ -2187,6 +2194,7 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, if (!reload) { devlink_set_features(devlink, DEVLINK_F_RELOAD); + devl_unlock(devlink); devlink_register(devlink); } return 0; @@ -2214,12 +2222,14 @@ err_alloc_lag_mapping: mlxsw_ports_fini(mlxsw_core, reload); err_ports_init: if (!reload) - devlink_resources_unregister(devlink); + devl_resources_unregister(devlink); err_register_resources: mlxsw_bus->fini(bus_priv); err_bus_init: - if (!reload) + if (!reload) { + devl_unlock(devlink); devlink_free(devlink); + } err_devlink_alloc: return err; } @@ -2255,8 +2265,10 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, { struct devlink *devlink = priv_to_devlink(mlxsw_core); - if (!reload) + if (!reload) { devlink_unregister(devlink); + devl_lock(devlink); + } if (devlink_is_reload_failed(devlink)) { if (!reload) @@ -2281,16 +2293,19 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, kfree(mlxsw_core->lag.mapping); mlxsw_ports_fini(mlxsw_core, reload); if (!reload) - devlink_resources_unregister(devlink); + devl_resources_unregister(devlink); mlxsw_core->bus->fini(mlxsw_core->bus_priv); - if (!reload) + if (!reload) { + devl_unlock(devlink); devlink_free(devlink); + } return; reload_fail_deinit: mlxsw_core_params_unregister(mlxsw_core); - devlink_resources_unregister(devlink); + devl_resources_unregister(devlink); + devl_unlock(devlink); devlink_free(devlink); } EXPORT_SYMBOL(mlxsw_core_bus_device_unregister); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index a3491ef2aa7e..9d2e8a8d3a75 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -296,7 +296,8 @@ struct mlxsw_config_profile { used_ar_sec:1, used_adaptive_routing_group_cap:1, used_ubridge:1, - used_kvd_sizes:1; + used_kvd_sizes:1, + used_cqe_time_stamp_type:1; u8 max_vepa_channels; u16 max_mid; u16 max_pgt; @@ -319,6 +320,7 @@ struct mlxsw_config_profile { u32 kvd_linear_size; u8 kvd_hash_single_parts; u8 kvd_hash_double_parts; + u8 cqe_time_stamp_type; struct mlxsw_swid_config swid_config[MLXSW_CONFIG_PROFILE_SWID_COUNT]; }; @@ -485,7 +487,7 @@ struct mlxsw_bus_info { u8 vsd[MLXSW_CMD_BOARDINFO_VSD_LEN]; u8 psid[MLXSW_CMD_BOARDINFO_PSID_LEN]; u8 low_frequency:1, - read_frc_capable:1; + read_clock_capable:1; }; struct mlxsw_hwmon; diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 41f0f68bc911..83659fb0559a 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -505,6 +505,12 @@ static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci, mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num); } +static unsigned int mlxsw_pci_read32_off(struct mlxsw_pci *mlxsw_pci, + ptrdiff_t off) +{ + return ioread32be(mlxsw_pci->hw_addr + off); +} + static void mlxsw_pci_cqe_sdq_handle(struct mlxsw_pci *mlxsw_pci, struct mlxsw_pci_queue *q, u16 consumer_counter_limit, @@ -1267,6 +1273,13 @@ static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox, mlxsw_cmd_mbox_config_profile_cqe_version_set(mbox, 1); } + if (profile->used_cqe_time_stamp_type) { + mlxsw_cmd_mbox_config_profile_set_cqe_time_stamp_type_set(mbox, + 1); + mlxsw_cmd_mbox_config_profile_cqe_time_stamp_type_set(mbox, + profile->cqe_time_stamp_type); + } + return mlxsw_cmd_config_profile_set(mlxsw_pci->core, mbox); } @@ -1802,19 +1815,19 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod, static u32 mlxsw_pci_read_frc_h(void *bus_priv) { struct mlxsw_pci *mlxsw_pci = bus_priv; - u64 frc_offset; + u64 frc_offset_h; - frc_offset = mlxsw_pci->free_running_clock_offset; - return mlxsw_pci_read32(mlxsw_pci, FREE_RUNNING_CLOCK_H(frc_offset)); + frc_offset_h = mlxsw_pci->free_running_clock_offset; + return mlxsw_pci_read32_off(mlxsw_pci, frc_offset_h); } static u32 mlxsw_pci_read_frc_l(void *bus_priv) { struct mlxsw_pci *mlxsw_pci = bus_priv; - u64 frc_offset; + u64 frc_offset_l; - frc_offset = mlxsw_pci->free_running_clock_offset; - return mlxsw_pci_read32(mlxsw_pci, FREE_RUNNING_CLOCK_L(frc_offset)); + frc_offset_l = mlxsw_pci->free_running_clock_offset + 4; + return mlxsw_pci_read32_off(mlxsw_pci, frc_offset_l); } static const struct mlxsw_bus mlxsw_pci_bus = { @@ -1916,7 +1929,7 @@ static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) mlxsw_pci->bus_info.device_kind = driver_name; mlxsw_pci->bus_info.device_name = pci_name(mlxsw_pci->pdev); mlxsw_pci->bus_info.dev = &pdev->dev; - mlxsw_pci->bus_info.read_frc_capable = true; + mlxsw_pci->bus_info.read_clock_capable = true; mlxsw_pci->id = id; err = mlxsw_core_bus_device_register(&mlxsw_pci->bus_info, diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h index 7b531228d6c0..48dbfea0a2a1 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h +++ b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h @@ -41,9 +41,6 @@ #define MLXSW_PCI_DOORBELL(offset, type_offset, num) \ ((offset) + (type_offset) + (num) * 4) -#define MLXSW_PCI_FREE_RUNNING_CLOCK_H(offset) (offset) -#define MLXSW_PCI_FREE_RUNNING_CLOCK_L(offset) ((offset) + 4) - #define MLXSW_PCI_CQS_MAX 96 #define MLXSW_PCI_EQS_COUNT 2 #define MLXSW_PCI_EQ_ASYNC_NUM 0 @@ -217,6 +214,25 @@ MLXSW_ITEM32(pci, cqe0, dqn, 0x0C, 1, 5); MLXSW_ITEM32(pci, cqe12, dqn, 0x0C, 1, 6); mlxsw_pci_cqe_item_helpers(dqn, 0, 12, 12); +/* pci_cqe_time_stamp_low + * Time stamp of the CQE + * Format according to time_stamp_type: + * 0: uSec - 1.024uSec (default for devices which do not support + * time_stamp_type). Only bits 15:0 are valid + * 1: FRC - Free Running Clock - units of 1nSec + * 2: UTC - time_stamp[37:30] = Sec + * - time_stamp[29:0] = nSec + * 3: Mirror_UTC. UTC time stamp of the original packet that has + * MIRROR_SESSION traps + * - time_stamp[37:30] = Sec + * - time_stamp[29:0] = nSec + * Formats 0..2 are configured by + * CONFIG_PROFILE.cqe_time_stamp_type for PTP traps + * Format 3 is used for MIRROR_SESSION traps + * Note that Spectrum does not reveal FRC, UTC and Mirror_UTC + */ +MLXSW_ITEM32(pci, cqe2, time_stamp_low, 0x0C, 16, 16); + #define MLXSW_PCI_CQE2_MIRROR_TCLASS_INVALID 0x1F /* pci_cqe_mirror_tclass @@ -280,8 +296,67 @@ MLXSW_ITEM32(pci, cqe2, user_def_val_orig_pkt_len, 0x14, 0, 20); */ MLXSW_ITEM32(pci, cqe2, mirror_reason, 0x18, 24, 8); +enum mlxsw_pci_cqe_time_stamp_type { + MLXSW_PCI_CQE_TIME_STAMP_TYPE_USEC, + MLXSW_PCI_CQE_TIME_STAMP_TYPE_FRC, + MLXSW_PCI_CQE_TIME_STAMP_TYPE_UTC, + MLXSW_PCI_CQE_TIME_STAMP_TYPE_MIRROR_UTC, +}; + +/* pci_cqe_time_stamp_type + * Time stamp type: + * 0: uSec - 1.024uSec (default for devices which do not support + * time_stamp_type) + * 1: FRC - Free Running Clock - units of 1nSec + * 2: UTC + * 3: Mirror_UTC. UTC time stamp of the original packet that has + * MIRROR_SESSION traps + */ +MLXSW_ITEM32(pci, cqe2, time_stamp_type, 0x18, 22, 2); + #define MLXSW_PCI_CQE2_MIRROR_LATENCY_INVALID 0xFFFFFF +/* pci_cqe_time_stamp_high + * Time stamp of the CQE + * Format according to time_stamp_type: + * 0: uSec - 1.024uSec (default for devices which do not support + * time_stamp_type). Only bits 15:0 are valid + * 1: FRC - Free Running Clock - units of 1nSec + * 2: UTC - time_stamp[37:30] = Sec + * - time_stamp[29:0] = nSec + * 3: Mirror_UTC. UTC time stamp of the original packet that has + * MIRROR_SESSION traps + * - time_stamp[37:30] = Sec + * - time_stamp[29:0] = nSec + * Formats 0..2 are configured by + * CONFIG_PROFILE.cqe_time_stamp_type for PTP traps + * Format 3 is used for MIRROR_SESSION traps + * Note that Spectrum does not reveal FRC, UTC and Mirror_UTC + */ +MLXSW_ITEM32(pci, cqe2, time_stamp_high, 0x18, 0, 22); + +static inline u64 mlxsw_pci_cqe2_time_stamp_get(const char *cqe) +{ + u64 ts_high = mlxsw_pci_cqe2_time_stamp_high_get(cqe); + u64 ts_low = mlxsw_pci_cqe2_time_stamp_low_get(cqe); + + return ts_high << 16 | ts_low; +} + +static inline u8 mlxsw_pci_cqe2_time_stamp_sec_get(const char *cqe) +{ + u64 full_ts = mlxsw_pci_cqe2_time_stamp_get(cqe); + + return full_ts >> 30 & 0xFF; +} + +static inline u32 mlxsw_pci_cqe2_time_stamp_nsec_get(const char *cqe) +{ + u64 full_ts = mlxsw_pci_cqe2_time_stamp_get(cqe); + + return full_ts & 0x3FFFFFFF; +} + /* pci_cqe_mirror_latency * End-to-end latency of the original packet that does mirroring to the CPU. * Value of 0xFFFFFF means that the latency is invalid. Units are according to diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index 17ce28e65464..ddab5476c8b2 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -10347,6 +10347,8 @@ MLXSW_REG_DEFINE(mtutc, MLXSW_REG_MTUTC_ID, MLXSW_REG_MTUTC_LEN); enum mlxsw_reg_mtutc_operation { MLXSW_REG_MTUTC_OPERATION_SET_TIME_AT_NEXT_SEC = 0, + MLXSW_REG_MTUTC_OPERATION_SET_TIME_IMMEDIATE = 1, + MLXSW_REG_MTUTC_OPERATION_ADJUST_TIME = 2, MLXSW_REG_MTUTC_OPERATION_ADJUST_FREQ = 3, }; @@ -10359,25 +10361,50 @@ MLXSW_ITEM32(reg, mtutc, operation, 0x00, 0, 4); /* reg_mtutc_freq_adjustment * Frequency adjustment: Every PPS the HW frequency will be * adjusted by this value. Units of HW clock, where HW counts - * 10^9 HW clocks for 1 HW second. + * 10^9 HW clocks for 1 HW second. Range is from -50,000,000 to +50,000,000. + * In Spectrum-2, the field is reversed, positive values mean to decrease the + * frequency. * Access: RW */ MLXSW_ITEM32(reg, mtutc, freq_adjustment, 0x04, 0, 32); +#define MLXSW_REG_MTUTC_MAX_FREQ_ADJ (50 * 1000 * 1000) + /* reg_mtutc_utc_sec * UTC seconds. * Access: WO */ MLXSW_ITEM32(reg, mtutc, utc_sec, 0x10, 0, 32); +/* reg_mtutc_utc_nsec + * UTC nSecs. + * Range 0..(10^9-1) + * Updated when operation is SET_TIME_IMMEDIATE. + * Reserved on Spectrum-1. + * Access: WO + */ +MLXSW_ITEM32(reg, mtutc, utc_nsec, 0x14, 0, 30); + +/* reg_mtutc_time_adjustment + * Time adjustment. + * Units of nSec. + * Range is from -32768 to +32767. + * Updated when operation is ADJUST_TIME. + * Reserved on Spectrum-1. + * Access: WO + */ +MLXSW_ITEM32(reg, mtutc, time_adjustment, 0x18, 0, 32); + static inline void mlxsw_reg_mtutc_pack(char *payload, enum mlxsw_reg_mtutc_operation oper, - u32 freq_adj, u32 utc_sec) + u32 freq_adj, u32 utc_sec, u32 utc_nsec, u32 time_adj) { MLXSW_REG_ZERO(mtutc, payload); mlxsw_reg_mtutc_operation_set(payload, oper); mlxsw_reg_mtutc_freq_adjustment_set(payload, freq_adj); mlxsw_reg_mtutc_utc_sec_set(payload, utc_sec); + mlxsw_reg_mtutc_utc_nsec_set(payload, utc_nsec); + mlxsw_reg_mtutc_time_adjustment_set(payload, time_adj); } /* MCQI - Management Component Query Information @@ -11045,15 +11072,76 @@ MLXSW_ITEM32(reg, mtptpt, trap_id, 0x00, 0, 4); */ MLXSW_ITEM32(reg, mtptpt, message_type, 0x04, 0, 16); -static inline void mlxsw_reg_mtptptp_pack(char *payload, - enum mlxsw_reg_mtptpt_trap_id trap_id, - u16 message_type) +static inline void mlxsw_reg_mtptpt_pack(char *payload, + enum mlxsw_reg_mtptpt_trap_id trap_id, + u16 message_type) { MLXSW_REG_ZERO(mtptpt, payload); mlxsw_reg_mtptpt_trap_id_set(payload, trap_id); mlxsw_reg_mtptpt_message_type_set(payload, message_type); } +/* MTPCPC - Monitoring Time Precision Correction Port Configuration Register + * ------------------------------------------------------------------------- + */ +#define MLXSW_REG_MTPCPC_ID 0x9093 +#define MLXSW_REG_MTPCPC_LEN 0x2C + +MLXSW_REG_DEFINE(mtpcpc, MLXSW_REG_MTPCPC_ID, MLXSW_REG_MTPCPC_LEN); + +/* reg_mtpcpc_pport + * Per port: + * 0: config is global. When reading - the local_port is 1. + * 1: config is per port. + * Access: Index + */ +MLXSW_ITEM32(reg, mtpcpc, pport, 0x00, 31, 1); + +/* reg_mtpcpc_local_port + * Local port number. + * Supported to/from CPU port. + * Reserved when pport = 0. + * Access: Index + */ +MLXSW_ITEM32_LP(reg, mtpcpc, 0x00, 16, 0x00, 12); + +/* reg_mtpcpc_ptp_trap_en + * Enable PTP traps. + * The trap_id is configured by MTPTPT. + * Access: RW + */ +MLXSW_ITEM32(reg, mtpcpc, ptp_trap_en, 0x04, 0, 1); + +/* reg_mtpcpc_ing_correction_message_type + * Bitwise vector of PTP message types to update correction-field at ingress. + * MessageType field as defined by IEEE 1588 Each bit corresponds to a value + * (e.g. Bit0: Sync, Bit1: Delay_Req). Supported also from CPU port. + * Default all 0 + * Access: RW + */ +MLXSW_ITEM32(reg, mtpcpc, ing_correction_message_type, 0x10, 0, 16); + +/* reg_mtpcpc_egr_correction_message_type + * Bitwise vector of PTP message types to update correction-field at egress. + * MessageType field as defined by IEEE 1588 Each bit corresponds to a value + * (e.g. Bit0: Sync, Bit1: Delay_Req). Supported also from CPU port. + * Default all 0 + * Access: RW + */ +MLXSW_ITEM32(reg, mtpcpc, egr_correction_message_type, 0x14, 0, 16); + +static inline void mlxsw_reg_mtpcpc_pack(char *payload, bool pport, + u16 local_port, bool ptp_trap_en, + u16 ing, u16 egr) +{ + MLXSW_REG_ZERO(mtpcpc, payload); + mlxsw_reg_mtpcpc_pport_set(payload, pport); + mlxsw_reg_mtpcpc_local_port_set(payload, pport ? local_port : 0); + mlxsw_reg_mtpcpc_ptp_trap_en_set(payload, ptp_trap_en); + mlxsw_reg_mtpcpc_ing_correction_message_type_set(payload, ing); + mlxsw_reg_mtpcpc_egr_correction_message_type_set(payload, egr); +} + /* MFGD - Monitoring FW General Debug Register * ------------------------------------------- */ @@ -12770,6 +12858,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(mtpppc), MLXSW_REG(mtpptr), MLXSW_REG(mtptpt), + MLXSW_REG(mtpcpc), MLXSW_REG(mfgd), MLXSW_REG(mgpir), MLXSW_REG(mbct), diff --git a/drivers/net/ethernet/mellanox/mlxsw/resources.h b/drivers/net/ethernet/mellanox/mlxsw/resources.h index 826e47fb4586..19ae0d1c74a8 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/resources.h +++ b/drivers/net/ethernet/mellanox/mlxsw/resources.h @@ -24,6 +24,7 @@ enum mlxsw_res_id { MLXSW_RES_ID_COUNTER_SIZE_PACKETS_BYTES, MLXSW_RES_ID_COUNTER_SIZE_ROUTER_BASIC, MLXSW_RES_ID_MAX_SYSTEM_PORT, + MLXSW_RES_ID_FID, MLXSW_RES_ID_MAX_LAG, MLXSW_RES_ID_MAX_LAG_MEMBERS, MLXSW_RES_ID_GUARANTEED_SHARED_BUFFER, @@ -83,6 +84,7 @@ static u16 mlxsw_res_ids[] = { [MLXSW_RES_ID_COUNTER_SIZE_PACKETS_BYTES] = 0x2443, [MLXSW_RES_ID_COUNTER_SIZE_ROUTER_BASIC] = 0x2449, [MLXSW_RES_ID_MAX_SYSTEM_PORT] = 0x2502, + [MLXSW_RES_ID_FID] = 0x2512, [MLXSW_RES_ID_MAX_LAG] = 0x2520, [MLXSW_RES_ID_MAX_LAG_MEMBERS] = 0x2521, [MLXSW_RES_ID_GUARANTEED_SHARED_BUFFER] = 0x2805, /* Bytes */ diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index a703ca257198..641078060b02 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -166,7 +166,7 @@ MLXSW_ITEM32(tx, hdr, port_mid, 0x04, 16, 16); * set, otherwise calculated based on the packet's VID using VID to FID mapping. * Valid for data packets only. */ -MLXSW_ITEM32(tx, hdr, fid, 0x08, 0, 16); +MLXSW_ITEM32(tx, hdr, fid, 0x08, 16, 16); /* tx_hdr_type * 0 - Data packets @@ -1999,7 +1999,6 @@ __mlxsw_sp_port_mapping_events_cancel(struct mlxsw_sp *mlxsw_sp) static void mlxsw_sp_ports_remove(struct mlxsw_sp *mlxsw_sp) { unsigned int max_ports = mlxsw_core_max_ports(mlxsw_sp->core); - struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); int i; for (i = 1; i < max_ports; i++) @@ -2007,12 +2006,10 @@ static void mlxsw_sp_ports_remove(struct mlxsw_sp *mlxsw_sp) /* Make sure all scheduled events are processed */ __mlxsw_sp_port_mapping_events_cancel(mlxsw_sp); - devl_lock(devlink); for (i = 1; i < max_ports; i++) if (mlxsw_sp_port_created(mlxsw_sp, i)) mlxsw_sp_port_remove(mlxsw_sp, i); mlxsw_sp_cpu_port_remove(mlxsw_sp); - devl_unlock(devlink); kfree(mlxsw_sp->ports); mlxsw_sp->ports = NULL; } @@ -2034,7 +2031,6 @@ mlxsw_sp_ports_remove_selected(struct mlxsw_core *mlxsw_core, static int mlxsw_sp_ports_create(struct mlxsw_sp *mlxsw_sp) { unsigned int max_ports = mlxsw_core_max_ports(mlxsw_sp->core); - struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); struct mlxsw_sp_port_mapping_events *events; struct mlxsw_sp_port_mapping *port_mapping; size_t alloc_size; @@ -2057,7 +2053,6 @@ static int mlxsw_sp_ports_create(struct mlxsw_sp *mlxsw_sp) goto err_event_enable; } - devl_lock(devlink); err = mlxsw_sp_cpu_port_create(mlxsw_sp); if (err) goto err_cpu_port_create; @@ -2070,7 +2065,6 @@ static int mlxsw_sp_ports_create(struct mlxsw_sp *mlxsw_sp) if (err) goto err_port_create; } - devl_unlock(devlink); return 0; err_port_create: @@ -2080,7 +2074,6 @@ err_port_create: i = max_ports; mlxsw_sp_cpu_port_remove(mlxsw_sp); err_cpu_port_create: - devl_unlock(devlink); err_event_enable: for (i--; i >= 1; i--) mlxsw_sp_port_mapping_event_set(mlxsw_sp, i, false); @@ -3103,7 +3096,7 @@ static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core, goto err_router_init; } - if (mlxsw_sp->bus_info->read_frc_capable) { + if (mlxsw_sp->bus_info->read_clock_capable) { /* NULL is a valid return value from clock_init */ mlxsw_sp->clock = mlxsw_sp->ptp_ops->clock_init(mlxsw_sp, @@ -3418,6 +3411,8 @@ static const struct mlxsw_config_profile mlxsw_sp2_config_profile = { .type = MLXSW_PORT_SWID_TYPE_ETH, } }, + .used_cqe_time_stamp_type = 1, + .cqe_time_stamp_type = MLXSW_CMD_MBOX_CONFIG_PROFILE_CQE_TIME_STAMP_TYPE_UTC, }; static void @@ -3477,19 +3472,19 @@ static int mlxsw_sp1_resources_kvd_register(struct mlxsw_core *mlxsw_core) &hash_single_size_params); kvd_size = MLXSW_CORE_RES_GET(mlxsw_core, KVD_SIZE); - err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD, - kvd_size, MLXSW_SP_RESOURCE_KVD, - DEVLINK_RESOURCE_ID_PARENT_TOP, - &kvd_size_params); + err = devl_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD, + kvd_size, MLXSW_SP_RESOURCE_KVD, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &kvd_size_params); if (err) return err; linear_size = profile->kvd_linear_size; - err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_LINEAR, - linear_size, - MLXSW_SP_RESOURCE_KVD_LINEAR, - MLXSW_SP_RESOURCE_KVD, - &linear_size_params); + err = devl_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_LINEAR, + linear_size, + MLXSW_SP_RESOURCE_KVD_LINEAR, + MLXSW_SP_RESOURCE_KVD, + &linear_size_params); if (err) return err; @@ -3502,20 +3497,20 @@ static int mlxsw_sp1_resources_kvd_register(struct mlxsw_core *mlxsw_core) double_size /= profile->kvd_hash_double_parts + profile->kvd_hash_single_parts; double_size = rounddown(double_size, MLXSW_SP_KVD_GRANULARITY); - err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_HASH_DOUBLE, - double_size, - MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE, - MLXSW_SP_RESOURCE_KVD, - &hash_double_size_params); + err = devl_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_HASH_DOUBLE, + double_size, + MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE, + MLXSW_SP_RESOURCE_KVD, + &hash_double_size_params); if (err) return err; single_size = kvd_size - double_size - linear_size; - err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_HASH_SINGLE, - single_size, - MLXSW_SP_RESOURCE_KVD_HASH_SINGLE, - MLXSW_SP_RESOURCE_KVD, - &hash_single_size_params); + err = devl_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_HASH_SINGLE, + single_size, + MLXSW_SP_RESOURCE_KVD_HASH_SINGLE, + MLXSW_SP_RESOURCE_KVD, + &hash_single_size_params); if (err) return err; @@ -3536,10 +3531,10 @@ static int mlxsw_sp2_resources_kvd_register(struct mlxsw_core *mlxsw_core) MLXSW_SP_KVD_GRANULARITY, DEVLINK_RESOURCE_UNIT_ENTRY); - return devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD, - kvd_size, MLXSW_SP_RESOURCE_KVD, - DEVLINK_RESOURCE_ID_PARENT_TOP, - &kvd_size_params); + return devl_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD, + kvd_size, MLXSW_SP_RESOURCE_KVD, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &kvd_size_params); } static int mlxsw_sp_resources_span_register(struct mlxsw_core *mlxsw_core) @@ -3555,10 +3550,10 @@ static int mlxsw_sp_resources_span_register(struct mlxsw_core *mlxsw_core) devlink_resource_size_params_init(&span_size_params, max_span, max_span, 1, DEVLINK_RESOURCE_UNIT_ENTRY); - return devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_SPAN, - max_span, MLXSW_SP_RESOURCE_SPAN, - DEVLINK_RESOURCE_ID_PARENT_TOP, - &span_size_params); + return devl_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_SPAN, + max_span, MLXSW_SP_RESOURCE_SPAN, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &span_size_params); } static int @@ -3577,12 +3572,12 @@ mlxsw_sp_resources_rif_mac_profile_register(struct mlxsw_core *mlxsw_core) max_rif_mac_profiles, 1, DEVLINK_RESOURCE_UNIT_ENTRY); - return devlink_resource_register(devlink, - "rif_mac_profiles", - max_rif_mac_profiles, - MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, - DEVLINK_RESOURCE_ID_PARENT_TOP, - &size_params); + return devl_resource_register(devlink, + "rif_mac_profiles", + max_rif_mac_profiles, + MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &size_params); } static int mlxsw_sp_resources_rifs_register(struct mlxsw_core *mlxsw_core) @@ -3598,10 +3593,10 @@ static int mlxsw_sp_resources_rifs_register(struct mlxsw_core *mlxsw_core) devlink_resource_size_params_init(&size_params, max_rifs, max_rifs, 1, DEVLINK_RESOURCE_UNIT_ENTRY); - return devlink_resource_register(devlink, "rifs", max_rifs, - MLXSW_SP_RESOURCE_RIFS, - DEVLINK_RESOURCE_ID_PARENT_TOP, - &size_params); + return devl_resource_register(devlink, "rifs", max_rifs, + MLXSW_SP_RESOURCE_RIFS, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &size_params); } static int mlxsw_sp1_resources_register(struct mlxsw_core *mlxsw_core) @@ -3639,7 +3634,7 @@ err_resources_rif_mac_profile_register: err_policer_resources_register: err_resources_counter_register: err_resources_span_register: - devlink_resources_unregister(priv_to_devlink(mlxsw_core)); + devl_resources_unregister(priv_to_devlink(mlxsw_core)); return err; } @@ -3678,7 +3673,7 @@ err_resources_rif_mac_profile_register: err_policer_resources_register: err_resources_counter_register: err_resources_span_register: - devlink_resources_unregister(priv_to_devlink(mlxsw_core)); + devl_resources_unregister(priv_to_devlink(mlxsw_core)); return err; } @@ -3702,15 +3697,15 @@ static int mlxsw_sp_kvd_sizes_get(struct mlxsw_core *mlxsw_core, * granularity from the profile. In case the user * provided the sizes they are obtained via devlink. */ - err = devlink_resource_size_get(devlink, - MLXSW_SP_RESOURCE_KVD_LINEAR, - p_linear_size); + err = devl_resource_size_get(devlink, + MLXSW_SP_RESOURCE_KVD_LINEAR, + p_linear_size); if (err) *p_linear_size = profile->kvd_linear_size; - err = devlink_resource_size_get(devlink, - MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE, - p_double_size); + err = devl_resource_size_get(devlink, + MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE, + p_double_size); if (err) { double_size = MLXSW_CORE_RES_GET(mlxsw_core, KVD_SIZE) - *p_linear_size; @@ -3721,9 +3716,9 @@ static int mlxsw_sp_kvd_sizes_get(struct mlxsw_core *mlxsw_core, MLXSW_SP_KVD_GRANULARITY); } - err = devlink_resource_size_get(devlink, - MLXSW_SP_RESOURCE_KVD_HASH_SINGLE, - p_single_size); + err = devl_resource_size_get(devlink, + MLXSW_SP_RESOURCE_KVD_HASH_SINGLE, + p_single_size); if (err) *p_single_size = MLXSW_CORE_RES_GET(mlxsw_core, KVD_SIZE) - *p_double_size - *p_linear_size; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum1_kvdl.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum1_kvdl.c index d20e794e01ca..1e3fc989393c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum1_kvdl.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum1_kvdl.c @@ -216,8 +216,8 @@ mlxsw_sp1_kvdl_part_init(struct mlxsw_sp *mlxsw_sp, u64 resource_size; int err; - err = devlink_resource_size_get(devlink, info->resource_id, - &resource_size); + err = devl_resource_size_get(devlink, info->resource_id, + &resource_size); if (err) { need_update = false; resource_size = info->end_index - info->start_index + 1; @@ -338,22 +338,22 @@ static int mlxsw_sp1_kvdl_init(struct mlxsw_sp *mlxsw_sp, void *priv) err = mlxsw_sp1_kvdl_parts_init(mlxsw_sp, kvdl); if (err) return err; - devlink_resource_occ_get_register(devlink, - MLXSW_SP_RESOURCE_KVD_LINEAR, - mlxsw_sp1_kvdl_occ_get, - kvdl); - devlink_resource_occ_get_register(devlink, - MLXSW_SP_RESOURCE_KVD_LINEAR_SINGLE, - mlxsw_sp1_kvdl_single_occ_get, - kvdl); - devlink_resource_occ_get_register(devlink, - MLXSW_SP_RESOURCE_KVD_LINEAR_CHUNKS, - mlxsw_sp1_kvdl_chunks_occ_get, - kvdl); - devlink_resource_occ_get_register(devlink, - MLXSW_SP_RESOURCE_KVD_LINEAR_LARGE_CHUNKS, - mlxsw_sp1_kvdl_large_chunks_occ_get, - kvdl); + devl_resource_occ_get_register(devlink, + MLXSW_SP_RESOURCE_KVD_LINEAR, + mlxsw_sp1_kvdl_occ_get, + kvdl); + devl_resource_occ_get_register(devlink, + MLXSW_SP_RESOURCE_KVD_LINEAR_SINGLE, + mlxsw_sp1_kvdl_single_occ_get, + kvdl); + devl_resource_occ_get_register(devlink, + MLXSW_SP_RESOURCE_KVD_LINEAR_CHUNKS, + mlxsw_sp1_kvdl_chunks_occ_get, + kvdl); + devl_resource_occ_get_register(devlink, + MLXSW_SP_RESOURCE_KVD_LINEAR_LARGE_CHUNKS, + mlxsw_sp1_kvdl_large_chunks_occ_get, + kvdl); return 0; } @@ -362,14 +362,14 @@ static void mlxsw_sp1_kvdl_fini(struct mlxsw_sp *mlxsw_sp, void *priv) struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); struct mlxsw_sp1_kvdl *kvdl = priv; - devlink_resource_occ_get_unregister(devlink, - MLXSW_SP_RESOURCE_KVD_LINEAR_LARGE_CHUNKS); - devlink_resource_occ_get_unregister(devlink, - MLXSW_SP_RESOURCE_KVD_LINEAR_CHUNKS); - devlink_resource_occ_get_unregister(devlink, - MLXSW_SP_RESOURCE_KVD_LINEAR_SINGLE); - devlink_resource_occ_get_unregister(devlink, - MLXSW_SP_RESOURCE_KVD_LINEAR); + devl_resource_occ_get_unregister(devlink, + MLXSW_SP_RESOURCE_KVD_LINEAR_LARGE_CHUNKS); + devl_resource_occ_get_unregister(devlink, + MLXSW_SP_RESOURCE_KVD_LINEAR_CHUNKS); + devl_resource_occ_get_unregister(devlink, + MLXSW_SP_RESOURCE_KVD_LINEAR_SINGLE); + devl_resource_occ_get_unregister(devlink, + MLXSW_SP_RESOURCE_KVD_LINEAR); mlxsw_sp1_kvdl_parts_fini(kvdl); } @@ -396,32 +396,32 @@ int mlxsw_sp1_kvdl_resources_register(struct mlxsw_core *mlxsw_core) devlink_resource_size_params_init(&size_params, 0, kvdl_max_size, MLXSW_SP1_KVDL_SINGLE_ALLOC_SIZE, DEVLINK_RESOURCE_UNIT_ENTRY); - err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_LINEAR_SINGLES, - MLXSW_SP1_KVDL_SINGLE_SIZE, - MLXSW_SP_RESOURCE_KVD_LINEAR_SINGLE, - MLXSW_SP_RESOURCE_KVD_LINEAR, - &size_params); + err = devl_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_LINEAR_SINGLES, + MLXSW_SP1_KVDL_SINGLE_SIZE, + MLXSW_SP_RESOURCE_KVD_LINEAR_SINGLE, + MLXSW_SP_RESOURCE_KVD_LINEAR, + &size_params); if (err) return err; devlink_resource_size_params_init(&size_params, 0, kvdl_max_size, MLXSW_SP1_KVDL_CHUNKS_ALLOC_SIZE, DEVLINK_RESOURCE_UNIT_ENTRY); - err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_LINEAR_CHUNKS, - MLXSW_SP1_KVDL_CHUNKS_SIZE, - MLXSW_SP_RESOURCE_KVD_LINEAR_CHUNKS, - MLXSW_SP_RESOURCE_KVD_LINEAR, - &size_params); + err = devl_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_LINEAR_CHUNKS, + MLXSW_SP1_KVDL_CHUNKS_SIZE, + MLXSW_SP_RESOURCE_KVD_LINEAR_CHUNKS, + MLXSW_SP_RESOURCE_KVD_LINEAR, + &size_params); if (err) return err; devlink_resource_size_params_init(&size_params, 0, kvdl_max_size, MLXSW_SP1_KVDL_LARGE_CHUNKS_ALLOC_SIZE, DEVLINK_RESOURCE_UNIT_ENTRY); - err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_LINEAR_LARGE_CHUNKS, - MLXSW_SP1_KVDL_LARGE_CHUNKS_SIZE, - MLXSW_SP_RESOURCE_KVD_LINEAR_LARGE_CHUNKS, - MLXSW_SP_RESOURCE_KVD_LINEAR, - &size_params); + err = devl_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_LINEAR_LARGE_CHUNKS, + MLXSW_SP1_KVDL_LARGE_CHUNKS_SIZE, + MLXSW_SP_RESOURCE_KVD_LINEAR_LARGE_CHUNKS, + MLXSW_SP_RESOURCE_KVD_LINEAR, + &size_params); return err; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c index c68fc8f7ca99..c9f1c79f3f9d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c @@ -1290,12 +1290,12 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) if (err) goto err_sb_mms_init; mlxsw_sp_pool_count(mlxsw_sp, &ing_pool_count, &eg_pool_count); - err = devlink_sb_register(priv_to_devlink(mlxsw_sp->core), 0, - mlxsw_sp->sb->sb_size, - ing_pool_count, - eg_pool_count, - MLXSW_SP_SB_ING_TC_COUNT, - MLXSW_SP_SB_EG_TC_COUNT); + err = devl_sb_register(priv_to_devlink(mlxsw_sp->core), 0, + mlxsw_sp->sb->sb_size, + ing_pool_count, + eg_pool_count, + MLXSW_SP_SB_ING_TC_COUNT, + MLXSW_SP_SB_EG_TC_COUNT); if (err) goto err_devlink_sb_register; @@ -1314,7 +1314,7 @@ err_sb_ports_init: void mlxsw_sp_buffers_fini(struct mlxsw_sp *mlxsw_sp) { - devlink_sb_unregister(priv_to_devlink(mlxsw_sp->core), 0); + devl_sb_unregister(priv_to_devlink(mlxsw_sp->core), 0); mlxsw_sp_sb_ports_fini(mlxsw_sp); kfree(mlxsw_sp->sb); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c index fc2257753b9b..ee59c79156e4 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_cnt.c @@ -67,16 +67,16 @@ static int mlxsw_sp_counter_sub_pools_init(struct mlxsw_sp *mlxsw_sp) return -EIO; sub_pool->entry_size = mlxsw_core_res_get(mlxsw_sp->core, res_id); - err = devlink_resource_size_get(devlink, - sub_pool->resource_id, - &sub_pool->size); + err = devl_resource_size_get(devlink, + sub_pool->resource_id, + &sub_pool->size); if (err) goto err_resource_size_get; - devlink_resource_occ_get_register(devlink, - sub_pool->resource_id, - mlxsw_sp_counter_sub_pool_occ_get, - sub_pool); + devl_resource_occ_get_register(devlink, + sub_pool->resource_id, + mlxsw_sp_counter_sub_pool_occ_get, + sub_pool); sub_pool->base_index = base_index; base_index += sub_pool->size; @@ -88,8 +88,8 @@ err_resource_size_get: for (i--; i >= 0; i--) { sub_pool = &pool->sub_pools[i]; - devlink_resource_occ_get_unregister(devlink, - sub_pool->resource_id); + devl_resource_occ_get_unregister(devlink, + sub_pool->resource_id); } return err; } @@ -105,8 +105,8 @@ static void mlxsw_sp_counter_sub_pools_fini(struct mlxsw_sp *mlxsw_sp) sub_pool = &pool->sub_pools[i]; WARN_ON(atomic_read(&sub_pool->active_entries_count)); - devlink_resource_occ_get_unregister(devlink, - sub_pool->resource_id); + devl_resource_occ_get_unregister(devlink, + sub_pool->resource_id); } } @@ -135,12 +135,12 @@ int mlxsw_sp_counter_pool_init(struct mlxsw_sp *mlxsw_sp) spin_lock_init(&pool->counter_pool_lock); atomic_set(&pool->active_entries_count, 0); - err = devlink_resource_size_get(devlink, MLXSW_SP_RESOURCE_COUNTERS, - &pool->pool_size); + err = devl_resource_size_get(devlink, MLXSW_SP_RESOURCE_COUNTERS, + &pool->pool_size); if (err) goto err_pool_resource_size_get; - devlink_resource_occ_get_register(devlink, MLXSW_SP_RESOURCE_COUNTERS, - mlxsw_sp_counter_pool_occ_get, pool); + devl_resource_occ_get_register(devlink, MLXSW_SP_RESOURCE_COUNTERS, + mlxsw_sp_counter_pool_occ_get, pool); pool->usage = bitmap_zalloc(pool->pool_size, GFP_KERNEL); if (!pool->usage) { @@ -157,8 +157,8 @@ int mlxsw_sp_counter_pool_init(struct mlxsw_sp *mlxsw_sp) err_sub_pools_init: bitmap_free(pool->usage); err_usage_alloc: - devlink_resource_occ_get_unregister(devlink, - MLXSW_SP_RESOURCE_COUNTERS); + devl_resource_occ_get_unregister(devlink, + MLXSW_SP_RESOURCE_COUNTERS); err_pool_resource_size_get: kfree(pool); return err; @@ -174,8 +174,8 @@ void mlxsw_sp_counter_pool_fini(struct mlxsw_sp *mlxsw_sp) pool->pool_size); WARN_ON(atomic_read(&pool->active_entries_count)); bitmap_free(pool->usage); - devlink_resource_occ_get_unregister(devlink, - MLXSW_SP_RESOURCE_COUNTERS); + devl_resource_occ_get_unregister(devlink, + MLXSW_SP_RESOURCE_COUNTERS); kfree(pool); } @@ -262,12 +262,12 @@ int mlxsw_sp_counter_resources_register(struct mlxsw_core *mlxsw_core) devlink_resource_size_params_init(&size_params, pool_size, pool_size, bank_size, DEVLINK_RESOURCE_UNIT_ENTRY); - err = devlink_resource_register(devlink, - MLXSW_SP_RESOURCE_NAME_COUNTERS, - pool_size, - MLXSW_SP_RESOURCE_COUNTERS, - DEVLINK_RESOURCE_ID_PARENT_TOP, - &size_params); + err = devl_resource_register(devlink, + MLXSW_SP_RESOURCE_NAME_COUNTERS, + pool_size, + MLXSW_SP_RESOURCE_COUNTERS, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &size_params); if (err) return err; @@ -287,12 +287,12 @@ int mlxsw_sp_counter_resources_register(struct mlxsw_core *mlxsw_core) devlink_resource_size_params_init(&size_params, sub_pool_size, sub_pool_size, bank_size, DEVLINK_RESOURCE_UNIT_ENTRY); - err = devlink_resource_register(devlink, - sub_pool->resource_name, - sub_pool_size, - sub_pool->resource_id, - MLXSW_SP_RESOURCE_COUNTERS, - &size_params); + err = devl_resource_register(devlink, + sub_pool->resource_name, + sub_pool_size, + sub_pool->resource_id, + MLXSW_SP_RESOURCE_COUNTERS, + &size_params); if (err) return err; total_bank_config += sub_pool->bank_count; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c index c2540292702d..5416093c0e35 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c @@ -295,17 +295,17 @@ static int mlxsw_sp_dpipe_erif_table_init(struct mlxsw_sp *mlxsw_sp) { struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); - return devlink_dpipe_table_register(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_ERIF, - &mlxsw_sp_erif_ops, - mlxsw_sp, false); + return devl_dpipe_table_register(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_ERIF, + &mlxsw_sp_erif_ops, + mlxsw_sp, false); } static void mlxsw_sp_dpipe_erif_table_fini(struct mlxsw_sp *mlxsw_sp) { struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); - devlink_dpipe_table_unregister(devlink, MLXSW_SP_DPIPE_TABLE_NAME_ERIF); + devl_dpipe_table_unregister(devlink, MLXSW_SP_DPIPE_TABLE_NAME_ERIF); } static int mlxsw_sp_dpipe_table_host_matches_dump(struct sk_buff *skb, int type) @@ -749,25 +749,25 @@ static int mlxsw_sp_dpipe_host4_table_init(struct mlxsw_sp *mlxsw_sp) struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); int err; - err = devlink_dpipe_table_register(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_HOST4, - &mlxsw_sp_host4_ops, - mlxsw_sp, false); + err = devl_dpipe_table_register(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_HOST4, + &mlxsw_sp_host4_ops, + mlxsw_sp, false); if (err) return err; - err = devlink_dpipe_table_resource_set(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_HOST4, - MLXSW_SP_RESOURCE_KVD_HASH_SINGLE, - MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST4); + err = devl_dpipe_table_resource_set(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_HOST4, + MLXSW_SP_RESOURCE_KVD_HASH_SINGLE, + MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST4); if (err) goto err_resource_set; return 0; err_resource_set: - devlink_dpipe_table_unregister(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_HOST4); + devl_dpipe_table_unregister(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_HOST4); return err; } @@ -775,8 +775,8 @@ static void mlxsw_sp_dpipe_host4_table_fini(struct mlxsw_sp *mlxsw_sp) { struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); - devlink_dpipe_table_unregister(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_HOST4); + devl_dpipe_table_unregister(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_HOST4); } static int @@ -826,25 +826,25 @@ static int mlxsw_sp_dpipe_host6_table_init(struct mlxsw_sp *mlxsw_sp) struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); int err; - err = devlink_dpipe_table_register(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_HOST6, - &mlxsw_sp_host6_ops, - mlxsw_sp, false); + err = devl_dpipe_table_register(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_HOST6, + &mlxsw_sp_host6_ops, + mlxsw_sp, false); if (err) return err; - err = devlink_dpipe_table_resource_set(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_HOST6, - MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE, - MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST6); + err = devl_dpipe_table_resource_set(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_HOST6, + MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE, + MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST6); if (err) goto err_resource_set; return 0; err_resource_set: - devlink_dpipe_table_unregister(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_HOST6); + devl_dpipe_table_unregister(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_HOST6); return err; } @@ -852,8 +852,8 @@ static void mlxsw_sp_dpipe_host6_table_fini(struct mlxsw_sp *mlxsw_sp) { struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); - devlink_dpipe_table_unregister(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_HOST6); + devl_dpipe_table_unregister(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_HOST6); } static int mlxsw_sp_dpipe_table_adj_matches_dump(void *priv, @@ -1231,25 +1231,25 @@ static int mlxsw_sp_dpipe_adj_table_init(struct mlxsw_sp *mlxsw_sp) struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); int err; - err = devlink_dpipe_table_register(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_ADJ, - &mlxsw_sp_dpipe_table_adj_ops, - mlxsw_sp, false); + err = devl_dpipe_table_register(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_ADJ, + &mlxsw_sp_dpipe_table_adj_ops, + mlxsw_sp, false); if (err) return err; - err = devlink_dpipe_table_resource_set(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_ADJ, - MLXSW_SP_RESOURCE_KVD_LINEAR, - MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_ADJ); + err = devl_dpipe_table_resource_set(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_ADJ, + MLXSW_SP_RESOURCE_KVD_LINEAR, + MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_ADJ); if (err) goto err_resource_set; return 0; err_resource_set: - devlink_dpipe_table_unregister(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_ADJ); + devl_dpipe_table_unregister(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_ADJ); return err; } @@ -1257,8 +1257,8 @@ static void mlxsw_sp_dpipe_adj_table_fini(struct mlxsw_sp *mlxsw_sp) { struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); - devlink_dpipe_table_unregister(devlink, - MLXSW_SP_DPIPE_TABLE_NAME_ADJ); + devl_dpipe_table_unregister(devlink, + MLXSW_SP_DPIPE_TABLE_NAME_ADJ); } int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp) @@ -1266,7 +1266,7 @@ int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp) struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); int err; - devlink_dpipe_headers_register(devlink, &mlxsw_sp_dpipe_headers); + devl_dpipe_headers_register(devlink, &mlxsw_sp_dpipe_headers); err = mlxsw_sp_dpipe_erif_table_init(mlxsw_sp); if (err) @@ -1292,7 +1292,7 @@ err_host6_table_init: err_host4_table_init: mlxsw_sp_dpipe_erif_table_fini(mlxsw_sp); err_erif_table_init: - devlink_dpipe_headers_unregister(priv_to_devlink(mlxsw_sp->core)); + devl_dpipe_headers_unregister(priv_to_devlink(mlxsw_sp->core)); return err; } @@ -1304,5 +1304,5 @@ void mlxsw_sp_dpipe_fini(struct mlxsw_sp *mlxsw_sp) mlxsw_sp_dpipe_host6_table_fini(mlxsw_sp); mlxsw_sp_dpipe_host4_table_fini(mlxsw_sp); mlxsw_sp_dpipe_erif_table_fini(mlxsw_sp); - devlink_dpipe_headers_unregister(devlink); + devl_dpipe_headers_unregister(devlink); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_policer.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_policer.c index 39052e5c12fd..22ebb207ce4d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_policer.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_policer.c @@ -94,10 +94,10 @@ mlxsw_sp_policer_single_rate_family_init(struct mlxsw_sp_policer_family *family) atomic_set(&family->policers_count, 0); devlink = priv_to_devlink(core); - devlink_resource_occ_get_register(devlink, - MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS, - mlxsw_sp_policer_single_rate_occ_get, - family); + devl_resource_occ_get_register(devlink, + MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS, + mlxsw_sp_policer_single_rate_occ_get, + family); return 0; } @@ -107,8 +107,8 @@ mlxsw_sp_policer_single_rate_family_fini(struct mlxsw_sp_policer_family *family) { struct devlink *devlink = priv_to_devlink(family->mlxsw_sp->core); - devlink_resource_occ_get_unregister(devlink, - MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS); + devl_resource_occ_get_unregister(devlink, + MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS); WARN_ON(atomic_read(&family->policers_count) != 0); } @@ -419,22 +419,22 @@ int mlxsw_sp_policer_resources_register(struct mlxsw_core *mlxsw_core) devlink_resource_size_params_init(&size_params, global_policers, global_policers, 1, DEVLINK_RESOURCE_UNIT_ENTRY); - err = devlink_resource_register(devlink, "global_policers", - global_policers, - MLXSW_SP_RESOURCE_GLOBAL_POLICERS, - DEVLINK_RESOURCE_ID_PARENT_TOP, - &size_params); + err = devl_resource_register(devlink, "global_policers", + global_policers, + MLXSW_SP_RESOURCE_GLOBAL_POLICERS, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &size_params); if (err) return err; devlink_resource_size_params_init(&size_params, single_rate_policers, single_rate_policers, 1, DEVLINK_RESOURCE_UNIT_ENTRY); - err = devlink_resource_register(devlink, "single_rate_policers", - single_rate_policers, - MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS, - MLXSW_SP_RESOURCE_GLOBAL_POLICERS, - &size_params); + err = devl_resource_register(devlink, "single_rate_policers", + single_rate_policers, + MLXSW_SP_RESOURCE_SINGLE_RATE_POLICERS, + MLXSW_SP_RESOURCE_GLOBAL_POLICERS, + &size_params); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c index 35422e64d89f..5116d7ebe258 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c @@ -29,6 +29,10 @@ struct mlxsw_sp_ptp_state { struct mlxsw_sp *mlxsw_sp; +}; + +struct mlxsw_sp1_ptp_state { + struct mlxsw_sp_ptp_state common; struct rhltable unmatched_ht; spinlock_t unmatched_lock; /* protects the HT */ struct delayed_work ht_gc_dw; @@ -60,20 +64,37 @@ static const struct rhashtable_params mlxsw_sp1_ptp_unmatched_ht_params = { struct mlxsw_sp_ptp_clock { struct mlxsw_core *core; + struct ptp_clock *ptp; + struct ptp_clock_info ptp_info; +}; + +struct mlxsw_sp1_ptp_clock { + struct mlxsw_sp_ptp_clock common; spinlock_t lock; /* protect this structure */ struct cyclecounter cycles; struct timecounter tc; u32 nominal_c_mult; - struct ptp_clock *ptp; - struct ptp_clock_info ptp_info; unsigned long overflow_period; struct delayed_work overflow_work; }; -static u64 __mlxsw_sp1_ptp_read_frc(struct mlxsw_sp_ptp_clock *clock, +static struct mlxsw_sp1_ptp_state * +mlxsw_sp1_ptp_state(struct mlxsw_sp *mlxsw_sp) +{ + return container_of(mlxsw_sp->ptp_state, struct mlxsw_sp1_ptp_state, + common); +} + +static struct mlxsw_sp1_ptp_clock * +mlxsw_sp1_ptp_clock(struct ptp_clock_info *ptp) +{ + return container_of(ptp, struct mlxsw_sp1_ptp_clock, common.ptp_info); +} + +static u64 __mlxsw_sp1_ptp_read_frc(struct mlxsw_sp1_ptp_clock *clock, struct ptp_system_timestamp *sts) { - struct mlxsw_core *mlxsw_core = clock->core; + struct mlxsw_core *mlxsw_core = clock->common.core; u32 frc_h1, frc_h2, frc_l; frc_h1 = mlxsw_core_read_frc_h(mlxsw_core); @@ -94,20 +115,20 @@ static u64 __mlxsw_sp1_ptp_read_frc(struct mlxsw_sp_ptp_clock *clock, static u64 mlxsw_sp1_ptp_read_frc(const struct cyclecounter *cc) { - struct mlxsw_sp_ptp_clock *clock = - container_of(cc, struct mlxsw_sp_ptp_clock, cycles); + struct mlxsw_sp1_ptp_clock *clock = + container_of(cc, struct mlxsw_sp1_ptp_clock, cycles); return __mlxsw_sp1_ptp_read_frc(clock, NULL) & cc->mask; } static int -mlxsw_sp1_ptp_phc_adjfreq(struct mlxsw_sp_ptp_clock *clock, int freq_adj) +mlxsw_sp_ptp_phc_adjfreq(struct mlxsw_sp_ptp_clock *clock, int freq_adj) { struct mlxsw_core *mlxsw_core = clock->core; char mtutc_pl[MLXSW_REG_MTUTC_LEN]; mlxsw_reg_mtutc_pack(mtutc_pl, MLXSW_REG_MTUTC_OPERATION_ADJUST_FREQ, - freq_adj, 0); + freq_adj, 0, 0, 0); return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mtutc), mtutc_pl); } @@ -122,9 +143,9 @@ static u64 mlxsw_sp1_ptp_ns2cycles(const struct timecounter *tc, u64 nsec) } static int -mlxsw_sp1_ptp_phc_settime(struct mlxsw_sp_ptp_clock *clock, u64 nsec) +mlxsw_sp1_ptp_phc_settime(struct mlxsw_sp1_ptp_clock *clock, u64 nsec) { - struct mlxsw_core *mlxsw_core = clock->core; + struct mlxsw_core *mlxsw_core = clock->common.core; u64 next_sec, next_sec_in_nsec, cycles; char mtutc_pl[MLXSW_REG_MTUTC_LEN]; char mtpps_pl[MLXSW_REG_MTPPS_LEN]; @@ -144,14 +165,13 @@ mlxsw_sp1_ptp_phc_settime(struct mlxsw_sp_ptp_clock *clock, u64 nsec) mlxsw_reg_mtutc_pack(mtutc_pl, MLXSW_REG_MTUTC_OPERATION_SET_TIME_AT_NEXT_SEC, - 0, next_sec); + 0, next_sec, 0, 0); return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mtutc), mtutc_pl); } static int mlxsw_sp1_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) { - struct mlxsw_sp_ptp_clock *clock = - container_of(ptp, struct mlxsw_sp_ptp_clock, ptp_info); + struct mlxsw_sp1_ptp_clock *clock = mlxsw_sp1_ptp_clock(ptp); int neg_adj = 0; u32 diff; u64 adj; @@ -174,13 +194,12 @@ static int mlxsw_sp1_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) clock->nominal_c_mult + diff; spin_unlock_bh(&clock->lock); - return mlxsw_sp1_ptp_phc_adjfreq(clock, neg_adj ? -ppb : ppb); + return mlxsw_sp_ptp_phc_adjfreq(&clock->common, neg_adj ? -ppb : ppb); } static int mlxsw_sp1_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) { - struct mlxsw_sp_ptp_clock *clock = - container_of(ptp, struct mlxsw_sp_ptp_clock, ptp_info); + struct mlxsw_sp1_ptp_clock *clock = mlxsw_sp1_ptp_clock(ptp); u64 nsec; spin_lock_bh(&clock->lock); @@ -195,8 +214,7 @@ static int mlxsw_sp1_ptp_gettimex(struct ptp_clock_info *ptp, struct timespec64 *ts, struct ptp_system_timestamp *sts) { - struct mlxsw_sp_ptp_clock *clock = - container_of(ptp, struct mlxsw_sp_ptp_clock, ptp_info); + struct mlxsw_sp1_ptp_clock *clock = mlxsw_sp1_ptp_clock(ptp); u64 cycles, nsec; spin_lock_bh(&clock->lock); @@ -212,8 +230,7 @@ static int mlxsw_sp1_ptp_gettimex(struct ptp_clock_info *ptp, static int mlxsw_sp1_ptp_settime(struct ptp_clock_info *ptp, const struct timespec64 *ts) { - struct mlxsw_sp_ptp_clock *clock = - container_of(ptp, struct mlxsw_sp_ptp_clock, ptp_info); + struct mlxsw_sp1_ptp_clock *clock = mlxsw_sp1_ptp_clock(ptp); u64 nsec = timespec64_to_ns(ts); spin_lock_bh(&clock->lock); @@ -237,9 +254,9 @@ static const struct ptp_clock_info mlxsw_sp1_ptp_clock_info = { static void mlxsw_sp1_ptp_clock_overflow(struct work_struct *work) { struct delayed_work *dwork = to_delayed_work(work); - struct mlxsw_sp_ptp_clock *clock; + struct mlxsw_sp1_ptp_clock *clock; - clock = container_of(dwork, struct mlxsw_sp_ptp_clock, overflow_work); + clock = container_of(dwork, struct mlxsw_sp1_ptp_clock, overflow_work); spin_lock_bh(&clock->lock); timecounter_read(&clock->tc); @@ -251,7 +268,7 @@ struct mlxsw_sp_ptp_clock * mlxsw_sp1_ptp_clock_init(struct mlxsw_sp *mlxsw_sp, struct device *dev) { u64 overflow_cycles, nsec, frac = 0; - struct mlxsw_sp_ptp_clock *clock; + struct mlxsw_sp1_ptp_clock *clock; int err; clock = kzalloc(sizeof(*clock), GFP_KERNEL); @@ -265,10 +282,9 @@ mlxsw_sp1_ptp_clock_init(struct mlxsw_sp *mlxsw_sp, struct device *dev) clock->cycles.shift); clock->nominal_c_mult = clock->cycles.mult; clock->cycles.mask = CLOCKSOURCE_MASK(MLXSW_SP1_PTP_CLOCK_MASK); - clock->core = mlxsw_sp->core; + clock->common.core = mlxsw_sp->core; - timecounter_init(&clock->tc, &clock->cycles, - ktime_to_ns(ktime_get_real())); + timecounter_init(&clock->tc, &clock->cycles, 0); /* Calculate period in seconds to call the overflow watchdog - to make * sure counter is checked at least twice every wrap around. @@ -286,15 +302,15 @@ mlxsw_sp1_ptp_clock_init(struct mlxsw_sp *mlxsw_sp, struct device *dev) INIT_DELAYED_WORK(&clock->overflow_work, mlxsw_sp1_ptp_clock_overflow); mlxsw_core_schedule_dw(&clock->overflow_work, 0); - clock->ptp_info = mlxsw_sp1_ptp_clock_info; - clock->ptp = ptp_clock_register(&clock->ptp_info, dev); - if (IS_ERR(clock->ptp)) { - err = PTR_ERR(clock->ptp); + clock->common.ptp_info = mlxsw_sp1_ptp_clock_info; + clock->common.ptp = ptp_clock_register(&clock->common.ptp_info, dev); + if (IS_ERR(clock->common.ptp)) { + err = PTR_ERR(clock->common.ptp); dev_err(dev, "ptp_clock_register failed %d\n", err); goto err_ptp_clock_register; } - return clock; + return &clock->common; err_ptp_clock_register: cancel_delayed_work_sync(&clock->overflow_work); @@ -302,9 +318,12 @@ err_ptp_clock_register: return ERR_PTR(err); } -void mlxsw_sp1_ptp_clock_fini(struct mlxsw_sp_ptp_clock *clock) +void mlxsw_sp1_ptp_clock_fini(struct mlxsw_sp_ptp_clock *clock_common) { - ptp_clock_unregister(clock->ptp); + struct mlxsw_sp1_ptp_clock *clock = + container_of(clock_common, struct mlxsw_sp1_ptp_clock, common); + + ptp_clock_unregister(clock_common->ptp); cancel_delayed_work_sync(&clock->overflow_work); kfree(clock); } @@ -348,7 +367,7 @@ mlxsw_sp1_ptp_unmatched_save(struct mlxsw_sp *mlxsw_sp, u64 timestamp) { int cycles = MLXSW_SP1_PTP_HT_GC_TIMEOUT / MLXSW_SP1_PTP_HT_GC_INTERVAL; - struct mlxsw_sp_ptp_state *ptp_state = mlxsw_sp->ptp_state; + struct mlxsw_sp1_ptp_state *ptp_state = mlxsw_sp1_ptp_state(mlxsw_sp); struct mlxsw_sp1_ptp_unmatched *unmatched; int err; @@ -359,7 +378,7 @@ mlxsw_sp1_ptp_unmatched_save(struct mlxsw_sp *mlxsw_sp, unmatched->key = key; unmatched->skb = skb; unmatched->timestamp = timestamp; - unmatched->gc_cycle = mlxsw_sp->ptp_state->gc_cycle + cycles; + unmatched->gc_cycle = ptp_state->gc_cycle + cycles; err = rhltable_insert(&ptp_state->unmatched_ht, &unmatched->ht_node, mlxsw_sp1_ptp_unmatched_ht_params); @@ -373,11 +392,12 @@ static struct mlxsw_sp1_ptp_unmatched * mlxsw_sp1_ptp_unmatched_lookup(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp1_ptp_key key, int *p_length) { + struct mlxsw_sp1_ptp_state *ptp_state = mlxsw_sp1_ptp_state(mlxsw_sp); struct mlxsw_sp1_ptp_unmatched *unmatched, *last = NULL; struct rhlist_head *tmp, *list; int length = 0; - list = rhltable_lookup(&mlxsw_sp->ptp_state->unmatched_ht, &key, + list = rhltable_lookup(&ptp_state->unmatched_ht, &key, mlxsw_sp1_ptp_unmatched_ht_params); rhl_for_each_entry_rcu(unmatched, tmp, list, ht_node) { last = unmatched; @@ -392,7 +412,9 @@ static int mlxsw_sp1_ptp_unmatched_remove(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp1_ptp_unmatched *unmatched) { - return rhltable_remove(&mlxsw_sp->ptp_state->unmatched_ht, + struct mlxsw_sp1_ptp_state *ptp_state = mlxsw_sp1_ptp_state(mlxsw_sp); + + return rhltable_remove(&ptp_state->unmatched_ht, &unmatched->ht_node, mlxsw_sp1_ptp_unmatched_ht_params); } @@ -438,12 +460,16 @@ static void mlxsw_sp1_packet_timestamp(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, u64 timestamp) { + struct mlxsw_sp_ptp_clock *clock_common = mlxsw_sp->clock; + struct mlxsw_sp1_ptp_clock *clock = + container_of(clock_common, struct mlxsw_sp1_ptp_clock, common); + struct skb_shared_hwtstamps hwtstamps; u64 nsec; - spin_lock_bh(&mlxsw_sp->clock->lock); - nsec = timecounter_cyc2time(&mlxsw_sp->clock->tc, timestamp); - spin_unlock_bh(&mlxsw_sp->clock->lock); + spin_lock_bh(&clock->lock); + nsec = timecounter_cyc2time(&clock->tc, timestamp); + spin_unlock_bh(&clock->lock); hwtstamps.hwtstamp = ns_to_ktime(nsec); mlxsw_sp1_ptp_packet_finish(mlxsw_sp, skb, @@ -481,13 +507,14 @@ static void mlxsw_sp1_ptp_got_piece(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp1_ptp_key key, struct sk_buff *skb, u64 timestamp) { + struct mlxsw_sp1_ptp_state *ptp_state = mlxsw_sp1_ptp_state(mlxsw_sp); struct mlxsw_sp1_ptp_unmatched *unmatched; int length; int err; rcu_read_lock(); - spin_lock(&mlxsw_sp->ptp_state->unmatched_lock); + spin_lock(&ptp_state->unmatched_lock); unmatched = mlxsw_sp1_ptp_unmatched_lookup(mlxsw_sp, key, &length); if (skb && unmatched && unmatched->timestamp) { @@ -515,7 +542,7 @@ static void mlxsw_sp1_ptp_got_piece(struct mlxsw_sp *mlxsw_sp, WARN_ON_ONCE(err); } - spin_unlock(&mlxsw_sp->ptp_state->unmatched_lock); + spin_unlock(&ptp_state->unmatched_lock); if (unmatched) mlxsw_sp1_ptp_unmatched_finish(mlxsw_sp, unmatched); @@ -611,9 +638,10 @@ void mlxsw_sp1_ptp_transmitted(struct mlxsw_sp *mlxsw_sp, } static void -mlxsw_sp1_ptp_ht_gc_collect(struct mlxsw_sp_ptp_state *ptp_state, +mlxsw_sp1_ptp_ht_gc_collect(struct mlxsw_sp1_ptp_state *ptp_state, struct mlxsw_sp1_ptp_unmatched *unmatched) { + struct mlxsw_sp *mlxsw_sp = ptp_state->common.mlxsw_sp; struct mlxsw_sp_ptp_port_dir_stats *stats; struct mlxsw_sp_port *mlxsw_sp_port; int err; @@ -636,7 +664,7 @@ mlxsw_sp1_ptp_ht_gc_collect(struct mlxsw_sp_ptp_state *ptp_state, /* The packet was matched with timestamp during the walk. */ goto out; - mlxsw_sp_port = ptp_state->mlxsw_sp->ports[unmatched->key.local_port]; + mlxsw_sp_port = mlxsw_sp->ports[unmatched->key.local_port]; if (mlxsw_sp_port) { stats = unmatched->key.ingress ? &mlxsw_sp_port->ptp.stats.rx_gcd : @@ -653,7 +681,7 @@ mlxsw_sp1_ptp_ht_gc_collect(struct mlxsw_sp_ptp_state *ptp_state, * netif_receive_skb(), in process context, is seen elsewhere in the * kernel, notably in pktgen. */ - mlxsw_sp1_ptp_unmatched_finish(ptp_state->mlxsw_sp, unmatched); + mlxsw_sp1_ptp_unmatched_finish(mlxsw_sp, unmatched); out: local_bh_enable(); @@ -663,12 +691,12 @@ static void mlxsw_sp1_ptp_ht_gc(struct work_struct *work) { struct delayed_work *dwork = to_delayed_work(work); struct mlxsw_sp1_ptp_unmatched *unmatched; - struct mlxsw_sp_ptp_state *ptp_state; + struct mlxsw_sp1_ptp_state *ptp_state; struct rhashtable_iter iter; u32 gc_cycle; void *obj; - ptp_state = container_of(dwork, struct mlxsw_sp_ptp_state, ht_gc_dw); + ptp_state = container_of(dwork, struct mlxsw_sp1_ptp_state, ht_gc_dw); gc_cycle = ptp_state->gc_cycle++; rhltable_walk_enter(&ptp_state->unmatched_ht, &iter); @@ -694,7 +722,7 @@ static int mlxsw_sp_ptp_mtptpt_set(struct mlxsw_sp *mlxsw_sp, { char mtptpt_pl[MLXSW_REG_MTPTPT_LEN]; - mlxsw_reg_mtptptp_pack(mtptpt_pl, trap_id, message_type); + mlxsw_reg_mtptpt_pack(mtptpt_pl, trap_id, message_type); return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mtptpt), mtptpt_pl); } @@ -809,7 +837,7 @@ static int mlxsw_sp1_ptp_shaper_params_set(struct mlxsw_sp *mlxsw_sp) struct mlxsw_sp_ptp_state *mlxsw_sp1_ptp_init(struct mlxsw_sp *mlxsw_sp) { - struct mlxsw_sp_ptp_state *ptp_state; + struct mlxsw_sp1_ptp_state *ptp_state; u16 message_type; int err; @@ -820,7 +848,7 @@ struct mlxsw_sp_ptp_state *mlxsw_sp1_ptp_init(struct mlxsw_sp *mlxsw_sp) ptp_state = kzalloc(sizeof(*ptp_state), GFP_KERNEL); if (!ptp_state) return ERR_PTR(-ENOMEM); - ptp_state->mlxsw_sp = mlxsw_sp; + ptp_state->common.mlxsw_sp = mlxsw_sp; spin_lock_init(&ptp_state->unmatched_lock); @@ -853,7 +881,7 @@ struct mlxsw_sp_ptp_state *mlxsw_sp1_ptp_init(struct mlxsw_sp *mlxsw_sp) INIT_DELAYED_WORK(&ptp_state->ht_gc_dw, mlxsw_sp1_ptp_ht_gc); mlxsw_core_schedule_dw(&ptp_state->ht_gc_dw, MLXSW_SP1_PTP_HT_GC_INTERVAL); - return ptp_state; + return &ptp_state->common; err_fifo_clr: mlxsw_sp_ptp_mtptpt_set(mlxsw_sp, MLXSW_REG_MTPTPT_TRAP_ID_PTP1, 0); @@ -866,9 +894,12 @@ err_hashtable_init: return ERR_PTR(err); } -void mlxsw_sp1_ptp_fini(struct mlxsw_sp_ptp_state *ptp_state) +void mlxsw_sp1_ptp_fini(struct mlxsw_sp_ptp_state *ptp_state_common) { - struct mlxsw_sp *mlxsw_sp = ptp_state->mlxsw_sp; + struct mlxsw_sp *mlxsw_sp = ptp_state_common->mlxsw_sp; + struct mlxsw_sp1_ptp_state *ptp_state; + + ptp_state = mlxsw_sp1_ptp_state(mlxsw_sp); cancel_delayed_work_sync(&ptp_state->ht_gc_dw); mlxsw_sp1_ptp_mtpppc_set(mlxsw_sp, 0, 0); @@ -887,9 +918,10 @@ int mlxsw_sp1_ptp_hwtstamp_get(struct mlxsw_sp_port *mlxsw_sp_port, return 0; } -static int mlxsw_sp_ptp_get_message_types(const struct hwtstamp_config *config, - u16 *p_ing_types, u16 *p_egr_types, - enum hwtstamp_rx_filters *p_rx_filter) +static int +mlxsw_sp1_ptp_get_message_types(const struct hwtstamp_config *config, + u16 *p_ing_types, u16 *p_egr_types, + enum hwtstamp_rx_filters *p_rx_filter) { enum hwtstamp_rx_filters rx_filter = config->rx_filter; enum hwtstamp_tx_types tx_type = config->tx_type; @@ -1050,8 +1082,8 @@ int mlxsw_sp1_ptp_hwtstamp_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 egr_types; int err; - err = mlxsw_sp_ptp_get_message_types(config, &ing_types, &egr_types, - &rx_filter); + err = mlxsw_sp1_ptp_get_message_types(config, &ing_types, &egr_types, + &rx_filter); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 09009e80cd71..2c4443c6b964 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -5292,7 +5292,7 @@ static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, { const struct fib_nh *nh = fib_info_nh(fi, 0); - return nh->fib_nh_scope == RT_SCOPE_LINK || + return nh->fib_nh_gw_family || mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); } @@ -8590,9 +8590,7 @@ static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, unsigned long event, struct netlink_ext_ack *extack) { - if (netif_is_bridge_port(port_dev) || - netif_is_lag_port(port_dev) || - netif_is_ovs_port(port_dev)) + if (netif_is_any_bridge_port(port_dev) || netif_is_lag_port(port_dev)) return 0; return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, @@ -9962,14 +9960,14 @@ static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) idr_init(&mlxsw_sp->router->rif_mac_profiles_idr); atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0); atomic_set(&mlxsw_sp->router->rifs_count, 0); - devlink_resource_occ_get_register(devlink, - MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, - mlxsw_sp_rif_mac_profiles_occ_get, - mlxsw_sp); - devlink_resource_occ_get_register(devlink, - MLXSW_SP_RESOURCE_RIFS, - mlxsw_sp_rifs_occ_get, - mlxsw_sp); + devl_resource_occ_get_register(devlink, + MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, + mlxsw_sp_rif_mac_profiles_occ_get, + mlxsw_sp); + devl_resource_occ_get_register(devlink, + MLXSW_SP_RESOURCE_RIFS, + mlxsw_sp_rifs_occ_get, + mlxsw_sp); return 0; } @@ -9983,9 +9981,9 @@ static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); - devlink_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_RIFS); - devlink_resource_occ_get_unregister(devlink, - MLXSW_SP_RESOURCE_RIF_MAC_PROFILES); + devl_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_RIFS); + devl_resource_occ_get_unregister(devlink, + MLXSW_SP_RESOURCE_RIF_MAC_PROFILES); WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr)); idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr); kfree(mlxsw_sp->router->rifs); @@ -10144,7 +10142,7 @@ static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, unsigned long *fields = config->fields; u32 hash_fields; - switch (net->ipv4.sysctl_fib_multipath_hash_policy) { + switch (READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_policy)) { case 0: mlxsw_sp_mp4_hash_outer_addr(config); break; @@ -10162,7 +10160,7 @@ static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_mp_hash_inner_l3(config); break; case 3: - hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields; + hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields); /* Outer */ MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); @@ -10343,13 +10341,14 @@ static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) { struct net *net = mlxsw_sp_net(mlxsw_sp); - bool usp = net->ipv4.sysctl_ip_fwd_update_priority; char rgcr_pl[MLXSW_REG_RGCR_LEN]; u64 max_rifs; + bool usp; if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) return -EIO; max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); + usp = READ_ONCE(net->ipv4.sysctl_ip_fwd_update_priority); mlxsw_reg_rgcr_pack(rgcr_pl, true, true); mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c index fe663b0ab708..39904dacf4f0 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c @@ -106,8 +106,8 @@ int mlxsw_sp_span_init(struct mlxsw_sp *mlxsw_sp) if (err) goto err_init; - devlink_resource_occ_get_register(devlink, MLXSW_SP_RESOURCE_SPAN, - mlxsw_sp_span_occ_get, mlxsw_sp); + devl_resource_occ_get_register(devlink, MLXSW_SP_RESOURCE_SPAN, + mlxsw_sp_span_occ_get, mlxsw_sp); INIT_WORK(&span->work, mlxsw_sp_span_respin_work); return 0; @@ -123,7 +123,7 @@ void mlxsw_sp_span_fini(struct mlxsw_sp *mlxsw_sp) struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); cancel_work_sync(&mlxsw_sp->span->work); - devlink_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_SPAN); + devl_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_SPAN); WARN_ON_ONCE(!list_empty(&mlxsw_sp->span->trigger_entries_list)); WARN_ON_ONCE(!list_empty(&mlxsw_sp->span->analyzed_ports_list)); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c index d0baba38d2a3..f4bfdb6dab9c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c @@ -1298,8 +1298,8 @@ static int mlxsw_sp_trap_policers_init(struct mlxsw_sp *mlxsw_sp) for (i = 0; i < trap->policers_count; i++) { policer_item = &trap->policer_items_arr[i]; - err = devlink_trap_policers_register(devlink, - &policer_item->policer, 1); + err = devl_trap_policers_register(devlink, + &policer_item->policer, 1); if (err) goto err_trap_policer_register; } @@ -1309,8 +1309,8 @@ static int mlxsw_sp_trap_policers_init(struct mlxsw_sp *mlxsw_sp) err_trap_policer_register: for (i--; i >= 0; i--) { policer_item = &trap->policer_items_arr[i]; - devlink_trap_policers_unregister(devlink, - &policer_item->policer, 1); + devl_trap_policers_unregister(devlink, + &policer_item->policer, 1); } mlxsw_sp_trap_policer_items_arr_fini(mlxsw_sp); return err; @@ -1325,8 +1325,8 @@ static void mlxsw_sp_trap_policers_fini(struct mlxsw_sp *mlxsw_sp) for (i = trap->policers_count - 1; i >= 0; i--) { policer_item = &trap->policer_items_arr[i]; - devlink_trap_policers_unregister(devlink, - &policer_item->policer, 1); + devl_trap_policers_unregister(devlink, + &policer_item->policer, 1); } mlxsw_sp_trap_policer_items_arr_fini(mlxsw_sp); } @@ -1381,8 +1381,7 @@ static int mlxsw_sp_trap_groups_init(struct mlxsw_sp *mlxsw_sp) for (i = 0; i < trap->groups_count; i++) { group_item = &trap->group_items_arr[i]; - err = devlink_trap_groups_register(devlink, &group_item->group, - 1); + err = devl_trap_groups_register(devlink, &group_item->group, 1); if (err) goto err_trap_group_register; } @@ -1392,7 +1391,7 @@ static int mlxsw_sp_trap_groups_init(struct mlxsw_sp *mlxsw_sp) err_trap_group_register: for (i--; i >= 0; i--) { group_item = &trap->group_items_arr[i]; - devlink_trap_groups_unregister(devlink, &group_item->group, 1); + devl_trap_groups_unregister(devlink, &group_item->group, 1); } mlxsw_sp_trap_group_items_arr_fini(mlxsw_sp); return err; @@ -1408,7 +1407,7 @@ static void mlxsw_sp_trap_groups_fini(struct mlxsw_sp *mlxsw_sp) const struct mlxsw_sp_trap_group_item *group_item; group_item = &trap->group_items_arr[i]; - devlink_trap_groups_unregister(devlink, &group_item->group, 1); + devl_trap_groups_unregister(devlink, &group_item->group, 1); } mlxsw_sp_trap_group_items_arr_fini(mlxsw_sp); } @@ -1469,8 +1468,8 @@ static int mlxsw_sp_traps_init(struct mlxsw_sp *mlxsw_sp) for (i = 0; i < trap->traps_count; i++) { trap_item = &trap->trap_items_arr[i]; - err = devlink_traps_register(devlink, &trap_item->trap, 1, - mlxsw_sp); + err = devl_traps_register(devlink, &trap_item->trap, 1, + mlxsw_sp); if (err) goto err_trap_register; } @@ -1480,7 +1479,7 @@ static int mlxsw_sp_traps_init(struct mlxsw_sp *mlxsw_sp) err_trap_register: for (i--; i >= 0; i--) { trap_item = &trap->trap_items_arr[i]; - devlink_traps_unregister(devlink, &trap_item->trap, 1); + devl_traps_unregister(devlink, &trap_item->trap, 1); } mlxsw_sp_trap_items_arr_fini(mlxsw_sp); return err; @@ -1496,7 +1495,7 @@ static void mlxsw_sp_traps_fini(struct mlxsw_sp *mlxsw_sp) const struct mlxsw_sp_trap_item *trap_item; trap_item = &trap->trap_items_arr[i]; - devlink_traps_unregister(devlink, &trap_item->trap, 1); + devl_traps_unregister(devlink, &trap_item->trap, 1); } mlxsw_sp_trap_items_arr_fini(mlxsw_sp); } diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c b/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c index 005e56ea5da1..5893770bfd94 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c @@ -75,6 +75,9 @@ static int __lan966x_mac_learn(struct lan966x *lan966x, int pgid, unsigned int vid, enum macaccess_entry_type type) { + int ret; + + spin_lock(&lan966x->mac_lock); lan966x_mac_select(lan966x, mac, vid); /* Issue a write command */ @@ -86,7 +89,10 @@ static int __lan966x_mac_learn(struct lan966x *lan966x, int pgid, ANA_MACACCESS_MAC_TABLE_CMD_SET(MACACCESS_CMD_LEARN), lan966x, ANA_MACACCESS); - return lan966x_mac_wait_for_completion(lan966x); + ret = lan966x_mac_wait_for_completion(lan966x); + spin_unlock(&lan966x->mac_lock); + + return ret; } /* The mask of the front ports is encoded inside the mac parameter via a call @@ -113,11 +119,13 @@ int lan966x_mac_learn(struct lan966x *lan966x, int port, return __lan966x_mac_learn(lan966x, port, false, mac, vid, type); } -int lan966x_mac_forget(struct lan966x *lan966x, - const unsigned char mac[ETH_ALEN], - unsigned int vid, - enum macaccess_entry_type type) +static int lan966x_mac_forget_locked(struct lan966x *lan966x, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type) { + lockdep_assert_held(&lan966x->mac_lock); + lan966x_mac_select(lan966x, mac, vid); /* Issue a forget command */ @@ -128,6 +136,20 @@ int lan966x_mac_forget(struct lan966x *lan966x, return lan966x_mac_wait_for_completion(lan966x); } +int lan966x_mac_forget(struct lan966x *lan966x, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type) +{ + int ret; + + spin_lock(&lan966x->mac_lock); + ret = lan966x_mac_forget_locked(lan966x, mac, vid, type); + spin_unlock(&lan966x->mac_lock); + + return ret; +} + int lan966x_mac_cpu_learn(struct lan966x *lan966x, const char *addr, u16 vid) { return lan966x_mac_learn(lan966x, PGID_CPU, addr, vid, ENTRYTYPE_LOCKED); @@ -161,7 +183,7 @@ static struct lan966x_mac_entry *lan966x_mac_alloc_entry(const unsigned char *ma { struct lan966x_mac_entry *mac_entry; - mac_entry = kzalloc(sizeof(*mac_entry), GFP_KERNEL); + mac_entry = kzalloc(sizeof(*mac_entry), GFP_ATOMIC); if (!mac_entry) return NULL; @@ -179,7 +201,6 @@ static struct lan966x_mac_entry *lan966x_mac_find_entry(struct lan966x *lan966x, struct lan966x_mac_entry *res = NULL; struct lan966x_mac_entry *mac_entry; - spin_lock(&lan966x->mac_lock); list_for_each_entry(mac_entry, &lan966x->mac_entries, list) { if (mac_entry->vid == vid && ether_addr_equal(mac, mac_entry->mac) && @@ -188,7 +209,6 @@ static struct lan966x_mac_entry *lan966x_mac_find_entry(struct lan966x *lan966x, break; } } - spin_unlock(&lan966x->mac_lock); return res; } @@ -231,8 +251,11 @@ int lan966x_mac_add_entry(struct lan966x *lan966x, struct lan966x_port *port, { struct lan966x_mac_entry *mac_entry; - if (lan966x_mac_lookup(lan966x, addr, vid, ENTRYTYPE_NORMAL)) + spin_lock(&lan966x->mac_lock); + if (lan966x_mac_lookup(lan966x, addr, vid, ENTRYTYPE_NORMAL)) { + spin_unlock(&lan966x->mac_lock); return 0; + } /* In case the entry already exists, don't add it again to SW, * just update HW, but we need to look in the actual HW because @@ -241,21 +264,25 @@ int lan966x_mac_add_entry(struct lan966x *lan966x, struct lan966x_port *port, * add the entry but without the extern_learn flag. */ mac_entry = lan966x_mac_find_entry(lan966x, addr, vid, port->chip_port); - if (mac_entry) - return lan966x_mac_learn(lan966x, port->chip_port, - addr, vid, ENTRYTYPE_LOCKED); + if (mac_entry) { + spin_unlock(&lan966x->mac_lock); + goto mac_learn; + } mac_entry = lan966x_mac_alloc_entry(addr, vid, port->chip_port); - if (!mac_entry) + if (!mac_entry) { + spin_unlock(&lan966x->mac_lock); return -ENOMEM; + } - spin_lock(&lan966x->mac_lock); list_add_tail(&mac_entry->list, &lan966x->mac_entries); spin_unlock(&lan966x->mac_lock); - lan966x_mac_learn(lan966x, port->chip_port, addr, vid, ENTRYTYPE_LOCKED); lan966x_fdb_call_notifiers(SWITCHDEV_FDB_OFFLOADED, addr, vid, port->dev); +mac_learn: + lan966x_mac_learn(lan966x, port->chip_port, addr, vid, ENTRYTYPE_LOCKED); + return 0; } @@ -269,8 +296,9 @@ int lan966x_mac_del_entry(struct lan966x *lan966x, const unsigned char *addr, list) { if (mac_entry->vid == vid && ether_addr_equal(addr, mac_entry->mac)) { - lan966x_mac_forget(lan966x, mac_entry->mac, mac_entry->vid, - ENTRYTYPE_LOCKED); + lan966x_mac_forget_locked(lan966x, mac_entry->mac, + mac_entry->vid, + ENTRYTYPE_LOCKED); list_del(&mac_entry->list); kfree(mac_entry); @@ -288,8 +316,8 @@ void lan966x_mac_purge_entries(struct lan966x *lan966x) spin_lock(&lan966x->mac_lock); list_for_each_entry_safe(mac_entry, tmp, &lan966x->mac_entries, list) { - lan966x_mac_forget(lan966x, mac_entry->mac, mac_entry->vid, - ENTRYTYPE_LOCKED); + lan966x_mac_forget_locked(lan966x, mac_entry->mac, + mac_entry->vid, ENTRYTYPE_LOCKED); list_del(&mac_entry->list); kfree(mac_entry); @@ -325,10 +353,13 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row, { struct lan966x_mac_entry *mac_entry, *tmp; unsigned char mac[ETH_ALEN] __aligned(2); + struct list_head mac_deleted_entries; u32 dest_idx; u32 column; u16 vid; + INIT_LIST_HEAD(&mac_deleted_entries); + spin_lock(&lan966x->mac_lock); list_for_each_entry_safe(mac_entry, tmp, &lan966x->mac_entries, list) { bool found = false; @@ -362,20 +393,26 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row, } if (!found) { - /* Notify the bridge that the entry doesn't exist - * anymore in the HW and remove the entry from the SW - * list - */ - lan966x_mac_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, - mac_entry->mac, mac_entry->vid, - lan966x->ports[mac_entry->port_index]->dev); - list_del(&mac_entry->list); - kfree(mac_entry); + /* Move the entry from SW list to a tmp list such that + * it would be deleted later + */ + list_add_tail(&mac_entry->list, &mac_deleted_entries); } } spin_unlock(&lan966x->mac_lock); + list_for_each_entry_safe(mac_entry, tmp, &mac_deleted_entries, list) { + /* Notify the bridge that the entry doesn't exist + * anymore in the HW + */ + lan966x_mac_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, + mac_entry->mac, mac_entry->vid, + lan966x->ports[mac_entry->port_index]->dev); + list_del(&mac_entry->list); + kfree(mac_entry); + } + /* Now go to the list of columns and see if any entry was not in the SW * list, then that means that the entry is new so it needs to notify the * bridge. @@ -396,13 +433,20 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row, if (WARN_ON(dest_idx >= lan966x->num_phys_ports)) continue; + spin_lock(&lan966x->mac_lock); + mac_entry = lan966x_mac_find_entry(lan966x, mac, vid, dest_idx); + if (mac_entry) { + spin_unlock(&lan966x->mac_lock); + continue; + } + mac_entry = lan966x_mac_alloc_entry(mac, vid, dest_idx); - if (!mac_entry) + if (!mac_entry) { + spin_unlock(&lan966x->mac_lock); return; + } mac_entry->row = row; - - spin_lock(&lan966x->mac_lock); list_add_tail(&mac_entry->list, &lan966x->mac_entries); spin_unlock(&lan966x->mac_lock); @@ -424,6 +468,7 @@ irqreturn_t lan966x_mac_irq_handler(struct lan966x *lan966x) lan966x, ANA_MACTINDX); while (1) { + spin_lock(&lan966x->mac_lock); lan_rmw(ANA_MACACCESS_MAC_TABLE_CMD_SET(MACACCESS_CMD_SYNC_GET_NEXT), ANA_MACACCESS_MAC_TABLE_CMD, lan966x, ANA_MACACCESS); @@ -447,12 +492,15 @@ irqreturn_t lan966x_mac_irq_handler(struct lan966x *lan966x) stop = false; if (column == LAN966X_MAC_COLUMNS - 1 && - index == 0 && stop) + index == 0 && stop) { + spin_unlock(&lan966x->mac_lock); break; + } entry[column].mach = lan_rd(lan966x, ANA_MACHDATA); entry[column].macl = lan_rd(lan966x, ANA_MACLDATA); entry[column].maca = lan_rd(lan966x, ANA_MACACCESS); + spin_unlock(&lan966x->mac_lock); /* Once all the columns are read process them */ if (column == LAN966X_MAC_COLUMNS - 1) { diff --git a/drivers/net/ethernet/netronome/nfp/flower/action.c b/drivers/net/ethernet/netronome/nfp/flower/action.c index 7453cc5ba832..3c7220a4603e 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/action.c +++ b/drivers/net/ethernet/netronome/nfp/flower/action.c @@ -474,7 +474,7 @@ nfp_fl_set_tun(struct nfp_app *app, struct nfp_fl_set_tun *set_tun, set_tun->ttl = ip4_dst_hoplimit(&rt->dst); ip_rt_put(rt); } else { - set_tun->ttl = net->ipv4.sysctl_ip_default_ttl; + set_tun->ttl = READ_ONCE(net->ipv4.sysctl_ip_default_ttl); } } diff --git a/drivers/net/ethernet/sfc/Makefile b/drivers/net/ethernet/sfc/Makefile index b9298031ea51..4c759488fc77 100644 --- a/drivers/net/ethernet/sfc/Makefile +++ b/drivers/net/ethernet/sfc/Makefile @@ -8,7 +8,7 @@ sfc-y += efx.o efx_common.o efx_channels.o nic.o \ ef100.o ef100_nic.o ef100_netdev.o \ ef100_ethtool.o ef100_rx.o ef100_tx.o sfc-$(CONFIG_SFC_MTD) += mtd.o -sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o +sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o mae.o obj-$(CONFIG_SFC) += sfc.o diff --git a/drivers/net/ethernet/sfc/ef100_netdev.c b/drivers/net/ethernet/sfc/ef100_netdev.c index 060392ddd612..9e65de1ab889 100644 --- a/drivers/net/ethernet/sfc/ef100_netdev.c +++ b/drivers/net/ethernet/sfc/ef100_netdev.c @@ -85,6 +85,7 @@ static int ef100_net_stop(struct net_device *net_dev) netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", raw_smp_processor_id()); + efx_detach_reps(efx); netif_stop_queue(net_dev); efx_stop_all(efx); efx_mcdi_mac_fini_stats(efx); @@ -176,6 +177,8 @@ static int ef100_net_open(struct net_device *net_dev) mutex_unlock(&efx->mac_lock); efx->state = STATE_NET_UP; + if (netif_running(efx->net_dev)) + efx_attach_reps(efx); return 0; @@ -195,6 +198,15 @@ static netdev_tx_t ef100_hard_start_xmit(struct sk_buff *skb, struct net_device *net_dev) { struct efx_nic *efx = efx_netdev_priv(net_dev); + + return __ef100_hard_start_xmit(skb, efx, net_dev, NULL); +} + +netdev_tx_t __ef100_hard_start_xmit(struct sk_buff *skb, + struct efx_nic *efx, + struct net_device *net_dev, + struct efx_rep *efv) +{ struct efx_tx_queue *tx_queue; struct efx_channel *channel; int rc; @@ -209,7 +221,7 @@ static netdev_tx_t ef100_hard_start_xmit(struct sk_buff *skb, } tx_queue = &channel->tx_queue[0]; - rc = ef100_enqueue_skb(tx_queue, skb); + rc = __ef100_enqueue_skb(tx_queue, skb, efv); if (rc == 0) return NETDEV_TX_OK; @@ -312,7 +324,7 @@ void ef100_remove_netdev(struct efx_probe_data *probe_data) unregister_netdevice_notifier(&efx->netdev_notifier); #if defined(CONFIG_SFC_SRIOV) if (!efx->type->is_vf) - efx_ef100_pci_sriov_disable(efx); + efx_ef100_pci_sriov_disable(efx, true); #endif ef100_unregister_netdev(efx); diff --git a/drivers/net/ethernet/sfc/ef100_netdev.h b/drivers/net/ethernet/sfc/ef100_netdev.h index 38b032ba0953..86bf985e0951 100644 --- a/drivers/net/ethernet/sfc/ef100_netdev.h +++ b/drivers/net/ethernet/sfc/ef100_netdev.h @@ -10,7 +10,12 @@ */ #include <linux/netdevice.h> +#include "ef100_rep.h" +netdev_tx_t __ef100_hard_start_xmit(struct sk_buff *skb, + struct efx_nic *efx, + struct net_device *net_dev, + struct efx_rep *efv); int ef100_netdev_event(struct notifier_block *this, unsigned long event, void *ptr); int ef100_probe_netdev(struct efx_probe_data *probe_data); diff --git a/drivers/net/ethernet/sfc/ef100_nic.c b/drivers/net/ethernet/sfc/ef100_nic.c index f89e695cf8ac..4625d35269e6 100644 --- a/drivers/net/ethernet/sfc/ef100_nic.c +++ b/drivers/net/ethernet/sfc/ef100_nic.c @@ -946,6 +946,7 @@ static int ef100_probe_main(struct efx_nic *efx) unsigned int bar_size = resource_size(&efx->pci_dev->resource[efx->mem_bar]); struct ef100_nic_data *nic_data; char fw_version[32]; + u32 priv_mask = 0; int i, rc; if (WARN_ON(bar_size == 0)) @@ -1027,6 +1028,12 @@ static int ef100_probe_main(struct efx_nic *efx) efx_mcdi_print_fwver(efx, fw_version, sizeof(fw_version)); pci_dbg(efx->pci_dev, "Firmware version %s\n", fw_version); + rc = efx_mcdi_get_privilege_mask(efx, &priv_mask); + if (rc) /* non-fatal, and priv_mask will still be 0 */ + pci_info(efx->pci_dev, + "Failed to get privilege mask from FW, rc %d\n", rc); + nic_data->grp_mae = !!(priv_mask & MC_CMD_PRIVILEGE_MASK_IN_GRP_MAE); + if (compare_versions(fw_version, "1.1.0.1000") < 0) { pci_info(efx->pci_dev, "Firmware uses old event descriptors\n"); rc = -EINVAL; diff --git a/drivers/net/ethernet/sfc/ef100_nic.h b/drivers/net/ethernet/sfc/ef100_nic.h index 744dbbdb4adc..40f84a275057 100644 --- a/drivers/net/ethernet/sfc/ef100_nic.h +++ b/drivers/net/ethernet/sfc/ef100_nic.h @@ -72,6 +72,7 @@ struct ef100_nic_data { u8 port_id[ETH_ALEN]; DECLARE_BITMAP(evq_phases, EFX_MAX_CHANNELS); u64 stats[EF100_STAT_COUNT]; + bool grp_mae; /* MAE Privilege */ u16 tso_max_hdr_len; u16 tso_max_payload_num_segs; u16 tso_max_frames; diff --git a/drivers/net/ethernet/sfc/ef100_regs.h b/drivers/net/ethernet/sfc/ef100_regs.h index 710bbdb19885..982b6ab1eb62 100644 --- a/drivers/net/ethernet/sfc/ef100_regs.h +++ b/drivers/net/ethernet/sfc/ef100_regs.h @@ -2,7 +2,7 @@ /**************************************************************************** * Driver for Solarflare network controllers and boards * Copyright 2018 Solarflare Communications Inc. - * Copyright 2019-2020 Xilinx Inc. + * Copyright 2019-2022 Xilinx Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -181,12 +181,6 @@ /* RHEAD_BASE_EVENT */ #define ESF_GZ_E_TYPE_LBN 60 #define ESF_GZ_E_TYPE_WIDTH 4 -#define ESE_GZ_EF100_EV_DRIVER 5 -#define ESE_GZ_EF100_EV_MCDI 4 -#define ESE_GZ_EF100_EV_CONTROL 3 -#define ESE_GZ_EF100_EV_TX_TIMESTAMP 2 -#define ESE_GZ_EF100_EV_TX_COMPLETION 1 -#define ESE_GZ_EF100_EV_RX_PKTS 0 #define ESF_GZ_EV_EVQ_PHASE_LBN 59 #define ESF_GZ_EV_EVQ_PHASE_WIDTH 1 #define ESE_GZ_RHEAD_BASE_EVENT_STRUCT_SIZE 64 @@ -369,14 +363,18 @@ #define ESF_GZ_RX_PREFIX_VLAN_STRIP_TCI_WIDTH 16 #define ESF_GZ_RX_PREFIX_CSUM_FRAME_LBN 144 #define ESF_GZ_RX_PREFIX_CSUM_FRAME_WIDTH 16 -#define ESF_GZ_RX_PREFIX_INGRESS_VPORT_LBN 128 -#define ESF_GZ_RX_PREFIX_INGRESS_VPORT_WIDTH 16 +#define ESF_GZ_RX_PREFIX_INGRESS_MPORT_LBN 128 +#define ESF_GZ_RX_PREFIX_INGRESS_MPORT_WIDTH 16 #define ESF_GZ_RX_PREFIX_USER_MARK_LBN 96 #define ESF_GZ_RX_PREFIX_USER_MARK_WIDTH 32 #define ESF_GZ_RX_PREFIX_RSS_HASH_LBN 64 #define ESF_GZ_RX_PREFIX_RSS_HASH_WIDTH 32 -#define ESF_GZ_RX_PREFIX_PARTIAL_TSTAMP_LBN 32 -#define ESF_GZ_RX_PREFIX_PARTIAL_TSTAMP_WIDTH 32 +#define ESF_GZ_RX_PREFIX_PARTIAL_TSTAMP_LBN 34 +#define ESF_GZ_RX_PREFIX_PARTIAL_TSTAMP_WIDTH 30 +#define ESF_GZ_RX_PREFIX_VSWITCH_STATUS_LBN 33 +#define ESF_GZ_RX_PREFIX_VSWITCH_STATUS_WIDTH 1 +#define ESF_GZ_RX_PREFIX_VLAN_STRIPPED_LBN 32 +#define ESF_GZ_RX_PREFIX_VLAN_STRIPPED_WIDTH 1 #define ESF_GZ_RX_PREFIX_CLASS_LBN 16 #define ESF_GZ_RX_PREFIX_CLASS_WIDTH 16 #define ESF_GZ_RX_PREFIX_USER_FLAG_LBN 15 @@ -454,12 +452,8 @@ #define ESF_GZ_M2M_TRANSLATE_ADDR_WIDTH 1 #define ESF_GZ_M2M_RSVD_LBN 120 #define ESF_GZ_M2M_RSVD_WIDTH 2 -#define ESF_GZ_M2M_ADDR_SPC_LBN 108 -#define ESF_GZ_M2M_ADDR_SPC_WIDTH 12 -#define ESF_GZ_M2M_ADDR_SPC_PASID_LBN 86 -#define ESF_GZ_M2M_ADDR_SPC_PASID_WIDTH 22 -#define ESF_GZ_M2M_ADDR_SPC_MODE_LBN 84 -#define ESF_GZ_M2M_ADDR_SPC_MODE_WIDTH 2 +#define ESF_GZ_M2M_ADDR_SPC_ID_LBN 84 +#define ESF_GZ_M2M_ADDR_SPC_ID_WIDTH 36 #define ESF_GZ_M2M_LEN_MINUS_1_LBN 64 #define ESF_GZ_M2M_LEN_MINUS_1_WIDTH 20 #define ESF_GZ_M2M_ADDR_LBN 0 @@ -492,12 +486,8 @@ #define ESF_GZ_TX_SEG_TRANSLATE_ADDR_WIDTH 1 #define ESF_GZ_TX_SEG_RSVD2_LBN 120 #define ESF_GZ_TX_SEG_RSVD2_WIDTH 2 -#define ESF_GZ_TX_SEG_ADDR_SPC_LBN 108 -#define ESF_GZ_TX_SEG_ADDR_SPC_WIDTH 12 -#define ESF_GZ_TX_SEG_ADDR_SPC_PASID_LBN 86 -#define ESF_GZ_TX_SEG_ADDR_SPC_PASID_WIDTH 22 -#define ESF_GZ_TX_SEG_ADDR_SPC_MODE_LBN 84 -#define ESF_GZ_TX_SEG_ADDR_SPC_MODE_WIDTH 2 +#define ESF_GZ_TX_SEG_ADDR_SPC_ID_LBN 84 +#define ESF_GZ_TX_SEG_ADDR_SPC_ID_WIDTH 36 #define ESF_GZ_TX_SEG_RSVD_LBN 80 #define ESF_GZ_TX_SEG_RSVD_WIDTH 4 #define ESF_GZ_TX_SEG_LEN_LBN 64 @@ -583,6 +573,12 @@ #define ESE_GZ_SF_TX_TSO_DSC_FMT_STRUCT_SIZE 124 +/* Enum D2VIO_MSG_OP */ +#define ESE_GZ_QUE_JBDNE 3 +#define ESE_GZ_QUE_EVICT 2 +#define ESE_GZ_QUE_EMPTY 1 +#define ESE_GZ_NOP 0 + /* Enum DESIGN_PARAMS */ #define ESE_EF100_DP_GZ_RX_MAX_RUNT 17 #define ESE_EF100_DP_GZ_VI_STRIDES 16 @@ -630,6 +626,19 @@ #define ESE_GZ_PCI_BASE_CONFIG_SPACE_SIZE 256 #define ESE_GZ_PCI_EXPRESS_XCAP_HDR_SIZE 4 +/* Enum RH_DSC_TYPE */ +#define ESE_GZ_TX_TOMB 0xF +#define ESE_GZ_TX_VIO 0xE +#define ESE_GZ_TX_TSO_OVRRD 0x8 +#define ESE_GZ_TX_D2CMP 0x7 +#define ESE_GZ_TX_DATA 0x6 +#define ESE_GZ_TX_D2M 0x5 +#define ESE_GZ_TX_M2M 0x4 +#define ESE_GZ_TX_SEG 0x3 +#define ESE_GZ_TX_TSO 0x2 +#define ESE_GZ_TX_OVRRD 0x1 +#define ESE_GZ_TX_SEND 0x0 + /* Enum RH_HCLASS_L2_CLASS */ #define ESE_GZ_RH_HCLASS_L2_CLASS_E2_0123VLAN 1 #define ESE_GZ_RH_HCLASS_L2_CLASS_OTHER 0 @@ -666,6 +675,25 @@ #define ESE_GZ_RH_HCLASS_TUNNEL_CLASS_VXLAN 1 #define ESE_GZ_RH_HCLASS_TUNNEL_CLASS_NONE 0 +/* Enum SF_CTL_EVENT_SUBTYPE */ +#define ESE_GZ_EF100_CTL_EV_EVQ_TIMEOUT 0x3 +#define ESE_GZ_EF100_CTL_EV_FLUSH 0x2 +#define ESE_GZ_EF100_CTL_EV_TIME_SYNC 0x1 +#define ESE_GZ_EF100_CTL_EV_UNSOL_OVERFLOW 0x0 + +/* Enum SF_EVENT_TYPE */ +#define ESE_GZ_EF100_EV_DRIVER 0x5 +#define ESE_GZ_EF100_EV_MCDI 0x4 +#define ESE_GZ_EF100_EV_CONTROL 0x3 +#define ESE_GZ_EF100_EV_TX_TIMESTAMP 0x2 +#define ESE_GZ_EF100_EV_TX_COMPLETION 0x1 +#define ESE_GZ_EF100_EV_RX_PKTS 0x0 + +/* Enum SF_EW_EVENT_TYPE */ +#define ESE_GZ_EF100_EWEV_VIRTQ_DESC 0x2 +#define ESE_GZ_EF100_EWEV_TXQ_DESC 0x1 +#define ESE_GZ_EF100_EWEV_64BIT 0x0 + /* Enum TX_DESC_CSO_PARTIAL_EN */ #define ESE_GZ_TX_DESC_CSO_PARTIAL_EN_TCP 2 #define ESE_GZ_TX_DESC_CSO_PARTIAL_EN_UDP 1 @@ -681,6 +709,15 @@ #define ESE_GZ_TX_DESC_IP4_ID_INC_MOD16 2 #define ESE_GZ_TX_DESC_IP4_ID_INC_MOD15 1 #define ESE_GZ_TX_DESC_IP4_ID_NO_OP 0 + +/* Enum VIRTIO_NET_HDR_F */ +#define ESE_GZ_NEEDS_CSUM 0x1 + +/* Enum VIRTIO_NET_HDR_GSO */ +#define ESE_GZ_TCPV6 0x4 +#define ESE_GZ_UDP 0x3 +#define ESE_GZ_TCPV4 0x1 +#define ESE_GZ_NONE 0x0 /**************************************************************************/ #define ESF_GZ_EV_DEBUG_EVENT_GEN_FLAGS_LBN 44 diff --git a/drivers/net/ethernet/sfc/ef100_rep.c b/drivers/net/ethernet/sfc/ef100_rep.c new file mode 100644 index 000000000000..d07539f091b8 --- /dev/null +++ b/drivers/net/ethernet/sfc/ef100_rep.c @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-2.0-only +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2019 Solarflare Communications Inc. + * Copyright 2020-2022 Xilinx Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#include "ef100_rep.h" +#include "ef100_netdev.h" +#include "ef100_nic.h" +#include "mae.h" + +#define EFX_EF100_REP_DRIVER "efx_ef100_rep" + +static int efx_ef100_rep_init_struct(struct efx_nic *efx, struct efx_rep *efv, + unsigned int i) +{ + efv->parent = efx; + efv->idx = i; + INIT_LIST_HEAD(&efv->list); + efv->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE | + NETIF_MSG_LINK | NETIF_MSG_IFDOWN | + NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | + NETIF_MSG_TX_ERR | NETIF_MSG_HW; + return 0; +} + +static netdev_tx_t efx_ef100_rep_xmit(struct sk_buff *skb, + struct net_device *dev) +{ + struct efx_rep *efv = netdev_priv(dev); + struct efx_nic *efx = efv->parent; + netdev_tx_t rc; + + /* __ef100_hard_start_xmit() will always return success even in the + * case of TX drops, where it will increment efx's tx_dropped. The + * efv stats really only count attempted TX, not success/failure. + */ + atomic64_inc(&efv->stats.tx_packets); + atomic64_add(skb->len, &efv->stats.tx_bytes); + netif_tx_lock(efx->net_dev); + rc = __ef100_hard_start_xmit(skb, efx, dev, efv); + netif_tx_unlock(efx->net_dev); + return rc; +} + +static int efx_ef100_rep_get_port_parent_id(struct net_device *dev, + struct netdev_phys_item_id *ppid) +{ + struct efx_rep *efv = netdev_priv(dev); + struct efx_nic *efx = efv->parent; + struct ef100_nic_data *nic_data; + + nic_data = efx->nic_data; + /* nic_data->port_id is a u8[] */ + ppid->id_len = sizeof(nic_data->port_id); + memcpy(ppid->id, nic_data->port_id, sizeof(nic_data->port_id)); + return 0; +} + +static int efx_ef100_rep_get_phys_port_name(struct net_device *dev, + char *buf, size_t len) +{ + struct efx_rep *efv = netdev_priv(dev); + struct efx_nic *efx = efv->parent; + struct ef100_nic_data *nic_data; + int ret; + + nic_data = efx->nic_data; + ret = snprintf(buf, len, "p%upf%uvf%u", efx->port_num, + nic_data->pf_index, efv->idx); + if (ret >= len) + return -EOPNOTSUPP; + + return 0; +} + +static const struct net_device_ops efx_ef100_rep_netdev_ops = { + .ndo_start_xmit = efx_ef100_rep_xmit, + .ndo_get_port_parent_id = efx_ef100_rep_get_port_parent_id, + .ndo_get_phys_port_name = efx_ef100_rep_get_phys_port_name, +}; + +static void efx_ef100_rep_get_drvinfo(struct net_device *dev, + struct ethtool_drvinfo *drvinfo) +{ + strscpy(drvinfo->driver, EFX_EF100_REP_DRIVER, sizeof(drvinfo->driver)); +} + +static u32 efx_ef100_rep_ethtool_get_msglevel(struct net_device *net_dev) +{ + struct efx_rep *efv = netdev_priv(net_dev); + + return efv->msg_enable; +} + +static void efx_ef100_rep_ethtool_set_msglevel(struct net_device *net_dev, + u32 msg_enable) +{ + struct efx_rep *efv = netdev_priv(net_dev); + + efv->msg_enable = msg_enable; +} + +static const struct ethtool_ops efx_ef100_rep_ethtool_ops = { + .get_drvinfo = efx_ef100_rep_get_drvinfo, + .get_msglevel = efx_ef100_rep_ethtool_get_msglevel, + .set_msglevel = efx_ef100_rep_ethtool_set_msglevel, +}; + +static struct efx_rep *efx_ef100_rep_create_netdev(struct efx_nic *efx, + unsigned int i) +{ + struct net_device *net_dev; + struct efx_rep *efv; + int rc; + + net_dev = alloc_etherdev_mq(sizeof(*efv), 1); + if (!net_dev) + return ERR_PTR(-ENOMEM); + + efv = netdev_priv(net_dev); + rc = efx_ef100_rep_init_struct(efx, efv, i); + if (rc) + goto fail1; + efv->net_dev = net_dev; + rtnl_lock(); + spin_lock_bh(&efx->vf_reps_lock); + list_add_tail(&efv->list, &efx->vf_reps); + spin_unlock_bh(&efx->vf_reps_lock); + if (netif_running(efx->net_dev) && efx->state == STATE_NET_UP) { + netif_device_attach(net_dev); + netif_carrier_on(net_dev); + } else { + netif_carrier_off(net_dev); + netif_tx_stop_all_queues(net_dev); + } + rtnl_unlock(); + + net_dev->netdev_ops = &efx_ef100_rep_netdev_ops; + net_dev->ethtool_ops = &efx_ef100_rep_ethtool_ops; + net_dev->min_mtu = EFX_MIN_MTU; + net_dev->max_mtu = EFX_MAX_MTU; + net_dev->features |= NETIF_F_LLTX; + net_dev->hw_features |= NETIF_F_LLTX; + return efv; +fail1: + free_netdev(net_dev); + return ERR_PTR(rc); +} + +static int efx_ef100_configure_rep(struct efx_rep *efv) +{ + struct efx_nic *efx = efv->parent; + u32 selector; + int rc; + + /* Construct mport selector for corresponding VF */ + efx_mae_mport_vf(efx, efv->idx, &selector); + /* Look up actual mport ID */ + rc = efx_mae_lookup_mport(efx, selector, &efv->mport); + if (rc) + return rc; + pci_dbg(efx->pci_dev, "VF %u has mport ID %#x\n", efv->idx, efv->mport); + /* mport label should fit in 16 bits */ + WARN_ON(efv->mport >> 16); + + return 0; +} + +static void efx_ef100_rep_destroy_netdev(struct efx_rep *efv) +{ + struct efx_nic *efx = efv->parent; + + rtnl_lock(); + spin_lock_bh(&efx->vf_reps_lock); + list_del(&efv->list); + spin_unlock_bh(&efx->vf_reps_lock); + rtnl_unlock(); + free_netdev(efv->net_dev); +} + +int efx_ef100_vfrep_create(struct efx_nic *efx, unsigned int i) +{ + struct efx_rep *efv; + int rc; + + efv = efx_ef100_rep_create_netdev(efx, i); + if (IS_ERR(efv)) { + rc = PTR_ERR(efv); + pci_err(efx->pci_dev, + "Failed to create representor for VF %d, rc %d\n", i, + rc); + return rc; + } + rc = efx_ef100_configure_rep(efv); + if (rc) { + pci_err(efx->pci_dev, + "Failed to configure representor for VF %d, rc %d\n", + i, rc); + goto fail; + } + rc = register_netdev(efv->net_dev); + if (rc) { + pci_err(efx->pci_dev, + "Failed to register representor for VF %d, rc %d\n", + i, rc); + goto fail; + } + pci_dbg(efx->pci_dev, "Representor for VF %d is %s\n", i, + efv->net_dev->name); + return 0; +fail: + efx_ef100_rep_destroy_netdev(efv); + return rc; +} + +void efx_ef100_vfrep_destroy(struct efx_nic *efx, struct efx_rep *efv) +{ + struct net_device *rep_dev; + + rep_dev = efv->net_dev; + if (!rep_dev) + return; + netif_dbg(efx, drv, rep_dev, "Removing VF representor\n"); + unregister_netdev(rep_dev); + efx_ef100_rep_destroy_netdev(efv); +} + +void efx_ef100_fini_vfreps(struct efx_nic *efx) +{ + struct ef100_nic_data *nic_data = efx->nic_data; + struct efx_rep *efv, *next; + + if (!nic_data->grp_mae) + return; + + list_for_each_entry_safe(efv, next, &efx->vf_reps, list) + efx_ef100_vfrep_destroy(efx, efv); +} diff --git a/drivers/net/ethernet/sfc/ef100_rep.h b/drivers/net/ethernet/sfc/ef100_rep.h new file mode 100644 index 000000000000..d47fd8ff6220 --- /dev/null +++ b/drivers/net/ethernet/sfc/ef100_rep.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2019 Solarflare Communications Inc. + * Copyright 2020-2022 Xilinx Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +/* Handling for ef100 representor netdevs */ +#ifndef EF100_REP_H +#define EF100_REP_H + +#include "net_driver.h" + +struct efx_rep_sw_stats { + atomic64_t rx_packets, tx_packets; + atomic64_t rx_bytes, tx_bytes; + atomic64_t rx_dropped, tx_errors; +}; + +/** + * struct efx_rep - Private data for an Efx representor + * + * @parent: the efx PF which manages this representor + * @net_dev: representor netdevice + * @msg_enable: log message enable flags + * @mport: m-port ID of corresponding VF + * @idx: VF index + * @list: entry on efx->vf_reps + * @stats: software traffic counters for netdev stats + */ +struct efx_rep { + struct efx_nic *parent; + struct net_device *net_dev; + u32 msg_enable; + u32 mport; + unsigned int idx; + struct list_head list; + struct efx_rep_sw_stats stats; +}; + +int efx_ef100_vfrep_create(struct efx_nic *efx, unsigned int i); +void efx_ef100_vfrep_destroy(struct efx_nic *efx, struct efx_rep *efv); +void efx_ef100_fini_vfreps(struct efx_nic *efx); + +#endif /* EF100_REP_H */ diff --git a/drivers/net/ethernet/sfc/ef100_sriov.c b/drivers/net/ethernet/sfc/ef100_sriov.c index 664578176bfe..94bdbfcb47e8 100644 --- a/drivers/net/ethernet/sfc/ef100_sriov.c +++ b/drivers/net/ethernet/sfc/ef100_sriov.c @@ -11,46 +11,62 @@ #include "ef100_sriov.h" #include "ef100_nic.h" +#include "ef100_rep.h" static int efx_ef100_pci_sriov_enable(struct efx_nic *efx, int num_vfs) { + struct ef100_nic_data *nic_data = efx->nic_data; struct pci_dev *dev = efx->pci_dev; - int rc; + struct efx_rep *efv, *next; + int rc, i; efx->vf_count = num_vfs; rc = pci_enable_sriov(dev, num_vfs); if (rc) - goto fail; + goto fail1; + if (!nic_data->grp_mae) + return 0; + + for (i = 0; i < num_vfs; i++) { + rc = efx_ef100_vfrep_create(efx, i); + if (rc) + goto fail2; + } return 0; -fail: +fail2: + list_for_each_entry_safe(efv, next, &efx->vf_reps, list) + efx_ef100_vfrep_destroy(efx, efv); + pci_disable_sriov(dev); +fail1: netif_err(efx, probe, efx->net_dev, "Failed to enable SRIOV VFs\n"); efx->vf_count = 0; return rc; } -int efx_ef100_pci_sriov_disable(struct efx_nic *efx) +int efx_ef100_pci_sriov_disable(struct efx_nic *efx, bool force) { struct pci_dev *dev = efx->pci_dev; unsigned int vfs_assigned; vfs_assigned = pci_vfs_assigned(dev); - if (vfs_assigned) { + if (vfs_assigned && !force) { netif_info(efx, drv, efx->net_dev, "VFs are assigned to guests; " "please detach them before disabling SR-IOV\n"); return -EBUSY; } - pci_disable_sriov(dev); - + efx_ef100_fini_vfreps(efx); + if (!vfs_assigned) + pci_disable_sriov(dev); return 0; } int efx_ef100_sriov_configure(struct efx_nic *efx, int num_vfs) { if (num_vfs == 0) - return efx_ef100_pci_sriov_disable(efx); + return efx_ef100_pci_sriov_disable(efx, false); else return efx_ef100_pci_sriov_enable(efx, num_vfs); } diff --git a/drivers/net/ethernet/sfc/ef100_sriov.h b/drivers/net/ethernet/sfc/ef100_sriov.h index c48fccd46c57..8ffdf464dd1d 100644 --- a/drivers/net/ethernet/sfc/ef100_sriov.h +++ b/drivers/net/ethernet/sfc/ef100_sriov.h @@ -11,4 +11,4 @@ #include "net_driver.h" int efx_ef100_sriov_configure(struct efx_nic *efx, int num_vfs); -int efx_ef100_pci_sriov_disable(struct efx_nic *efx); +int efx_ef100_pci_sriov_disable(struct efx_nic *efx, bool force); diff --git a/drivers/net/ethernet/sfc/ef100_tx.c b/drivers/net/ethernet/sfc/ef100_tx.c index 26ef51d6b542..102ddc7e206a 100644 --- a/drivers/net/ethernet/sfc/ef100_tx.c +++ b/drivers/net/ethernet/sfc/ef100_tx.c @@ -254,7 +254,8 @@ static void ef100_make_tso_desc(struct efx_nic *efx, static void ef100_tx_make_descriptors(struct efx_tx_queue *tx_queue, const struct sk_buff *skb, - unsigned int segment_count) + unsigned int segment_count, + struct efx_rep *efv) { unsigned int old_write_count = tx_queue->write_count; unsigned int new_write_count = old_write_count; @@ -272,6 +273,20 @@ static void ef100_tx_make_descriptors(struct efx_tx_queue *tx_queue, else next_desc_type = ESE_GZ_TX_DESC_TYPE_SEND; + if (unlikely(efv)) { + /* Create TX override descriptor */ + write_ptr = new_write_count & tx_queue->ptr_mask; + txd = ef100_tx_desc(tx_queue, write_ptr); + ++new_write_count; + + tx_queue->packet_write_count = new_write_count; + EFX_POPULATE_OWORD_3(*txd, + ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_PREFIX, + ESF_GZ_TX_PREFIX_EGRESS_MPORT, efv->mport, + ESF_GZ_TX_PREFIX_EGRESS_MPORT_EN, 1); + nr_descs--; + } + /* if it's a raw write (such as XDP) then always SEND single frames */ if (!skb) nr_descs = 1; @@ -306,6 +321,9 @@ static void ef100_tx_make_descriptors(struct efx_tx_queue *tx_queue, /* if it's a raw write (such as XDP) then always SEND */ next_desc_type = skb ? ESE_GZ_TX_DESC_TYPE_SEG : ESE_GZ_TX_DESC_TYPE_SEND; + /* mark as an EFV buffer if applicable */ + if (unlikely(efv)) + buffer->flags |= EFX_TX_BUF_EFV; } while (new_write_count != tx_queue->insert_count); @@ -324,7 +342,7 @@ static void ef100_tx_make_descriptors(struct efx_tx_queue *tx_queue, void ef100_tx_write(struct efx_tx_queue *tx_queue) { - ef100_tx_make_descriptors(tx_queue, NULL, 0); + ef100_tx_make_descriptors(tx_queue, NULL, 0, NULL); ef100_tx_push_buffers(tx_queue); } @@ -351,6 +369,12 @@ void ef100_ev_tx(struct efx_channel *channel, const efx_qword_t *p_event) */ int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) { + return __ef100_enqueue_skb(tx_queue, skb, NULL); +} + +int __ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb, + struct efx_rep *efv) +{ unsigned int old_insert_count = tx_queue->insert_count; struct efx_nic *efx = tx_queue->efx; bool xmit_more = netdev_xmit_more(); @@ -376,16 +400,64 @@ int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) return 0; } + if (unlikely(efv)) { + struct efx_tx_buffer *buffer = __efx_tx_queue_get_insert_buffer(tx_queue); + + /* Drop representor packets if the queue is stopped. + * We currently don't assert backoff to representors so this is + * to make sure representor traffic can't starve the main + * net device. + * And, of course, if there are no TX descriptors left. + */ + if (netif_tx_queue_stopped(tx_queue->core_txq) || + unlikely(efx_tx_buffer_in_use(buffer))) { + atomic64_inc(&efv->stats.tx_errors); + rc = -ENOSPC; + goto err; + } + + /* Also drop representor traffic if it could cause us to + * stop the queue. If we assert backoff and we haven't + * received traffic on the main net device recently then the + * TX watchdog can go off erroneously. + */ + fill_level = efx_channel_tx_old_fill_level(tx_queue->channel); + fill_level += efx_tx_max_skb_descs(efx); + if (fill_level > efx->txq_stop_thresh) { + struct efx_tx_queue *txq2; + + /* Refresh cached fill level and re-check */ + efx_for_each_channel_tx_queue(txq2, tx_queue->channel) + txq2->old_read_count = READ_ONCE(txq2->read_count); + + fill_level = efx_channel_tx_old_fill_level(tx_queue->channel); + fill_level += efx_tx_max_skb_descs(efx); + if (fill_level > efx->txq_stop_thresh) { + atomic64_inc(&efv->stats.tx_errors); + rc = -ENOSPC; + goto err; + } + } + + buffer->flags = EFX_TX_BUF_OPTION | EFX_TX_BUF_EFV; + tx_queue->insert_count++; + } + /* Map for DMA and create descriptors */ rc = efx_tx_map_data(tx_queue, skb, segments); if (rc) goto err; - ef100_tx_make_descriptors(tx_queue, skb, segments); + ef100_tx_make_descriptors(tx_queue, skb, segments, efv); fill_level = efx_channel_tx_old_fill_level(tx_queue->channel); if (fill_level > efx->txq_stop_thresh) { struct efx_tx_queue *txq2; + /* Because of checks above, representor traffic should + * not be able to stop the queue. + */ + WARN_ON(efv); + netif_tx_stop_queue(tx_queue->core_txq); /* Re-read after a memory barrier in case we've raced with * the completion path. Otherwise there's a danger we'll never @@ -404,8 +476,12 @@ int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) /* If xmit_more then we don't need to push the doorbell, unless there * are 256 descriptors already queued in which case we have to push to * ensure we never push more than 256 at once. + * + * Always push for representor traffic, and don't account it to parent + * PF netdevice's BQL. */ - if (__netdev_tx_sent_queue(tx_queue->core_txq, skb->len, xmit_more) || + if (unlikely(efv) || + __netdev_tx_sent_queue(tx_queue->core_txq, skb->len, xmit_more) || tx_queue->write_count - tx_queue->notify_count > 255) ef100_tx_push_buffers(tx_queue); diff --git a/drivers/net/ethernet/sfc/ef100_tx.h b/drivers/net/ethernet/sfc/ef100_tx.h index ddc4b98fa6db..e9e11540fcde 100644 --- a/drivers/net/ethernet/sfc/ef100_tx.h +++ b/drivers/net/ethernet/sfc/ef100_tx.h @@ -13,6 +13,7 @@ #define EFX_EF100_TX_H #include "net_driver.h" +#include "ef100_rep.h" int ef100_tx_probe(struct efx_tx_queue *tx_queue); void ef100_tx_init(struct efx_tx_queue *tx_queue); @@ -22,4 +23,6 @@ unsigned int ef100_tx_max_skb_descs(struct efx_nic *efx); void ef100_ev_tx(struct efx_channel *channel, const efx_qword_t *p_event); netdev_tx_t ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb); +int __ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb, + struct efx_rep *efv); #endif diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h index c05a83da9e44..4239c7ece123 100644 --- a/drivers/net/ethernet/sfc/efx.h +++ b/drivers/net/ethernet/sfc/efx.h @@ -12,6 +12,7 @@ #include "net_driver.h" #include "ef100_rx.h" #include "ef100_tx.h" +#include "efx_common.h" #include "filter.h" int efx_net_open(struct net_device *net_dev); @@ -206,6 +207,9 @@ static inline void efx_device_detach_sync(struct efx_nic *efx) { struct net_device *dev = efx->net_dev; + /* We must stop reps (which use our TX) before we stop ourselves. */ + efx_detach_reps(efx); + /* Lock/freeze all TX queues so that we can be sure the * TX scheduler is stopped when we're done and before * netif_device_present() becomes false. @@ -217,8 +221,11 @@ static inline void efx_device_detach_sync(struct efx_nic *efx) static inline void efx_device_attach_if_not_resetting(struct efx_nic *efx) { - if ((efx->state != STATE_DISABLED) && !efx->reset_pending) + if ((efx->state != STATE_DISABLED) && !efx->reset_pending) { netif_device_attach(efx->net_dev); + if (efx->state == STATE_NET_UP) + efx_attach_reps(efx); + } } static inline bool efx_rwsem_assert_write_locked(struct rw_semaphore *sem) diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c index 56eb717bb07a..a929a1aaba92 100644 --- a/drivers/net/ethernet/sfc/efx_common.c +++ b/drivers/net/ethernet/sfc/efx_common.c @@ -24,6 +24,7 @@ #include "mcdi_port_common.h" #include "io.h" #include "mcdi_pcol.h" +#include "ef100_rep.h" static unsigned int debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | NETIF_MSG_IFDOWN | @@ -1021,6 +1022,8 @@ int efx_init_struct(struct efx_nic *efx, struct pci_dev *pci_dev) efx->rps_hash_table = kcalloc(EFX_ARFS_HASH_TABLE_SIZE, sizeof(*efx->rps_hash_table), GFP_KERNEL); #endif + spin_lock_init(&efx->vf_reps_lock); + INIT_LIST_HEAD(&efx->vf_reps); INIT_WORK(&efx->mac_work, efx_mac_work); init_waitqueue_head(&efx->flush_wq); @@ -1389,3 +1392,38 @@ int efx_get_phys_port_name(struct net_device *net_dev, char *name, size_t len) return -EINVAL; return 0; } + +void efx_detach_reps(struct efx_nic *efx) +{ + struct net_device *rep_dev; + struct efx_rep *efv; + + ASSERT_RTNL(); + netif_dbg(efx, drv, efx->net_dev, "Detaching VF representors\n"); + list_for_each_entry(efv, &efx->vf_reps, list) { + rep_dev = efv->net_dev; + if (!rep_dev) + continue; + netif_carrier_off(rep_dev); + /* See efx_device_detach_sync() */ + netif_tx_lock_bh(rep_dev); + netif_tx_stop_all_queues(rep_dev); + netif_tx_unlock_bh(rep_dev); + } +} + +void efx_attach_reps(struct efx_nic *efx) +{ + struct net_device *rep_dev; + struct efx_rep *efv; + + ASSERT_RTNL(); + netif_dbg(efx, drv, efx->net_dev, "Attaching VF representors\n"); + list_for_each_entry(efv, &efx->vf_reps, list) { + rep_dev = efv->net_dev; + if (!rep_dev) + continue; + netif_tx_wake_all_queues(rep_dev); + netif_carrier_on(rep_dev); + } +} diff --git a/drivers/net/ethernet/sfc/efx_common.h b/drivers/net/ethernet/sfc/efx_common.h index 93babc1a2678..2c54dac3e662 100644 --- a/drivers/net/ethernet/sfc/efx_common.h +++ b/drivers/net/ethernet/sfc/efx_common.h @@ -111,4 +111,7 @@ int efx_get_phys_port_id(struct net_device *net_dev, int efx_get_phys_port_name(struct net_device *net_dev, char *name, size_t len); + +void efx_detach_reps(struct efx_nic *efx); +void efx_attach_reps(struct efx_nic *efx); #endif diff --git a/drivers/net/ethernet/sfc/mae.c b/drivers/net/ethernet/sfc/mae.c new file mode 100644 index 000000000000..011ebd46ada5 --- /dev/null +++ b/drivers/net/ethernet/sfc/mae.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2019 Solarflare Communications Inc. + * Copyright 2020-2022 Xilinx Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#include "mae.h" +#include "mcdi.h" +#include "mcdi_pcol.h" + +void efx_mae_mport_vf(struct efx_nic *efx __always_unused, u32 vf_id, u32 *out) +{ + efx_dword_t mport; + + EFX_POPULATE_DWORD_3(mport, + MAE_MPORT_SELECTOR_TYPE, MAE_MPORT_SELECTOR_TYPE_FUNC, + MAE_MPORT_SELECTOR_FUNC_PF_ID, MAE_MPORT_SELECTOR_FUNC_PF_ID_CALLER, + MAE_MPORT_SELECTOR_FUNC_VF_ID, vf_id); + *out = EFX_DWORD_VAL(mport); +} + +/* id is really only 24 bits wide */ +int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_MPORT_LOOKUP_OUT_LEN); + MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_MPORT_LOOKUP_IN_LEN); + size_t outlen; + int rc; + + MCDI_SET_DWORD(inbuf, MAE_MPORT_LOOKUP_IN_MPORT_SELECTOR, selector); + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_MPORT_LOOKUP, inbuf, sizeof(inbuf), + outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + if (outlen < sizeof(outbuf)) + return -EIO; + *id = MCDI_DWORD(outbuf, MAE_MPORT_LOOKUP_OUT_MPORT_ID); + return 0; +} diff --git a/drivers/net/ethernet/sfc/mae.h b/drivers/net/ethernet/sfc/mae.h new file mode 100644 index 000000000000..27e69e8a54b6 --- /dev/null +++ b/drivers/net/ethernet/sfc/mae.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2019 Solarflare Communications Inc. + * Copyright 2020-2022 Xilinx Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#ifndef EF100_MAE_H +#define EF100_MAE_H +/* MCDI interface for the ef100 Match-Action Engine */ + +#include "net_driver.h" + +void efx_mae_mport_vf(struct efx_nic *efx, u32 vf_id, u32 *out); + +int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id); + +#endif /* EF100_MAE_H */ diff --git a/drivers/net/ethernet/sfc/mcdi.c b/drivers/net/ethernet/sfc/mcdi.c index 3225fe64c397..af338208eae9 100644 --- a/drivers/net/ethernet/sfc/mcdi.c +++ b/drivers/net/ethernet/sfc/mcdi.c @@ -2129,6 +2129,52 @@ fail: return rc; } +/* Failure to read a privilege mask is never fatal, because we can always + * carry on as though we didn't have the privilege we were interested in. + * So use efx_mcdi_rpc_quiet(). + */ +int efx_mcdi_get_privilege_mask(struct efx_nic *efx, u32 *mask) +{ + MCDI_DECLARE_BUF(fi_outbuf, MC_CMD_GET_FUNCTION_INFO_OUT_LEN); + MCDI_DECLARE_BUF(pm_inbuf, MC_CMD_PRIVILEGE_MASK_IN_LEN); + MCDI_DECLARE_BUF(pm_outbuf, MC_CMD_PRIVILEGE_MASK_OUT_LEN); + size_t outlen; + u16 pf, vf; + int rc; + + if (!efx || !mask) + return -EINVAL; + + /* Get our function number */ + rc = efx_mcdi_rpc_quiet(efx, MC_CMD_GET_FUNCTION_INFO, NULL, 0, + fi_outbuf, MC_CMD_GET_FUNCTION_INFO_OUT_LEN, + &outlen); + if (rc != 0) + return rc; + if (outlen < MC_CMD_GET_FUNCTION_INFO_OUT_LEN) + return -EIO; + + pf = MCDI_DWORD(fi_outbuf, GET_FUNCTION_INFO_OUT_PF); + vf = MCDI_DWORD(fi_outbuf, GET_FUNCTION_INFO_OUT_VF); + + MCDI_POPULATE_DWORD_2(pm_inbuf, PRIVILEGE_MASK_IN_FUNCTION, + PRIVILEGE_MASK_IN_FUNCTION_PF, pf, + PRIVILEGE_MASK_IN_FUNCTION_VF, vf); + + rc = efx_mcdi_rpc_quiet(efx, MC_CMD_PRIVILEGE_MASK, + pm_inbuf, sizeof(pm_inbuf), + pm_outbuf, sizeof(pm_outbuf), &outlen); + + if (rc != 0) + return rc; + if (outlen < MC_CMD_PRIVILEGE_MASK_OUT_LEN) + return -EIO; + + *mask = MCDI_DWORD(pm_outbuf, PRIVILEGE_MASK_OUT_OLD_MASK); + + return 0; +} + #ifdef CONFIG_SFC_MTD #define EFX_MCDI_NVRAM_LEN_MAX 128 diff --git a/drivers/net/ethernet/sfc/mcdi.h b/drivers/net/ethernet/sfc/mcdi.h index 69c2924a147c..f74f6ce8b27d 100644 --- a/drivers/net/ethernet/sfc/mcdi.h +++ b/drivers/net/ethernet/sfc/mcdi.h @@ -366,6 +366,7 @@ int efx_mcdi_set_workaround(struct efx_nic *efx, u32 type, bool enabled, unsigned int *flags); int efx_mcdi_get_workarounds(struct efx_nic *efx, unsigned int *impl_out, unsigned int *enabled_out); +int efx_mcdi_get_privilege_mask(struct efx_nic *efx, u32 *mask); #ifdef CONFIG_SFC_MCDI_MON int efx_mcdi_mon_probe(struct efx_nic *efx); diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h index 7984f6f84a3c..cd297e19cddc 100644 --- a/drivers/net/ethernet/sfc/mcdi_pcol.h +++ b/drivers/net/ethernet/sfc/mcdi_pcol.h @@ -165,138 +165,8 @@ #define FSE_AZ_EV_CODE_MCDI_EVRESPONSE 0xc -/* Operation not permitted. */ -#define MC_CMD_ERR_EPERM 1 -/* Non-existent command target */ -#define MC_CMD_ERR_ENOENT 2 -/* assert() has killed the MC */ -#define MC_CMD_ERR_EINTR 4 -/* I/O failure */ -#define MC_CMD_ERR_EIO 5 -/* Already exists */ -#define MC_CMD_ERR_EEXIST 6 -/* Try again */ -#define MC_CMD_ERR_EAGAIN 11 -/* Out of memory */ -#define MC_CMD_ERR_ENOMEM 12 -/* Caller does not hold required locks */ -#define MC_CMD_ERR_EACCES 13 -/* Resource is currently unavailable (e.g. lock contention) */ -#define MC_CMD_ERR_EBUSY 16 -/* No such device */ -#define MC_CMD_ERR_ENODEV 19 -/* Invalid argument to target */ -#define MC_CMD_ERR_EINVAL 22 -/* Broken pipe */ -#define MC_CMD_ERR_EPIPE 32 -/* Read-only */ -#define MC_CMD_ERR_EROFS 30 -/* Out of range */ -#define MC_CMD_ERR_ERANGE 34 -/* Non-recursive resource is already acquired */ -#define MC_CMD_ERR_EDEADLK 35 -/* Operation not implemented */ -#define MC_CMD_ERR_ENOSYS 38 -/* Operation timed out */ -#define MC_CMD_ERR_ETIME 62 -/* Link has been severed */ -#define MC_CMD_ERR_ENOLINK 67 -/* Protocol error */ -#define MC_CMD_ERR_EPROTO 71 -/* Operation not supported */ -#define MC_CMD_ERR_ENOTSUP 95 -/* Address not available */ -#define MC_CMD_ERR_EADDRNOTAVAIL 99 -/* Not connected */ -#define MC_CMD_ERR_ENOTCONN 107 -/* Operation already in progress */ -#define MC_CMD_ERR_EALREADY 114 - -/* Resource allocation failed. */ -#define MC_CMD_ERR_ALLOC_FAIL 0x1000 -/* V-adaptor not found. */ -#define MC_CMD_ERR_NO_VADAPTOR 0x1001 -/* EVB port not found. */ -#define MC_CMD_ERR_NO_EVB_PORT 0x1002 -/* V-switch not found. */ -#define MC_CMD_ERR_NO_VSWITCH 0x1003 -/* Too many VLAN tags. */ -#define MC_CMD_ERR_VLAN_LIMIT 0x1004 -/* Bad PCI function number. */ -#define MC_CMD_ERR_BAD_PCI_FUNC 0x1005 -/* Invalid VLAN mode. */ -#define MC_CMD_ERR_BAD_VLAN_MODE 0x1006 -/* Invalid v-switch type. */ -#define MC_CMD_ERR_BAD_VSWITCH_TYPE 0x1007 -/* Invalid v-port type. */ -#define MC_CMD_ERR_BAD_VPORT_TYPE 0x1008 -/* MAC address exists. */ -#define MC_CMD_ERR_MAC_EXIST 0x1009 -/* Slave core not present */ -#define MC_CMD_ERR_SLAVE_NOT_PRESENT 0x100a -/* The datapath is disabled. */ -#define MC_CMD_ERR_DATAPATH_DISABLED 0x100b -/* The requesting client is not a function */ -#define MC_CMD_ERR_CLIENT_NOT_FN 0x100c -/* The requested operation might require the - command to be passed between MCs, and the - transport doesn't support that. Should - only ever been seen over the UART. */ -#define MC_CMD_ERR_TRANSPORT_NOPROXY 0x100d -/* VLAN tag(s) exists */ -#define MC_CMD_ERR_VLAN_EXIST 0x100e -/* No MAC address assigned to an EVB port */ -#define MC_CMD_ERR_NO_MAC_ADDR 0x100f -/* Notifies the driver that the request has been relayed - * to an admin function for authorization. The driver should - * wait for a PROXY_RESPONSE event and then resend its request. - * This error code is followed by a 32-bit handle that - * helps matching it with the respective PROXY_RESPONSE event. */ -#define MC_CMD_ERR_PROXY_PENDING 0x1010 -#define MC_CMD_ERR_PROXY_PENDING_HANDLE_OFST 4 -/* The request cannot be passed for authorization because - * another request from the same function is currently being - * authorized. The drvier should try again later. */ -#define MC_CMD_ERR_PROXY_INPROGRESS 0x1011 -/* Returned by MC_CMD_PROXY_COMPLETE if the caller is not the function - * that has enabled proxying or BLOCK_INDEX points to a function that - * doesn't await an authorization. */ -#define MC_CMD_ERR_PROXY_UNEXPECTED 0x1012 -/* This code is currently only used internally in FW. Its meaning is that - * an operation failed due to lack of SR-IOV privilege. - * Normally it is translated to EPERM by send_cmd_err(), - * but it may also be used to trigger some special mechanism - * for handling such case, e.g. to relay the failed request - * to a designated admin function for authorization. */ -#define MC_CMD_ERR_NO_PRIVILEGE 0x1013 -/* Workaround 26807 could not be turned on/off because some functions - * have already installed filters. See the comment at - * MC_CMD_WORKAROUND_BUG26807. - * May also returned for other operations such as sub-variant switching. */ -#define MC_CMD_ERR_FILTERS_PRESENT 0x1014 -/* The clock whose frequency you've attempted to set - * doesn't exist on this NIC */ -#define MC_CMD_ERR_NO_CLOCK 0x1015 -/* Returned by MC_CMD_TESTASSERT if the action that should - * have caused an assertion failed to do so. */ -#define MC_CMD_ERR_UNREACHABLE 0x1016 -/* This command needs to be processed in the background but there were no - * resources to do so. Send it again after a command has completed. */ -#define MC_CMD_ERR_QUEUE_FULL 0x1017 -/* The operation could not be completed because the PCIe link has gone - * away. This error code is never expected to be returned over the TLP - * transport. */ -#define MC_CMD_ERR_NO_PCIE 0x1018 -/* The operation could not be completed because the datapath has gone - * away. This is distinct from MC_CMD_ERR_DATAPATH_DISABLED in that the - * datapath absence may be temporary*/ -#define MC_CMD_ERR_NO_DATAPATH 0x1019 -/* The operation could not complete because some VIs are allocated */ -#define MC_CMD_ERR_VIS_PRESENT 0x101a -/* The operation could not complete because some PIO buffers are allocated */ -#define MC_CMD_ERR_PIOBUFS_PRESENT 0x101b - #define MC_CMD_ERR_CODE_OFST 0 +#define MC_CMD_ERR_PROXY_PENDING_HANDLE_OFST 4 /* We define 8 "escape" commands to allow for command number space extension */ @@ -365,10 +235,857 @@ */ #define MC_CMD_ERR_ARG_OFST 4 -/* No space */ -#define MC_CMD_ERR_ENOSPC 28 - -/* MCDI_EVENT structuredef */ +/* MC_CMD_ERR enum: Public MCDI error codes. Error codes that correspond to + * POSIX errnos should use the same numeric values that linux does. Error codes + * specific to Solarflare firmware should use values in the range 0x1000 - + * 0x10ff. The range 0x2000 - 0x20ff is reserved for private error codes (see + * MC_CMD_ERR_PRIV below). + */ +/* enum: Operation not permitted. */ +#define MC_CMD_ERR_EPERM 0x1 +/* enum: Non-existent command target */ +#define MC_CMD_ERR_ENOENT 0x2 +/* enum: assert() has killed the MC */ +#define MC_CMD_ERR_EINTR 0x4 +/* enum: I/O failure */ +#define MC_CMD_ERR_EIO 0x5 +/* enum: Already exists */ +#define MC_CMD_ERR_EEXIST 0x6 +/* enum: Try again */ +#define MC_CMD_ERR_EAGAIN 0xb +/* enum: Out of memory */ +#define MC_CMD_ERR_ENOMEM 0xc +/* enum: Caller does not hold required locks */ +#define MC_CMD_ERR_EACCES 0xd +/* enum: Resource is currently unavailable (e.g. lock contention) */ +#define MC_CMD_ERR_EBUSY 0x10 +/* enum: No such device */ +#define MC_CMD_ERR_ENODEV 0x13 +/* enum: Invalid argument to target */ +#define MC_CMD_ERR_EINVAL 0x16 +/* enum: No space */ +#define MC_CMD_ERR_ENOSPC 0x1c +/* enum: Read-only */ +#define MC_CMD_ERR_EROFS 0x1e +/* enum: Broken pipe */ +#define MC_CMD_ERR_EPIPE 0x20 +/* enum: Out of range */ +#define MC_CMD_ERR_ERANGE 0x22 +/* enum: Non-recursive resource is already acquired */ +#define MC_CMD_ERR_EDEADLK 0x23 +/* enum: Operation not implemented */ +#define MC_CMD_ERR_ENOSYS 0x26 +/* enum: Operation timed out */ +#define MC_CMD_ERR_ETIME 0x3e +/* enum: Link has been severed */ +#define MC_CMD_ERR_ENOLINK 0x43 +/* enum: Protocol error */ +#define MC_CMD_ERR_EPROTO 0x47 +/* enum: Bad message */ +#define MC_CMD_ERR_EBADMSG 0x4a +/* enum: Operation not supported */ +#define MC_CMD_ERR_ENOTSUP 0x5f +/* enum: Address not available */ +#define MC_CMD_ERR_EADDRNOTAVAIL 0x63 +/* enum: Not connected */ +#define MC_CMD_ERR_ENOTCONN 0x6b +/* enum: Operation already in progress */ +#define MC_CMD_ERR_EALREADY 0x72 +/* enum: Stale handle. The handle references a resource that no longer exists. + */ +#define MC_CMD_ERR_ESTALE 0x74 +/* enum: Resource allocation failed. */ +#define MC_CMD_ERR_ALLOC_FAIL 0x1000 +/* enum: V-adaptor not found. */ +#define MC_CMD_ERR_NO_VADAPTOR 0x1001 +/* enum: EVB port not found. */ +#define MC_CMD_ERR_NO_EVB_PORT 0x1002 +/* enum: V-switch not found. */ +#define MC_CMD_ERR_NO_VSWITCH 0x1003 +/* enum: Too many VLAN tags. */ +#define MC_CMD_ERR_VLAN_LIMIT 0x1004 +/* enum: Bad PCI function number. */ +#define MC_CMD_ERR_BAD_PCI_FUNC 0x1005 +/* enum: Invalid VLAN mode. */ +#define MC_CMD_ERR_BAD_VLAN_MODE 0x1006 +/* enum: Invalid v-switch type. */ +#define MC_CMD_ERR_BAD_VSWITCH_TYPE 0x1007 +/* enum: Invalid v-port type. */ +#define MC_CMD_ERR_BAD_VPORT_TYPE 0x1008 +/* enum: MAC address exists. */ +#define MC_CMD_ERR_MAC_EXIST 0x1009 +/* enum: Slave core not present */ +#define MC_CMD_ERR_SLAVE_NOT_PRESENT 0x100a +/* enum: The datapath is disabled. */ +#define MC_CMD_ERR_DATAPATH_DISABLED 0x100b +/* enum: The requesting client is not a function */ +#define MC_CMD_ERR_CLIENT_NOT_FN 0x100c +/* enum: The requested operation might require the command to be passed between + * MCs, and thetransport doesn't support that. Should only ever been seen over + * the UART. + */ +#define MC_CMD_ERR_TRANSPORT_NOPROXY 0x100d +/* enum: VLAN tag(s) exists */ +#define MC_CMD_ERR_VLAN_EXIST 0x100e +/* enum: No MAC address assigned to an EVB port */ +#define MC_CMD_ERR_NO_MAC_ADDR 0x100f +/* enum: Notifies the driver that the request has been relayed to an admin + * function for authorization. The driver should wait for a PROXY_RESPONSE + * event and then resend its request. This error code is followed by a 32-bit + * handle that helps matching it with the respective PROXY_RESPONSE event. + */ +#define MC_CMD_ERR_PROXY_PENDING 0x1010 +/* enum: The request cannot be passed for authorization because another request + * from the same function is currently being authorized. The drvier should try + * again later. + */ +#define MC_CMD_ERR_PROXY_INPROGRESS 0x1011 +/* enum: Returned by MC_CMD_PROXY_COMPLETE if the caller is not the function + * that has enabled proxying or BLOCK_INDEX points to a function that doesn't + * await an authorization. + */ +#define MC_CMD_ERR_PROXY_UNEXPECTED 0x1012 +/* enum: This code is currently only used internally in FW. Its meaning is that + * an operation failed due to lack of SR-IOV privilege. Normally it is + * translated to EPERM by send_cmd_err(), but it may also be used to trigger + * some special mechanism for handling such case, e.g. to relay the failed + * request to a designated admin function for authorization. + */ +#define MC_CMD_ERR_NO_PRIVILEGE 0x1013 +/* enum: Workaround 26807 could not be turned on/off because some functions + * have already installed filters. See the comment at + * MC_CMD_WORKAROUND_BUG26807. May also returned for other operations such as + * sub-variant switching. + */ +#define MC_CMD_ERR_FILTERS_PRESENT 0x1014 +/* enum: The clock whose frequency you've attempted to set set doesn't exist on + * this NIC + */ +#define MC_CMD_ERR_NO_CLOCK 0x1015 +/* enum: Returned by MC_CMD_TESTASSERT if the action that should have caused an + * assertion failed to do so. + */ +#define MC_CMD_ERR_UNREACHABLE 0x1016 +/* enum: This command needs to be processed in the background but there were no + * resources to do so. Send it again after a command has completed. + */ +#define MC_CMD_ERR_QUEUE_FULL 0x1017 +/* enum: The operation could not be completed because the PCIe link has gone + * away. This error code is never expected to be returned over the TLP + * transport. + */ +#define MC_CMD_ERR_NO_PCIE 0x1018 +/* enum: The operation could not be completed because the datapath has gone + * away. This is distinct from MC_CMD_ERR_DATAPATH_DISABLED in that the + * datapath absence may be temporary + */ +#define MC_CMD_ERR_NO_DATAPATH 0x1019 +/* enum: The operation could not complete because some VIs are allocated */ +#define MC_CMD_ERR_VIS_PRESENT 0x101a +/* enum: The operation could not complete because some PIO buffers are + * allocated + */ +#define MC_CMD_ERR_PIOBUFS_PRESENT 0x101b + +/* MC_CMD_RESOURCE_SPECIFIER enum */ +/* enum: Any */ +#define MC_CMD_RESOURCE_INSTANCE_ANY 0xffffffff +#define MC_CMD_RESOURCE_INSTANCE_NONE 0xfffffffe /* enum */ + +/* MC_CMD_FPGA_FLASH_INDEX enum */ +#define MC_CMD_FPGA_FLASH_PRIMARY 0x0 /* enum */ +#define MC_CMD_FPGA_FLASH_SECONDARY 0x1 /* enum */ + +/* MC_CMD_EXTERNAL_MAE_LINK_MODE enum */ +/* enum: Legacy mode as described in XN-200039-TC. */ +#define MC_CMD_EXTERNAL_MAE_LINK_MODE_LEGACY 0x0 +/* enum: Switchdev mode as described in XN-200039-TC. */ +#define MC_CMD_EXTERNAL_MAE_LINK_MODE_SWITCHDEV 0x1 +/* enum: Bootstrap mode as described in XN-200039-TC. */ +#define MC_CMD_EXTERNAL_MAE_LINK_MODE_BOOTSTRAP 0x2 +/* enum: Link-mode change is in-progress as described in XN-200039-TC. */ +#define MC_CMD_EXTERNAL_MAE_LINK_MODE_PENDING 0xf + +/* PCIE_INTERFACE enum: From EF100 onwards, SFC products can have multiple PCIe + * interfaces. There is a need to refer to interfaces explicitly from drivers + * (for example, a management driver on one interface administering a function + * on another interface). This enumeration provides stable identifiers to all + * interfaces present on a product. Product documentation will specify which + * interfaces exist and their associated identifier. In general, drivers, + * should not assign special meanings to specific values. Instead, behaviour + * should be determined by NIC configuration, which will identify interfaces + * where appropriate. + */ +/* enum: Primary host interfaces. Typically (i.e. for all known SFC products) + * the interface exposed on the edge connector (or form factor equivalent). + */ +#define PCIE_INTERFACE_HOST_PRIMARY 0x0 +/* enum: Riverhead and keystone products have a second PCIe interface to which + * an on-NIC ARM module is expected to be connected. + */ +#define PCIE_INTERFACE_NIC_EMBEDDED 0x1 +/* enum: For MCDI commands issued over a PCIe interface, this value is + * translated into the interface over which the command was issued. Not + * meaningful for other MCDI transports. + */ +#define PCIE_INTERFACE_CALLER 0xffffffff + +/* MC_CLIENT_ID_SPECIFIER enum */ +/* enum: Equivalent to the caller's client ID */ +#define MC_CMD_CLIENT_ID_SELF 0xffffffff + +/* MAE_FIELD_SUPPORT_STATUS enum */ +/* enum: The NIC does not support this field. The driver must ensure that any + * mask associated with this field in a match rule is zeroed. The NIC may + * either reject requests with an invalid mask for such a field, or may assume + * that the mask is zero. (This category only exists to describe behaviour for + * fields that a newer driver might know about but that older firmware does + * not. It is recommended that firmware report MAE_FIELD_FIELD_MATCH_NEVER for + * all match fields defined at the time of its compilation. If a driver see a + * field support status value that it does not recognise, it must treat that + * field as thought the field was reported as MAE_FIELD_SUPPORTED_MATCH_NEVER, + * and must never set a non-zero mask value for this field. + */ +#define MAE_FIELD_UNSUPPORTED 0x0 +/* enum: The NIC supports this field, but cannot use it in a match rule. The + * driver must ensure that any mask for such a field in a match rule is zeroed. + * The NIC will reject requests with an invalid mask for such a field. + */ +#define MAE_FIELD_SUPPORTED_MATCH_NEVER 0x1 +/* enum: The NIC supports this field, and must use it in all match rules. The + * driver must ensure that any mask for such a field is all ones. The NIC will + * reject requests with an invalid mask for such a field. + */ +#define MAE_FIELD_SUPPORTED_MATCH_ALWAYS 0x2 +/* enum: The NIC supports this field, and may optionally use it in match rules. + * The driver must ensure that any mask for such a field is either all zeroes + * or all ones. The NIC will reject requests with an invalid mask for such a + * field. + */ +#define MAE_FIELD_SUPPORTED_MATCH_OPTIONAL 0x3 +/* enum: The NIC supports this field, and may optionally use it in match rules. + * The driver must ensure that any mask for such a field is either all zeroes + * or a consecutive set of ones following by all zeroes (starting from MSB). + * The NIC will reject requests with an invalid mask for such a field. + */ +#define MAE_FIELD_SUPPORTED_MATCH_PREFIX 0x4 +/* enum: The NIC supports this field, and may optionally use it in match rules. + * The driver may provide an arbitrary mask for such a field. + */ +#define MAE_FIELD_SUPPORTED_MATCH_MASK 0x5 + +/* MAE_CT_VNI_MODE enum: Controls the layout of the VNI input to the conntrack + * lookup. (Values are not arbitrary - constrained by table access ABI.) + */ +/* enum: The VNI input to the conntrack lookup will be zero. */ +#define MAE_CT_VNI_MODE_ZERO 0x0 +/* enum: The VNI input to the conntrack lookup will be the VNI (VXLAN/Geneve) + * or VSID (NVGRE) field from the packet. + */ +#define MAE_CT_VNI_MODE_VNI 0x1 +/* enum: The VNI input to the conntrack lookup will be the VLAN ID from the + * outermost VLAN tag (in bottom 12 bits; top 12 bits zero). + */ +#define MAE_CT_VNI_MODE_1VLAN 0x2 +/* enum: The VNI input to the conntrack lookup will be the VLAN IDs from both + * VLAN tags (outermost in bottom 12 bits, innermost in top 12 bits). + */ +#define MAE_CT_VNI_MODE_2VLAN 0x3 + +/* MAE_FIELD enum: NB: this enum shares namespace with the support status enum. + */ +/* enum: Source mport upon entering the MAE. */ +#define MAE_FIELD_INGRESS_PORT 0x0 +#define MAE_FIELD_MARK 0x1 /* enum */ +/* enum: Table ID used in action rule. Initially zero, can be changed in action + * rule response. + */ +#define MAE_FIELD_RECIRC_ID 0x2 +#define MAE_FIELD_IS_IP_FRAG 0x3 /* enum */ +#define MAE_FIELD_DO_CT 0x4 /* enum */ +#define MAE_FIELD_CT_HIT 0x5 /* enum */ +/* enum: Undefined unless CT_HIT=1. */ +#define MAE_FIELD_CT_MARK 0x6 +/* enum: Undefined unless DO_CT=1. */ +#define MAE_FIELD_CT_DOMAIN 0x7 +/* enum: Undefined unless CT_HIT=1. */ +#define MAE_FIELD_CT_PRIVATE_FLAGS 0x8 +/* enum: 1 if the packet ingressed the NIC from one of the MACs, else 0. */ +#define MAE_FIELD_IS_FROM_NETWORK 0x9 +/* enum: 1 if the packet has 1 or more VLAN tags, else 0. */ +#define MAE_FIELD_HAS_OVLAN 0xa +/* enum: 1 if the packet has 2 or more VLAN tags, else 0. */ +#define MAE_FIELD_HAS_IVLAN 0xb +/* enum: 1 if the outer packet has 1 or more VLAN tags, else 0; only present + * when encap + */ +#define MAE_FIELD_ENC_HAS_OVLAN 0xc +/* enum: 1 if the outer packet has 2 or more VLAN tags, else 0; only present + * when encap + */ +#define MAE_FIELD_ENC_HAS_IVLAN 0xd +/* enum: Packet is IP fragment */ +#define MAE_FIELD_ENC_IP_FRAG 0xe +#define MAE_FIELD_ETHER_TYPE 0x21 /* enum */ +#define MAE_FIELD_VLAN0_TCI 0x22 /* enum */ +#define MAE_FIELD_VLAN0_PROTO 0x23 /* enum */ +#define MAE_FIELD_VLAN1_TCI 0x24 /* enum */ +#define MAE_FIELD_VLAN1_PROTO 0x25 /* enum */ +/* enum: Inner when encap */ +#define MAE_FIELD_ETH_SADDR 0x28 +/* enum: Inner when encap */ +#define MAE_FIELD_ETH_DADDR 0x29 +/* enum: Inner when encap. NB: IPv4 and IPv6 fields are mutually exclusive. */ +#define MAE_FIELD_SRC_IP4 0x2a +/* enum: Inner when encap */ +#define MAE_FIELD_SRC_IP6 0x2b +/* enum: Inner when encap */ +#define MAE_FIELD_DST_IP4 0x2c +/* enum: Inner when encap */ +#define MAE_FIELD_DST_IP6 0x2d +/* enum: Inner when encap */ +#define MAE_FIELD_IP_PROTO 0x2e +/* enum: Inner when encap */ +#define MAE_FIELD_IP_TOS 0x2f +/* enum: Inner when encap */ +#define MAE_FIELD_IP_TTL 0x30 +/* enum: Inner when encap TODO: how this is defined? The raw flags + + * frag_offset from the packet, or some derived value more amenable to ternary + * matching? TODO: there was a proposal for driver-allocation fields. The + * driver would provide some instruction for how to extract given field values, + * and would be given a field id in return. It could then use that field id in + * its matches. This feels like it would be extremely hard to implement in + * hardware, but I mention it for completeness. + */ +#define MAE_FIELD_IP_FLAGS 0x31 +/* enum: Ports (UDP, TCP) Inner when encap */ +#define MAE_FIELD_L4_SPORT 0x32 +/* enum: Ports (UDP, TCP) Inner when encap */ +#define MAE_FIELD_L4_DPORT 0x33 +/* enum: Inner when encap */ +#define MAE_FIELD_TCP_FLAGS 0x34 +/* enum: TCP packet with any of SYN, FIN or RST flag set */ +#define MAE_FIELD_TCP_SYN_FIN_RST 0x35 +/* enum: Packet is IP fragment with fragment offset 0 */ +#define MAE_FIELD_IP_FIRST_FRAG 0x36 +/* enum: The type of encapsulated used for this packet. Value as per + * ENCAP_TYPE_*. + */ +#define MAE_FIELD_ENCAP_TYPE 0x3f +/* enum: The ID of the outer rule that marked this packet as encapsulated. + * Useful for implicitly matching on outer fields. + */ +#define MAE_FIELD_OUTER_RULE_ID 0x40 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_ETHER_TYPE 0x41 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_VLAN0_TCI 0x42 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_VLAN0_PROTO 0x43 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_VLAN1_TCI 0x44 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_VLAN1_PROTO 0x45 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_ETH_SADDR 0x48 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_ETH_DADDR 0x49 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_SRC_IP4 0x4a +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_SRC_IP6 0x4b +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_DST_IP4 0x4c +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_DST_IP6 0x4d +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_IP_PROTO 0x4e +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_IP_TOS 0x4f +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_IP_TTL 0x50 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_IP_FLAGS 0x51 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_L4_SPORT 0x52 +/* enum: Outer; only present when encap */ +#define MAE_FIELD_ENC_L4_DPORT 0x53 +/* enum: VNI (when VXLAN or GENEVE) VSID (when NVGRE) Bottom 24 bits of Key + * (when L2GRE) Outer; only present when encap + */ +#define MAE_FIELD_ENC_VNET_ID 0x54 + +/* MAE_MCDI_ENCAP_TYPE enum: Encapsulation type. Defines how the payload will + * be parsed to an inner frame. Other values are reserved. Unknown values + * should be treated same as NONE. (Values are not arbitrary - constrained by + * table access ABI.) + */ +#define MAE_MCDI_ENCAP_TYPE_NONE 0x0 /* enum */ +/* enum: Don't assume enum aligns with support bitmask... */ +#define MAE_MCDI_ENCAP_TYPE_VXLAN 0x1 +#define MAE_MCDI_ENCAP_TYPE_NVGRE 0x2 /* enum */ +#define MAE_MCDI_ENCAP_TYPE_GENEVE 0x3 /* enum */ +#define MAE_MCDI_ENCAP_TYPE_L2GRE 0x4 /* enum */ + +/* MAE_MPORT_END enum: Selects which end of the logical link identified by an + * MPORT_SELECTOR is targeted by an operation. + */ +/* enum: Selects the port on the MAE virtual switch */ +#define MAE_MPORT_END_MAE 0x1 +/* enum: Selects the virtual NIC plugged into the MAE switch */ +#define MAE_MPORT_END_VNIC 0x2 + +/* MAE_COUNTER_TYPE enum: The datapath maintains several sets of counters, each + * being associated with a different table. Note that the same counter ID may + * be allocated by different counter blocks, so e.g. AR counter 42 is different + * from CT counter 42. Generation counts are also type-specific. This value is + * also present in the header of streaming counter packets, in the IDENTIFIER + * field (see packetiser packet format definitions). + */ +/* enum: Action Rule counters - can be referenced in AR response. */ +#define MAE_COUNTER_TYPE_AR 0x0 +/* enum: Conntrack counters - can be referenced in CT response. */ +#define MAE_COUNTER_TYPE_CT 0x1 +/* enum: Outer Rule counters - can be referenced in OR response. */ +#define MAE_COUNTER_TYPE_OR 0x2 + +/* TABLE_ID enum: Unique IDs for tables. The 32-bit ID values have been + * structured with bits [31:24] reserved (0), [23:16] indicating which major + * block the tables belongs to (0=VNIC TX, none currently; 1=MAE; 2=VNIC RX), + * [15:8] a unique ID within the block, and [7:0] reserved for future + * variations of the same table. (All of the tables currently defined within + * the streaming engines are listed here, but this does not imply that they are + * all supported - MC_CMD_TABLE_LIST returns the list of actually supported + * tables.) + */ +/* enum: Outer_Rule_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_OUTER_RULE_TABLE 0x10000 +/* enum: Outer_Rule_No_CT_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_OUTER_RULE_NO_CT_TABLE 0x10100 +/* enum: Mgmt_Filter_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_MGMT_FILTER_TABLE 0x10200 +/* enum: Conntrack_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_CONNTRACK_TABLE 0x10300 +/* enum: Action_Rule_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_ACTION_RULE_TABLE 0x10400 +/* enum: Mgroup_Default_Action_Set_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_MGROUP_DEFAULT_ACTION_SET_TABLE 0x10500 +/* enum: Encap_Hdr_Part1_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_ENCAP_HDR_PART1_TABLE 0x10600 +/* enum: Encap_Hdr_Part2_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_ENCAP_HDR_PART2_TABLE 0x10700 +/* enum: Replace_Src_MAC_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_REPLACE_SRC_MAC_TABLE 0x10800 +/* enum: Replace_Dst_MAC_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_REPLACE_DST_MAC_TABLE 0x10900 +/* enum: Dst_Mport_VC_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_DST_MPORT_VC_TABLE 0x10a00 +/* enum: LACP_LAG_Config_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_LACP_LAG_CONFIG_TABLE 0x10b00 +/* enum: LACP_Balance_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_LACP_BALANCE_TABLE 0x10c00 +/* enum: Dst_Mport_Host_Chan_Table in the MAE - refer to SF-123102-TC. */ +#define TABLE_ID_DST_MPORT_HOST_CHAN_TABLE 0x10d00 +/* enum: VNIC_Rx_Encap_Table in VNIC Rx - refer to SF-123102-TC. */ +#define TABLE_ID_VNIC_RX_ENCAP_TABLE 0x20000 +/* enum: Steering_Table in VNIC Rx - refer to SF-123102-TC. */ +#define TABLE_ID_STEERING_TABLE 0x20100 +/* enum: RSS_Context_Table in VNIC Rx - refer to SF-123102-TC. */ +#define TABLE_ID_RSS_CONTEXT_TABLE 0x20200 +/* enum: Indirection_Table in VNIC Rx - refer to SF-123102-TC. */ +#define TABLE_ID_INDIRECTION_TABLE 0x20300 + +/* TABLE_COMPRESSED_VLAN enum: Compressed VLAN TPID as used by some field + * types; can be calculated by (((ether_type_msb >> 2) & 0x4) ^ 0x4) | + * (ether_type_msb & 0x3); + */ +#define TABLE_COMPRESSED_VLAN_TPID_8100 0x5 /* enum */ +#define TABLE_COMPRESSED_VLAN_TPID_88A8 0x4 /* enum */ +#define TABLE_COMPRESSED_VLAN_TPID_9100 0x1 /* enum */ +#define TABLE_COMPRESSED_VLAN_TPID_9200 0x2 /* enum */ +#define TABLE_COMPRESSED_VLAN_TPID_9300 0x3 /* enum */ + +/* TABLE_NAT_DIR enum: NAT direction. */ +#define TABLE_NAT_DIR_SOURCE 0x0 /* enum */ +#define TABLE_NAT_DIR_DEST 0x1 /* enum */ + +/* TABLE_RSS_KEY_MODE enum: Defines how the value for Toeplitz hashing for RSS + * is constructed as a concatenation (indicated here by "++") of packet header + * fields. + */ +/* enum: IP src addr ++ IP dst addr */ +#define TABLE_RSS_KEY_MODE_SA_DA 0x0 +/* enum: IP src addr ++ IP dst addr ++ TCP/UDP src port ++ TCP/UDP dst port */ +#define TABLE_RSS_KEY_MODE_SA_DA_SP_DP 0x1 +/* enum: IP src addr */ +#define TABLE_RSS_KEY_MODE_SA 0x2 +/* enum: IP dst addr */ +#define TABLE_RSS_KEY_MODE_DA 0x3 +/* enum: IP src addr ++ TCP/UDP src port */ +#define TABLE_RSS_KEY_MODE_SA_SP 0x4 +/* enum: IP dest addr ++ TCP dest port */ +#define TABLE_RSS_KEY_MODE_DA_DP 0x5 +/* enum: Nothing (produces input of 0, resulting in output hash of 0) */ +#define TABLE_RSS_KEY_MODE_NONE 0x7 + +/* TABLE_RSS_SPREAD_MODE enum: RSS spreading mode. */ +/* enum: RSS uses Indirection_Table lookup. */ +#define TABLE_RSS_SPREAD_MODE_INDIRECTION 0x0 +/* enum: RSS uses even spreading calculation. */ +#define TABLE_RSS_SPREAD_MODE_EVEN 0x1 + +/* TABLE_FIELD_ID enum: Unique IDs for fields. Related concepts have been + * loosely grouped together into blocks with gaps for expansion, but the values + * are arbitrary. Field IDs are not specific to particular tables, and in some + * cases this sharing means that they are not used with the exact names of the + * corresponding table definitions in SF-123102-TC; however, the mapping should + * still be clear. The intent is that a list of fields, with their associated + * bit widths and semantics version code, unambiguously defines the semantics + * of the fields in a key or response. (Again, this list includes all of the + * fields currently defined within the streaming engines, but only a subset may + * actually be used by the supported list of tables.) + */ +/* enum: May appear multiple times within a key or response, and indicates that + * the field is unused and should be set to 0 (or masked out if permitted by + * the MASK_VALUE for this field). + */ +#define TABLE_FIELD_ID_UNUSED 0x0 +/* enum: Source m-port (a full m-port label). */ +#define TABLE_FIELD_ID_SRC_MPORT 0x1 +/* enum: Destination m-port (a full m-port label). */ +#define TABLE_FIELD_ID_DST_MPORT 0x2 +/* enum: Source m-group ID. */ +#define TABLE_FIELD_ID_SRC_MGROUP_ID 0x3 +/* enum: Physical network port ID (or m-port ID; same thing, for physical + * network ports). + */ +#define TABLE_FIELD_ID_NETWORK_PORT_ID 0x4 +/* enum: True if packet arrived via network port, false if it arrived via host. + */ +#define TABLE_FIELD_ID_IS_FROM_NETWORK 0x5 +/* enum: Full virtual channel from capsule header. */ +#define TABLE_FIELD_ID_CH_VC 0x6 +/* enum: Low bits of virtual channel from capsule header. */ +#define TABLE_FIELD_ID_CH_VC_LOW 0x7 +/* enum: User mark value in metadata and packet prefix. */ +#define TABLE_FIELD_ID_USER_MARK 0x8 +/* enum: User flag value in metadata and packet prefix. */ +#define TABLE_FIELD_ID_USER_FLAG 0x9 +/* enum: Counter ID associated with a response. All-bits-1 is a null value to + * suppress counting. + */ +#define TABLE_FIELD_ID_COUNTER_ID 0xa +/* enum: Discriminator which may be set by plugins in some lookup keys; this + * allows plugins to make a reinterpretation of packet fields in these keys + * without clashing with the normal interpretation. + */ +#define TABLE_FIELD_ID_DISCRIM 0xb +/* enum: Destination MAC address. The mapping from bytes in a frame to the + * 48-bit value for this field is in network order, i.e. a MAC address of + * AA:BB:CC:DD:EE:FF becomes a 48-bit value of 0xAABBCCDDEEFF. + */ +#define TABLE_FIELD_ID_DST_MAC 0x14 +/* enum: Source MAC address (see notes for DST_MAC). */ +#define TABLE_FIELD_ID_SRC_MAC 0x15 +/* enum: Outer VLAN tag TPID, compressed to an enumeration. */ +#define TABLE_FIELD_ID_OVLAN_TPID_COMPRESSED 0x16 +/* enum: Full outer VLAN tag TCI (16 bits). */ +#define TABLE_FIELD_ID_OVLAN 0x17 +/* enum: Outer VLAN ID (least significant 12 bits of full 16-bit TCI) only. */ +#define TABLE_FIELD_ID_OVLAN_VID 0x18 +/* enum: Inner VLAN tag TPID, compressed to an enumeration. */ +#define TABLE_FIELD_ID_IVLAN_TPID_COMPRESSED 0x19 +/* enum: Full inner VLAN tag TCI (16 bits). */ +#define TABLE_FIELD_ID_IVLAN 0x1a +/* enum: Inner VLAN ID (least significant 12 bits of full 16-bit TCI) only. */ +#define TABLE_FIELD_ID_IVLAN_VID 0x1b +/* enum: Ethertype. */ +#define TABLE_FIELD_ID_ETHER_TYPE 0x1c +/* enum: Source IP address, either IPv4 or IPv6. The mapping from bytes in a + * frame to the 128-bit value for this field is in network order, with IPv4 + * addresses assumed to have 12 bytes of trailing zeroes. i.e. the IPv6 address + * [2345::6789:ABCD] is 0x2345000000000000000000006789ABCD; the IPv4 address + * 192.168.1.2 is 0xC0A80102000000000000000000000000. + */ +#define TABLE_FIELD_ID_SRC_IP 0x1d +/* enum: Destination IP address (see notes for SRC_IP). */ +#define TABLE_FIELD_ID_DST_IP 0x1e +/* enum: IPv4 Type-of-Service or IPv6 Traffic Class field. */ +#define TABLE_FIELD_ID_IP_TOS 0x1f +/* enum: IP Protocol. */ +#define TABLE_FIELD_ID_IP_PROTO 0x20 +/* enum: Layer 4 source port. */ +#define TABLE_FIELD_ID_SRC_PORT 0x21 +/* enum: Layer 4 destination port. */ +#define TABLE_FIELD_ID_DST_PORT 0x22 +/* enum: TCP flags. */ +#define TABLE_FIELD_ID_TCP_FLAGS 0x23 +/* enum: Virtual Network Identifier (VXLAN) or Virtual Session ID (NVGRE). */ +#define TABLE_FIELD_ID_VNI 0x24 +/* enum: True if packet has any tunnel encapsulation header. */ +#define TABLE_FIELD_ID_HAS_ENCAP 0x32 +/* enum: True if encap header has an outer VLAN tag. */ +#define TABLE_FIELD_ID_HAS_ENC_OVLAN 0x33 +/* enum: True if encap header has an inner VLAN tag. */ +#define TABLE_FIELD_ID_HAS_ENC_IVLAN 0x34 +/* enum: True if encap header is some sort of IP. */ +#define TABLE_FIELD_ID_HAS_ENC_IP 0x35 +/* enum: True if encap header is specifically IPv4. */ +#define TABLE_FIELD_ID_HAS_ENC_IP4 0x36 +/* enum: True if encap header is UDP. */ +#define TABLE_FIELD_ID_HAS_ENC_UDP 0x37 +/* enum: True if only/inner frame has an outer VLAN tag. */ +#define TABLE_FIELD_ID_HAS_OVLAN 0x38 +/* enum: True if only/inner frame has an inner VLAN tag. */ +#define TABLE_FIELD_ID_HAS_IVLAN 0x39 +/* enum: True if only/inner frame is some sort of IP. */ +#define TABLE_FIELD_ID_HAS_IP 0x3a +/* enum: True if only/inner frame has a recognised L4 IP protocol (TCP or UDP). + */ +#define TABLE_FIELD_ID_HAS_L4 0x3b +/* enum: True if only/inner frame is an IP fragment. */ +#define TABLE_FIELD_ID_IP_FRAG 0x3c +/* enum: True if only/inner frame is the first IP fragment (fragment offset 0). + */ +#define TABLE_FIELD_ID_IP_FIRST_FRAG 0x3d +/* enum: True if only/inner frame has an IP Time-To-Live of <= 1. (Note: the + * implementation calls this "ip_ttl_is_one" but does in fact match packets + * with TTL=0 - which we shouldn't be seeing! - as well.) + */ +#define TABLE_FIELD_ID_IP_TTL_LE_ONE 0x3e +/* enum: True if only/inner frame has any of TCP SYN, FIN or RST flags set. */ +#define TABLE_FIELD_ID_TCP_INTERESTING_FLAGS 0x3f +/* enum: Plugin channel selection. */ +#define TABLE_FIELD_ID_RDP_PL_CHAN 0x50 +/* enum: Enable update of CH_ROUTE_RDP_C_PL route bit. */ +#define TABLE_FIELD_ID_RDP_C_PL_EN 0x51 +/* enum: New value of CH_ROUTE_RDP_C_PL route bit. */ +#define TABLE_FIELD_ID_RDP_C_PL 0x52 +/* enum: Enable update of CH_ROUTE_RDP_D_PL route bit. */ +#define TABLE_FIELD_ID_RDP_D_PL_EN 0x53 +/* enum: New value of CH_ROUTE_RDP_D_PL route bit. */ +#define TABLE_FIELD_ID_RDP_D_PL 0x54 +/* enum: Enable update of CH_ROUTE_RDP_OUT_HOST_CHAN route bit. */ +#define TABLE_FIELD_ID_RDP_OUT_HOST_CHAN_EN 0x55 +/* enum: New value of CH_ROUTE_RDP_OUT_HOST_CHAN route bit. */ +#define TABLE_FIELD_ID_RDP_OUT_HOST_CHAN 0x56 +/* enum: Recirculation ID for lookup sequences with two action rule lookups. */ +#define TABLE_FIELD_ID_RECIRC_ID 0x64 +/* enum: Domain ID passed to conntrack and action rule lookups. */ +#define TABLE_FIELD_ID_DOMAIN 0x65 +/* enum: Construction mode for encap_tunnel_id - see MAE_CT_VNI_MODE enum. */ +#define TABLE_FIELD_ID_CT_VNI_MODE 0x66 +/* enum: True to inhibit conntrack lookup if TCP SYN, FIN or RST flag is set. + */ +#define TABLE_FIELD_ID_CT_TCP_FLAGS_INHIBIT 0x67 +/* enum: True to do conntrack lookups for IPv4 TCP packets. */ +#define TABLE_FIELD_ID_DO_CT_IP4_TCP 0x68 +/* enum: True to do conntrack lookups for IPv4 UDP packets. */ +#define TABLE_FIELD_ID_DO_CT_IP4_UDP 0x69 +/* enum: True to do conntrack lookups for IPv6 TCP packets. */ +#define TABLE_FIELD_ID_DO_CT_IP6_TCP 0x6a +/* enum: True to do conntrack lookups for IPv6 UDP packets. */ +#define TABLE_FIELD_ID_DO_CT_IP6_UDP 0x6b +/* enum: Outer rule identifier. */ +#define TABLE_FIELD_ID_OUTER_RULE_ID 0x6c +/* enum: Encapsulation type - see MAE_MCDI_ENCAP_TYPE enum. */ +#define TABLE_FIELD_ID_ENCAP_TYPE 0x6d +/* enum: Encap tunnel ID for conntrack lookups from VNI, VLAN tag(s), or 0, + * depending on CT_VNI_MODE. + */ +#define TABLE_FIELD_ID_ENCAP_TUNNEL_ID 0x78 +/* enum: A conntrack entry identifier, passed to plugins. */ +#define TABLE_FIELD_ID_CT_ENTRY_ID 0x79 +/* enum: Either source or destination NAT replacement port. */ +#define TABLE_FIELD_ID_NAT_PORT 0x7a +/* enum: Either source or destination NAT replacement IPv4 address. Note that + * this is specifically an IPv4 address (IPv6 is not supported for NAT), with + * byte mapped to a 32-bit value in network order, i.e. the IPv4 address + * 192.168.1.2 is the value 0xC0A80102. + */ +#define TABLE_FIELD_ID_NAT_IP 0x7b +/* enum: NAT direction: 0=>source, 1=>destination. */ +#define TABLE_FIELD_ID_NAT_DIR 0x7c +/* enum: Conntrack mark value, passed to action rule lookup. Note that this is + * not related to the "user mark" in the metadata / packet prefix. + */ +#define TABLE_FIELD_ID_CT_MARK 0x7d +/* enum: Private flags for conntrack, passed to action rule lookup. */ +#define TABLE_FIELD_ID_CT_PRIV_FLAGS 0x7e +/* enum: True if the conntrack lookup resulted in a hit. */ +#define TABLE_FIELD_ID_CT_HIT 0x7f +/* enum: True to suppress delivery when source and destination m-ports match. + */ +#define TABLE_FIELD_ID_SUPPRESS_SELF_DELIVERY 0x8c +/* enum: True to perform tunnel decapsulation. */ +#define TABLE_FIELD_ID_DO_DECAP 0x8d +/* enum: True to copy outer frame DSCP to inner on decap. */ +#define TABLE_FIELD_ID_DECAP_DSCP_COPY 0x8e +/* enum: True to map outer frame ECN to inner on decap, by RFC 6040 rules. */ +#define TABLE_FIELD_ID_DECAP_ECN_RFC6040 0x8f +/* enum: True to replace DSCP field. */ +#define TABLE_FIELD_ID_DO_REPLACE_DSCP 0x90 +/* enum: True to replace ECN field. */ +#define TABLE_FIELD_ID_DO_REPLACE_ECN 0x91 +/* enum: True to decrement IP Time-To-Live. */ +#define TABLE_FIELD_ID_DO_DECR_IP_TTL 0x92 +/* enum: True to replace source MAC address. */ +#define TABLE_FIELD_ID_DO_SRC_MAC 0x93 +/* enum: True to replace destination MAC address. */ +#define TABLE_FIELD_ID_DO_DST_MAC 0x94 +/* enum: Number of VLAN tags to pop. Valid values are 0, 1, or 2. */ +#define TABLE_FIELD_ID_DO_VLAN_POP 0x95 +/* enum: Number of VLANs tags to push. Valid values are 0, 1, or 2. */ +#define TABLE_FIELD_ID_DO_VLAN_PUSH 0x96 +/* enum: True to count this packet. */ +#define TABLE_FIELD_ID_DO_COUNT 0x97 +/* enum: True to perform tunnel encapsulation. */ +#define TABLE_FIELD_ID_DO_ENCAP 0x98 +/* enum: True to copy inner frame DSCP to outer on encap. */ +#define TABLE_FIELD_ID_ENCAP_DSCP_COPY 0x99 +/* enum: True to copy inner frame ECN to outer on encap. */ +#define TABLE_FIELD_ID_ENCAP_ECN_COPY 0x9a +/* enum: True to deliver the packet (otherwise it is dropped). */ +#define TABLE_FIELD_ID_DO_DELIVER 0x9b +/* enum: True to set the user flag in the metadata. */ +#define TABLE_FIELD_ID_DO_FLAG 0x9c +/* enum: True to update the user mark in the metadata. */ +#define TABLE_FIELD_ID_DO_MARK 0x9d +/* enum: True to override the capsule virtual channel for network deliveries. + */ +#define TABLE_FIELD_ID_DO_SET_NET_CHAN 0x9e +/* enum: True to override the reported source m-port for host deliveries. */ +#define TABLE_FIELD_ID_DO_SET_SRC_MPORT 0x9f +/* enum: Encap header ID for DO_ENCAP, indexing Encap_Hdr_Part1/2_Table. */ +#define TABLE_FIELD_ID_ENCAP_HDR_ID 0xaa +/* enum: New DSCP value for DO_REPLACE_DSCP. */ +#define TABLE_FIELD_ID_DSCP_VALUE 0xab +/* enum: If DO_REPLACE_ECN is set, the new value for the ECN field. If + * DO_REPLACE_ECN is not set, ECN_CONTROL[0] and ECN_CONTROL[1] are set to + * request remapping of ECT0 and ECT1 ECN codepoints respectively to CE. + */ +#define TABLE_FIELD_ID_ECN_CONTROL 0xac +/* enum: Source MAC ID for DO_SRC_MAC, indexing Replace_Src_MAC_Table. */ +#define TABLE_FIELD_ID_SRC_MAC_ID 0xad +/* enum: Destination MAC ID for DO_DST_MAC, indexing Replace_Dst_MAC_Table. */ +#define TABLE_FIELD_ID_DST_MAC_ID 0xae +/* enum: Parameter for either DO_SET_NET_CHAN (only bottom 6 bits used in this + * case) or DO_SET_SRC_MPORT. + */ +#define TABLE_FIELD_ID_REPORTED_SRC_MPORT_OR_NET_CHAN 0xaf +/* enum: 64-byte chunk of added encapsulation header. */ +#define TABLE_FIELD_ID_CHUNK64 0xb4 +/* enum: 32-byte chunk of added encapsulation header. */ +#define TABLE_FIELD_ID_CHUNK32 0xb5 +/* enum: 16-byte chunk of added encapsulation header. */ +#define TABLE_FIELD_ID_CHUNK16 0xb6 +/* enum: 8-byte chunk of added encapsulation header. */ +#define TABLE_FIELD_ID_CHUNK8 0xb7 +/* enum: 4-byte chunk of added encapsulation header. */ +#define TABLE_FIELD_ID_CHUNK4 0xb8 +/* enum: 2-byte chunk of added encapsulation header. */ +#define TABLE_FIELD_ID_CHUNK2 0xb9 +/* enum: Added encapsulation header length in words. */ +#define TABLE_FIELD_ID_HDR_LEN_W 0xba +/* enum: Static value for layer 2/3 LACP hash of the encapsulation header. */ +#define TABLE_FIELD_ID_ENC_LACP_HASH_L23 0xbb +/* enum: Static value for layer 4 LACP hash of the encapsulation header. */ +#define TABLE_FIELD_ID_ENC_LACP_HASH_L4 0xbc +/* enum: True to use the static ENC_LACP_HASH values for the encap header + * instead of the calculated values for the inner frame when delivering a newly + * encapsulated packet to a LAG m-port. + */ +#define TABLE_FIELD_ID_USE_ENC_LACP_HASHES 0xbd +/* enum: True to trigger conntrack from first action rule lookup (AR=>CT=>AR + * sequence). + */ +#define TABLE_FIELD_ID_DO_CT 0xc8 +/* enum: True to perform NAT using parameters from conntrack lookup response. + */ +#define TABLE_FIELD_ID_DO_NAT 0xc9 +/* enum: True to trigger recirculated action rule lookup (AR=>AR sequence). */ +#define TABLE_FIELD_ID_DO_RECIRC 0xca +/* enum: Next action set payload ID for replay. The null value is all-1-bits. + */ +#define TABLE_FIELD_ID_NEXT_ACTION_SET_PAYLOAD 0xcb +/* enum: Next action set row ID for replay. The null value is all-1-bits. */ +#define TABLE_FIELD_ID_NEXT_ACTION_SET_ROW 0xcc +/* enum: Action set payload ID for additional delivery to management CPU. The + * null value is all-1-bits. + */ +#define TABLE_FIELD_ID_MC_ACTION_SET_PAYLOAD 0xcd +/* enum: Action set row ID for additional delivery to management CPU. The null + * value is all-1-bits. + */ +#define TABLE_FIELD_ID_MC_ACTION_SET_ROW 0xce +/* enum: True to include layer 4 in LACP hash on delivery to a LAG m-port. */ +#define TABLE_FIELD_ID_LACP_INC_L4 0xdc +/* enum: True to request that LACP is performed by a plugin. */ +#define TABLE_FIELD_ID_LACP_PLUGIN 0xdd +/* enum: LACP_Balance_Table base address divided by 64. */ +#define TABLE_FIELD_ID_BAL_TBL_BASE_DIV64 0xde +/* enum: Length of balance table region: 0=>64, 1=>128, 2=>256. */ +#define TABLE_FIELD_ID_BAL_TBL_LEN_ID 0xdf +/* enum: UDP port to match for UDP-based encapsulations; required to be 0 for + * other encapsulation types. + */ +#define TABLE_FIELD_ID_UDP_PORT 0xe6 +/* enum: True to perform RSS based on outer fields rather than inner fields. */ +#define TABLE_FIELD_ID_RSS_ON_OUTER 0xe7 +/* enum: True to perform steering table lookup on outer fields rather than + * inner fields. + */ +#define TABLE_FIELD_ID_STEER_ON_OUTER 0xe8 +/* enum: Destination queue ID for host delivery. */ +#define TABLE_FIELD_ID_DST_QID 0xf0 +/* enum: True to drop this packet. */ +#define TABLE_FIELD_ID_DROP 0xf1 +/* enum: True to strip outer VLAN tag from this packet. */ +#define TABLE_FIELD_ID_VLAN_STRIP 0xf2 +/* enum: True to override the user mark field with the supplied USER_MARK, or + * false to bitwise-OR the USER_MARK into it. + */ +#define TABLE_FIELD_ID_MARK_OVERRIDE 0xf3 +/* enum: True to override the user flag field with the supplied USER_FLAG, or + * false to bitwise-OR the USER_FLAG into it. + */ +#define TABLE_FIELD_ID_FLAG_OVERRIDE 0xf4 +/* enum: RSS context ID, indexing the RSS_Context_Table. */ +#define TABLE_FIELD_ID_RSS_CTX_ID 0xfa +/* enum: True to enable RSS. */ +#define TABLE_FIELD_ID_RSS_EN 0xfb +/* enum: Toeplitz hash key. */ +#define TABLE_FIELD_ID_KEY 0xfc +/* enum: Key mode for IPv4 TCP packets - see TABLE_RSS_KEY_MODE enum. */ +#define TABLE_FIELD_ID_TCP_V4_KEY_MODE 0xfd +/* enum: Key mode for IPv6 TCP packets - see TABLE_RSS_KEY_MODE enum. */ +#define TABLE_FIELD_ID_TCP_V6_KEY_MODE 0xfe +/* enum: Key mode for IPv4 UDP packets - see TABLE_RSS_KEY_MODE enum. */ +#define TABLE_FIELD_ID_UDP_V4_KEY_MODE 0xff +/* enum: Key mode for IPv6 UDP packets - see TABLE_RSS_KEY_MODE enum. */ +#define TABLE_FIELD_ID_UDP_V6_KEY_MODE 0x100 +/* enum: Key mode for other IPv4 packets - see TABLE_RSS_KEY_MODE enum. */ +#define TABLE_FIELD_ID_OTHER_V4_KEY_MODE 0x101 +/* enum: Key mode for other IPv6 packets - see TABLE_RSS_KEY_MODE enum. */ +#define TABLE_FIELD_ID_OTHER_V6_KEY_MODE 0x102 +/* enum: Spreading mode - 0=>indirection; 1=>even. */ +#define TABLE_FIELD_ID_SPREAD_MODE 0x103 +/* enum: For indirection spreading mode, the base address of a region within + * the Indirection_Table. For even spreading mode, the number of queues to + * spread across (only values 1-255 are valid for this mode). + */ +#define TABLE_FIELD_ID_INDIR_TBL_BASE 0x104 +/* enum: For indirection spreading mode, identifies the length of a region + * within the Indirection_Table, where length = 32 << len_id. Must be set to 0 + * for even spreading mode. + */ +#define TABLE_FIELD_ID_INDIR_TBL_LEN_ID 0x105 +/* enum: An offset to be applied to the base destination queue ID. */ +#define TABLE_FIELD_ID_INDIR_OFFSET 0x106 + +/* MCDI_EVENT structuredef: The structure of an MCDI_EVENT on Siena/EF10/EF100 + * platforms + */ #define MCDI_EVENT_LEN 8 #define MCDI_EVENT_CONT_LBN 32 #define MCDI_EVENT_CONT_WIDTH 1 @@ -447,17 +1164,21 @@ #define MCDI_EVENT_TX_ERR_TYPE_OFST 0 #define MCDI_EVENT_TX_ERR_TYPE_LBN 12 #define MCDI_EVENT_TX_ERR_TYPE_WIDTH 4 -/* enum: Descriptor loader reported failure */ +/* enum: Descriptor loader reported failure. Specific to EF10-family NICs. */ #define MCDI_EVENT_TX_ERR_DL_FAIL 0x1 -/* enum: Descriptor ring empty and no EOP seen for packet */ +/* enum: Descriptor ring empty and no EOP seen for packet. Specific to + * EF10-family NICs + */ #define MCDI_EVENT_TX_ERR_NO_EOP 0x2 -/* enum: Overlength packet */ +/* enum: Overlength packet. Specific to EF10-family NICs. */ #define MCDI_EVENT_TX_ERR_2BIG 0x3 -/* enum: Malformed option descriptor */ +/* enum: Malformed option descriptor. Specific to EF10-family NICs. */ #define MCDI_EVENT_TX_BAD_OPTDESC 0x5 -/* enum: Option descriptor part way through a packet */ +/* enum: Option descriptor part way through a packet. Specific to EF10-family + * NICs. + */ #define MCDI_EVENT_TX_OPT_IN_PKT 0x8 -/* enum: DMA or PIO data access error */ +/* enum: DMA or PIO data access error. Specific to EF10-family NICs */ #define MCDI_EVENT_TX_ERR_BAD_DMA_OR_PIO 0x9 #define MCDI_EVENT_TX_ERR_INFO_OFST 0 #define MCDI_EVENT_TX_ERR_INFO_LBN 16 @@ -773,6 +1494,12 @@ * SF-122927-TC for details. */ #define MCDI_EVENT_CODE_DESC_PROXY_FUNC_DRIVER_ATTACH 0x26 +/* enum: Notification that the mport journal has changed since it was last read + * and updates can be read using the MC_CMD_MAE_MPORT_READ_JOURNAL command. The + * firmware may moderate the events so that an event is not sent for every + * change to the journal. + */ +#define MCDI_EVENT_CODE_MPORT_JOURNAL_CHANGE 0x27 /* enum: Artificial event generated by host and posted via MC for test * purposes. */ @@ -1070,7 +1797,13 @@ #define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_OFST 8 #define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_LEN 8 #define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_LO_OFST 8 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_LO_LEN 4 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_LO_LBN 64 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_LO_WIDTH 32 #define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_HI_OFST 12 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_HI_LEN 4 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_HI_LBN 96 +#define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_HI_WIDTH 32 #define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_MINNUM 1 #define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_MAXNUM 30 #define FCDI_EXTENDED_EVENT_PPS_TIMESTAMPS_MAXNUM_MCDI2 126 @@ -1482,12 +2215,24 @@ #define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_BUILD_TIMESTAMP_OFST 260 #define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_BUILD_TIMESTAMP_LEN 8 #define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_BUILD_TIMESTAMP_LO_OFST 260 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_BUILD_TIMESTAMP_LO_LEN 4 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_BUILD_TIMESTAMP_LO_LBN 2080 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_BUILD_TIMESTAMP_LO_WIDTH 32 #define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_BUILD_TIMESTAMP_HI_OFST 264 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_BUILD_TIMESTAMP_HI_LEN 4 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_BUILD_TIMESTAMP_HI_LBN 2112 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_BUILD_TIMESTAMP_HI_WIDTH 32 /* MC firmware version number */ #define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_VERSION_OFST 268 #define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_VERSION_LEN 8 #define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_VERSION_LO_OFST 268 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_VERSION_LO_LEN 4 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_VERSION_LO_LBN 2144 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_VERSION_LO_WIDTH 32 #define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_VERSION_HI_OFST 272 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_VERSION_HI_LEN 4 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_VERSION_HI_LBN 2176 +#define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_VERSION_HI_WIDTH 32 /* MC firmware security level */ #define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_SECURITY_LEVEL_OFST 276 #define MC_CMD_GET_ASSERTS_OUT_V3_MC_FW_SECURITY_LEVEL_LEN 4 @@ -1571,7 +2316,13 @@ #define MC_CMD_GET_VERSION_OUT_VERSION_OFST 24 #define MC_CMD_GET_VERSION_OUT_VERSION_LEN 8 #define MC_CMD_GET_VERSION_OUT_VERSION_LO_OFST 24 +#define MC_CMD_GET_VERSION_OUT_VERSION_LO_LEN 4 +#define MC_CMD_GET_VERSION_OUT_VERSION_LO_LBN 192 +#define MC_CMD_GET_VERSION_OUT_VERSION_LO_WIDTH 32 #define MC_CMD_GET_VERSION_OUT_VERSION_HI_OFST 28 +#define MC_CMD_GET_VERSION_OUT_VERSION_HI_LEN 4 +#define MC_CMD_GET_VERSION_OUT_VERSION_HI_LBN 224 +#define MC_CMD_GET_VERSION_OUT_VERSION_HI_WIDTH 32 /* MC_CMD_GET_VERSION_EXT_OUT msgresponse */ #define MC_CMD_GET_VERSION_EXT_OUT_LEN 48 @@ -1587,7 +2338,13 @@ #define MC_CMD_GET_VERSION_EXT_OUT_VERSION_OFST 24 #define MC_CMD_GET_VERSION_EXT_OUT_VERSION_LEN 8 #define MC_CMD_GET_VERSION_EXT_OUT_VERSION_LO_OFST 24 +#define MC_CMD_GET_VERSION_EXT_OUT_VERSION_LO_LEN 4 +#define MC_CMD_GET_VERSION_EXT_OUT_VERSION_LO_LBN 192 +#define MC_CMD_GET_VERSION_EXT_OUT_VERSION_LO_WIDTH 32 #define MC_CMD_GET_VERSION_EXT_OUT_VERSION_HI_OFST 28 +#define MC_CMD_GET_VERSION_EXT_OUT_VERSION_HI_LEN 4 +#define MC_CMD_GET_VERSION_EXT_OUT_VERSION_HI_LBN 224 +#define MC_CMD_GET_VERSION_EXT_OUT_VERSION_HI_WIDTH 32 /* extra info */ #define MC_CMD_GET_VERSION_EXT_OUT_EXTRA_OFST 32 #define MC_CMD_GET_VERSION_EXT_OUT_EXTRA_LEN 16 @@ -1611,7 +2368,13 @@ #define MC_CMD_GET_VERSION_V2_OUT_VERSION_OFST 24 #define MC_CMD_GET_VERSION_V2_OUT_VERSION_LEN 8 #define MC_CMD_GET_VERSION_V2_OUT_VERSION_LO_OFST 24 +#define MC_CMD_GET_VERSION_V2_OUT_VERSION_LO_LEN 4 +#define MC_CMD_GET_VERSION_V2_OUT_VERSION_LO_LBN 192 +#define MC_CMD_GET_VERSION_V2_OUT_VERSION_LO_WIDTH 32 #define MC_CMD_GET_VERSION_V2_OUT_VERSION_HI_OFST 28 +#define MC_CMD_GET_VERSION_V2_OUT_VERSION_HI_LEN 4 +#define MC_CMD_GET_VERSION_V2_OUT_VERSION_HI_LBN 224 +#define MC_CMD_GET_VERSION_V2_OUT_VERSION_HI_WIDTH 32 /* extra info */ #define MC_CMD_GET_VERSION_V2_OUT_EXTRA_OFST 32 #define MC_CMD_GET_VERSION_V2_OUT_EXTRA_LEN 16 @@ -1633,6 +2396,33 @@ #define MC_CMD_GET_VERSION_V2_OUT_BOARD_EXT_INFO_PRESENT_OFST 48 #define MC_CMD_GET_VERSION_V2_OUT_BOARD_EXT_INFO_PRESENT_LBN 4 #define MC_CMD_GET_VERSION_V2_OUT_BOARD_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V2_OUT_DATAPATH_HW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V2_OUT_DATAPATH_HW_VERSION_PRESENT_LBN 5 +#define MC_CMD_GET_VERSION_V2_OUT_DATAPATH_HW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V2_OUT_DATAPATH_FW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V2_OUT_DATAPATH_FW_VERSION_PRESENT_LBN 6 +#define MC_CMD_GET_VERSION_V2_OUT_DATAPATH_FW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_BOOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_BOOT_VERSION_PRESENT_LBN 7 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_BOOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_UBOOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_UBOOT_VERSION_PRESENT_LBN 8 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_UBOOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_LBN 9 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_LBN 10 +#define MC_CMD_GET_VERSION_V2_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V2_OUT_SUCFW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V2_OUT_SUCFW_VERSION_PRESENT_LBN 11 +#define MC_CMD_GET_VERSION_V2_OUT_SUCFW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V2_OUT_BOARD_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V2_OUT_BOARD_VERSION_PRESENT_LBN 12 +#define MC_CMD_GET_VERSION_V2_OUT_BOARD_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V2_OUT_BUNDLE_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V2_OUT_BUNDLE_VERSION_PRESENT_LBN 13 +#define MC_CMD_GET_VERSION_V2_OUT_BUNDLE_VERSION_PRESENT_WIDTH 1 /* MC firmware unique build ID (as binary SHA-1 value) */ #define MC_CMD_GET_VERSION_V2_OUT_MCFW_BUILD_ID_OFST 52 #define MC_CMD_GET_VERSION_V2_OUT_MCFW_BUILD_ID_LEN 20 @@ -1650,7 +2440,13 @@ #define MC_CMD_GET_VERSION_V2_OUT_SUCFW_BUILD_DATE_OFST 156 #define MC_CMD_GET_VERSION_V2_OUT_SUCFW_BUILD_DATE_LEN 8 #define MC_CMD_GET_VERSION_V2_OUT_SUCFW_BUILD_DATE_LO_OFST 156 +#define MC_CMD_GET_VERSION_V2_OUT_SUCFW_BUILD_DATE_LO_LEN 4 +#define MC_CMD_GET_VERSION_V2_OUT_SUCFW_BUILD_DATE_LO_LBN 1248 +#define MC_CMD_GET_VERSION_V2_OUT_SUCFW_BUILD_DATE_LO_WIDTH 32 #define MC_CMD_GET_VERSION_V2_OUT_SUCFW_BUILD_DATE_HI_OFST 160 +#define MC_CMD_GET_VERSION_V2_OUT_SUCFW_BUILD_DATE_HI_LEN 4 +#define MC_CMD_GET_VERSION_V2_OUT_SUCFW_BUILD_DATE_HI_LBN 1280 +#define MC_CMD_GET_VERSION_V2_OUT_SUCFW_BUILD_DATE_HI_WIDTH 32 /* The ID of the SUC chip. This is specific to the platform but typically * indicates family, memory sizes etc. See SF-116728-SW for further details. */ @@ -1664,7 +2460,13 @@ #define MC_CMD_GET_VERSION_V2_OUT_CMCFW_BUILD_DATE_OFST 184 #define MC_CMD_GET_VERSION_V2_OUT_CMCFW_BUILD_DATE_LEN 8 #define MC_CMD_GET_VERSION_V2_OUT_CMCFW_BUILD_DATE_LO_OFST 184 +#define MC_CMD_GET_VERSION_V2_OUT_CMCFW_BUILD_DATE_LO_LEN 4 +#define MC_CMD_GET_VERSION_V2_OUT_CMCFW_BUILD_DATE_LO_LBN 1472 +#define MC_CMD_GET_VERSION_V2_OUT_CMCFW_BUILD_DATE_LO_WIDTH 32 #define MC_CMD_GET_VERSION_V2_OUT_CMCFW_BUILD_DATE_HI_OFST 188 +#define MC_CMD_GET_VERSION_V2_OUT_CMCFW_BUILD_DATE_HI_LEN 4 +#define MC_CMD_GET_VERSION_V2_OUT_CMCFW_BUILD_DATE_HI_LBN 1504 +#define MC_CMD_GET_VERSION_V2_OUT_CMCFW_BUILD_DATE_HI_WIDTH 32 /* FPGA version as three numbers. On Riverhead based systems this field uses * the same encoding as hardware version ID registers (MC_FPGA_BUILD_HWRD_REG): * FPGA_VERSION[0]: x => Image H{x} FPGA_VERSION[1]: Revision letter (0 => A, 1 @@ -1686,6 +2488,489 @@ #define MC_CMD_GET_VERSION_V2_OUT_BOARD_SERIAL_OFST 240 #define MC_CMD_GET_VERSION_V2_OUT_BOARD_SERIAL_LEN 64 +/* MC_CMD_GET_VERSION_V3_OUT msgresponse: Extended response providing version + * information for all adapter components. For Riverhead based designs, base MC + * firmware version fields refer to NMC firmware, while CMC firmware data is in + * dedicated CMC fields. Flags indicate which data is present in the response + * (depending on which components exist on a particular adapter) + */ +#define MC_CMD_GET_VERSION_V3_OUT_LEN 328 +/* MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0 */ +/* MC_CMD_GET_VERSION_OUT_FIRMWARE_LEN 4 */ +/* Enum values, see field(s): */ +/* MC_CMD_GET_VERSION_V0_OUT/MC_CMD_GET_VERSION_OUT_FIRMWARE */ +#define MC_CMD_GET_VERSION_V3_OUT_PCOL_OFST 4 +#define MC_CMD_GET_VERSION_V3_OUT_PCOL_LEN 4 +/* 128bit mask of functions supported by the current firmware */ +#define MC_CMD_GET_VERSION_V3_OUT_SUPPORTED_FUNCS_OFST 8 +#define MC_CMD_GET_VERSION_V3_OUT_SUPPORTED_FUNCS_LEN 16 +#define MC_CMD_GET_VERSION_V3_OUT_VERSION_OFST 24 +#define MC_CMD_GET_VERSION_V3_OUT_VERSION_LEN 8 +#define MC_CMD_GET_VERSION_V3_OUT_VERSION_LO_OFST 24 +#define MC_CMD_GET_VERSION_V3_OUT_VERSION_LO_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_VERSION_LO_LBN 192 +#define MC_CMD_GET_VERSION_V3_OUT_VERSION_LO_WIDTH 32 +#define MC_CMD_GET_VERSION_V3_OUT_VERSION_HI_OFST 28 +#define MC_CMD_GET_VERSION_V3_OUT_VERSION_HI_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_VERSION_HI_LBN 224 +#define MC_CMD_GET_VERSION_V3_OUT_VERSION_HI_WIDTH 32 +/* extra info */ +#define MC_CMD_GET_VERSION_V3_OUT_EXTRA_OFST 32 +#define MC_CMD_GET_VERSION_V3_OUT_EXTRA_LEN 16 +/* Flags indicating which extended fields are valid */ +#define MC_CMD_GET_VERSION_V3_OUT_FLAGS_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_FLAGS_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_MCFW_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_MCFW_EXT_INFO_PRESENT_LBN 0 +#define MC_CMD_GET_VERSION_V3_OUT_MCFW_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_EXT_INFO_PRESENT_LBN 1 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_CMC_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_CMC_EXT_INFO_PRESENT_LBN 2 +#define MC_CMD_GET_VERSION_V3_OUT_CMC_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_FPGA_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_FPGA_EXT_INFO_PRESENT_LBN 3 +#define MC_CMD_GET_VERSION_V3_OUT_FPGA_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_EXT_INFO_PRESENT_LBN 4 +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_HW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_HW_VERSION_PRESENT_LBN 5 +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_HW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_FW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_FW_VERSION_PRESENT_LBN 6 +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_FW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_BOOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_BOOT_VERSION_PRESENT_LBN 7 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_BOOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_UBOOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_UBOOT_VERSION_PRESENT_LBN 8 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_UBOOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_LBN 9 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_LBN 10 +#define MC_CMD_GET_VERSION_V3_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_VERSION_PRESENT_LBN 11 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_VERSION_PRESENT_LBN 12 +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V3_OUT_BUNDLE_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V3_OUT_BUNDLE_VERSION_PRESENT_LBN 13 +#define MC_CMD_GET_VERSION_V3_OUT_BUNDLE_VERSION_PRESENT_WIDTH 1 +/* MC firmware unique build ID (as binary SHA-1 value) */ +#define MC_CMD_GET_VERSION_V3_OUT_MCFW_BUILD_ID_OFST 52 +#define MC_CMD_GET_VERSION_V3_OUT_MCFW_BUILD_ID_LEN 20 +/* MC firmware security level */ +#define MC_CMD_GET_VERSION_V3_OUT_MCFW_SECURITY_LEVEL_OFST 72 +#define MC_CMD_GET_VERSION_V3_OUT_MCFW_SECURITY_LEVEL_LEN 4 +/* MC firmware build name (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V3_OUT_MCFW_BUILD_NAME_OFST 76 +#define MC_CMD_GET_VERSION_V3_OUT_MCFW_BUILD_NAME_LEN 64 +/* The SUC firmware version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_VERSION_OFST 140 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_VERSION_NUM 4 +/* SUC firmware build date (as 64-bit Unix timestamp) */ +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_BUILD_DATE_OFST 156 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_BUILD_DATE_LEN 8 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_BUILD_DATE_LO_OFST 156 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_BUILD_DATE_LO_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_BUILD_DATE_LO_LBN 1248 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_BUILD_DATE_LO_WIDTH 32 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_BUILD_DATE_HI_OFST 160 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_BUILD_DATE_HI_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_BUILD_DATE_HI_LBN 1280 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_BUILD_DATE_HI_WIDTH 32 +/* The ID of the SUC chip. This is specific to the platform but typically + * indicates family, memory sizes etc. See SF-116728-SW for further details. + */ +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_CHIP_ID_OFST 164 +#define MC_CMD_GET_VERSION_V3_OUT_SUCFW_CHIP_ID_LEN 4 +/* The CMC firmware version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_VERSION_OFST 168 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_VERSION_NUM 4 +/* CMC firmware build date (as 64-bit Unix timestamp) */ +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_BUILD_DATE_OFST 184 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_BUILD_DATE_LEN 8 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_BUILD_DATE_LO_OFST 184 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_BUILD_DATE_LO_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_BUILD_DATE_LO_LBN 1472 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_BUILD_DATE_LO_WIDTH 32 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_BUILD_DATE_HI_OFST 188 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_BUILD_DATE_HI_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_BUILD_DATE_HI_LBN 1504 +#define MC_CMD_GET_VERSION_V3_OUT_CMCFW_BUILD_DATE_HI_WIDTH 32 +/* FPGA version as three numbers. On Riverhead based systems this field uses + * the same encoding as hardware version ID registers (MC_FPGA_BUILD_HWRD_REG): + * FPGA_VERSION[0]: x => Image H{x} FPGA_VERSION[1]: Revision letter (0 => A, 1 + * => B, ...) FPGA_VERSION[2]: Sub-revision number + */ +#define MC_CMD_GET_VERSION_V3_OUT_FPGA_VERSION_OFST 192 +#define MC_CMD_GET_VERSION_V3_OUT_FPGA_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_FPGA_VERSION_NUM 3 +/* Extra FPGA revision information (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V3_OUT_FPGA_EXTRA_OFST 204 +#define MC_CMD_GET_VERSION_V3_OUT_FPGA_EXTRA_LEN 16 +/* Board name / adapter model (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_NAME_OFST 220 +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_NAME_LEN 16 +/* Board revision number */ +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_REVISION_OFST 236 +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_REVISION_LEN 4 +/* Board serial number (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_SERIAL_OFST 240 +#define MC_CMD_GET_VERSION_V3_OUT_BOARD_SERIAL_LEN 64 +/* The version of the datapath hardware design as three number - a.b.c */ +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_HW_VERSION_OFST 304 +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_HW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_HW_VERSION_NUM 3 +/* The version of the firmware library used to control the datapath as three + * number - a.b.c + */ +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_FW_VERSION_OFST 316 +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_FW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V3_OUT_DATAPATH_FW_VERSION_NUM 3 + +/* MC_CMD_GET_VERSION_V4_OUT msgresponse: Extended response providing SoC + * version information + */ +#define MC_CMD_GET_VERSION_V4_OUT_LEN 392 +/* MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0 */ +/* MC_CMD_GET_VERSION_OUT_FIRMWARE_LEN 4 */ +/* Enum values, see field(s): */ +/* MC_CMD_GET_VERSION_V0_OUT/MC_CMD_GET_VERSION_OUT_FIRMWARE */ +#define MC_CMD_GET_VERSION_V4_OUT_PCOL_OFST 4 +#define MC_CMD_GET_VERSION_V4_OUT_PCOL_LEN 4 +/* 128bit mask of functions supported by the current firmware */ +#define MC_CMD_GET_VERSION_V4_OUT_SUPPORTED_FUNCS_OFST 8 +#define MC_CMD_GET_VERSION_V4_OUT_SUPPORTED_FUNCS_LEN 16 +#define MC_CMD_GET_VERSION_V4_OUT_VERSION_OFST 24 +#define MC_CMD_GET_VERSION_V4_OUT_VERSION_LEN 8 +#define MC_CMD_GET_VERSION_V4_OUT_VERSION_LO_OFST 24 +#define MC_CMD_GET_VERSION_V4_OUT_VERSION_LO_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_VERSION_LO_LBN 192 +#define MC_CMD_GET_VERSION_V4_OUT_VERSION_LO_WIDTH 32 +#define MC_CMD_GET_VERSION_V4_OUT_VERSION_HI_OFST 28 +#define MC_CMD_GET_VERSION_V4_OUT_VERSION_HI_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_VERSION_HI_LBN 224 +#define MC_CMD_GET_VERSION_V4_OUT_VERSION_HI_WIDTH 32 +/* extra info */ +#define MC_CMD_GET_VERSION_V4_OUT_EXTRA_OFST 32 +#define MC_CMD_GET_VERSION_V4_OUT_EXTRA_LEN 16 +/* Flags indicating which extended fields are valid */ +#define MC_CMD_GET_VERSION_V4_OUT_FLAGS_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_FLAGS_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_MCFW_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_MCFW_EXT_INFO_PRESENT_LBN 0 +#define MC_CMD_GET_VERSION_V4_OUT_MCFW_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_EXT_INFO_PRESENT_LBN 1 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_CMC_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_CMC_EXT_INFO_PRESENT_LBN 2 +#define MC_CMD_GET_VERSION_V4_OUT_CMC_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_FPGA_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_FPGA_EXT_INFO_PRESENT_LBN 3 +#define MC_CMD_GET_VERSION_V4_OUT_FPGA_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_EXT_INFO_PRESENT_LBN 4 +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_HW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_HW_VERSION_PRESENT_LBN 5 +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_HW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_FW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_FW_VERSION_PRESENT_LBN 6 +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_FW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_BOOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_BOOT_VERSION_PRESENT_LBN 7 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_BOOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_UBOOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_UBOOT_VERSION_PRESENT_LBN 8 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_UBOOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_LBN 9 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_LBN 10 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_VERSION_PRESENT_LBN 11 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_VERSION_PRESENT_LBN 12 +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V4_OUT_BUNDLE_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V4_OUT_BUNDLE_VERSION_PRESENT_LBN 13 +#define MC_CMD_GET_VERSION_V4_OUT_BUNDLE_VERSION_PRESENT_WIDTH 1 +/* MC firmware unique build ID (as binary SHA-1 value) */ +#define MC_CMD_GET_VERSION_V4_OUT_MCFW_BUILD_ID_OFST 52 +#define MC_CMD_GET_VERSION_V4_OUT_MCFW_BUILD_ID_LEN 20 +/* MC firmware security level */ +#define MC_CMD_GET_VERSION_V4_OUT_MCFW_SECURITY_LEVEL_OFST 72 +#define MC_CMD_GET_VERSION_V4_OUT_MCFW_SECURITY_LEVEL_LEN 4 +/* MC firmware build name (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V4_OUT_MCFW_BUILD_NAME_OFST 76 +#define MC_CMD_GET_VERSION_V4_OUT_MCFW_BUILD_NAME_LEN 64 +/* The SUC firmware version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_VERSION_OFST 140 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_VERSION_NUM 4 +/* SUC firmware build date (as 64-bit Unix timestamp) */ +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_BUILD_DATE_OFST 156 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_BUILD_DATE_LEN 8 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_BUILD_DATE_LO_OFST 156 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_BUILD_DATE_LO_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_BUILD_DATE_LO_LBN 1248 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_BUILD_DATE_LO_WIDTH 32 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_BUILD_DATE_HI_OFST 160 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_BUILD_DATE_HI_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_BUILD_DATE_HI_LBN 1280 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_BUILD_DATE_HI_WIDTH 32 +/* The ID of the SUC chip. This is specific to the platform but typically + * indicates family, memory sizes etc. See SF-116728-SW for further details. + */ +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_CHIP_ID_OFST 164 +#define MC_CMD_GET_VERSION_V4_OUT_SUCFW_CHIP_ID_LEN 4 +/* The CMC firmware version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_VERSION_OFST 168 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_VERSION_NUM 4 +/* CMC firmware build date (as 64-bit Unix timestamp) */ +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_BUILD_DATE_OFST 184 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_BUILD_DATE_LEN 8 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_BUILD_DATE_LO_OFST 184 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_BUILD_DATE_LO_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_BUILD_DATE_LO_LBN 1472 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_BUILD_DATE_LO_WIDTH 32 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_BUILD_DATE_HI_OFST 188 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_BUILD_DATE_HI_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_BUILD_DATE_HI_LBN 1504 +#define MC_CMD_GET_VERSION_V4_OUT_CMCFW_BUILD_DATE_HI_WIDTH 32 +/* FPGA version as three numbers. On Riverhead based systems this field uses + * the same encoding as hardware version ID registers (MC_FPGA_BUILD_HWRD_REG): + * FPGA_VERSION[0]: x => Image H{x} FPGA_VERSION[1]: Revision letter (0 => A, 1 + * => B, ...) FPGA_VERSION[2]: Sub-revision number + */ +#define MC_CMD_GET_VERSION_V4_OUT_FPGA_VERSION_OFST 192 +#define MC_CMD_GET_VERSION_V4_OUT_FPGA_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_FPGA_VERSION_NUM 3 +/* Extra FPGA revision information (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V4_OUT_FPGA_EXTRA_OFST 204 +#define MC_CMD_GET_VERSION_V4_OUT_FPGA_EXTRA_LEN 16 +/* Board name / adapter model (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_NAME_OFST 220 +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_NAME_LEN 16 +/* Board revision number */ +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_REVISION_OFST 236 +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_REVISION_LEN 4 +/* Board serial number (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_SERIAL_OFST 240 +#define MC_CMD_GET_VERSION_V4_OUT_BOARD_SERIAL_LEN 64 +/* The version of the datapath hardware design as three number - a.b.c */ +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_HW_VERSION_OFST 304 +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_HW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_HW_VERSION_NUM 3 +/* The version of the firmware library used to control the datapath as three + * number - a.b.c + */ +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_FW_VERSION_OFST 316 +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_FW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_DATAPATH_FW_VERSION_NUM 3 +/* The SOC boot version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V4_OUT_SOC_BOOT_VERSION_OFST 328 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_BOOT_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_BOOT_VERSION_NUM 4 +/* The SOC uboot version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V4_OUT_SOC_UBOOT_VERSION_OFST 344 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_UBOOT_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_UBOOT_VERSION_NUM 4 +/* The SOC main rootfs version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V4_OUT_SOC_MAIN_ROOTFS_VERSION_OFST 360 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_MAIN_ROOTFS_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_MAIN_ROOTFS_VERSION_NUM 4 +/* The SOC recovery buildroot version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V4_OUT_SOC_RECOVERY_BUILDROOT_VERSION_OFST 376 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_RECOVERY_BUILDROOT_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V4_OUT_SOC_RECOVERY_BUILDROOT_VERSION_NUM 4 + +/* MC_CMD_GET_VERSION_V5_OUT msgresponse: Extended response providing bundle + * and board version information + */ +#define MC_CMD_GET_VERSION_V5_OUT_LEN 424 +/* MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0 */ +/* MC_CMD_GET_VERSION_OUT_FIRMWARE_LEN 4 */ +/* Enum values, see field(s): */ +/* MC_CMD_GET_VERSION_V0_OUT/MC_CMD_GET_VERSION_OUT_FIRMWARE */ +#define MC_CMD_GET_VERSION_V5_OUT_PCOL_OFST 4 +#define MC_CMD_GET_VERSION_V5_OUT_PCOL_LEN 4 +/* 128bit mask of functions supported by the current firmware */ +#define MC_CMD_GET_VERSION_V5_OUT_SUPPORTED_FUNCS_OFST 8 +#define MC_CMD_GET_VERSION_V5_OUT_SUPPORTED_FUNCS_LEN 16 +#define MC_CMD_GET_VERSION_V5_OUT_VERSION_OFST 24 +#define MC_CMD_GET_VERSION_V5_OUT_VERSION_LEN 8 +#define MC_CMD_GET_VERSION_V5_OUT_VERSION_LO_OFST 24 +#define MC_CMD_GET_VERSION_V5_OUT_VERSION_LO_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_VERSION_LO_LBN 192 +#define MC_CMD_GET_VERSION_V5_OUT_VERSION_LO_WIDTH 32 +#define MC_CMD_GET_VERSION_V5_OUT_VERSION_HI_OFST 28 +#define MC_CMD_GET_VERSION_V5_OUT_VERSION_HI_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_VERSION_HI_LBN 224 +#define MC_CMD_GET_VERSION_V5_OUT_VERSION_HI_WIDTH 32 +/* extra info */ +#define MC_CMD_GET_VERSION_V5_OUT_EXTRA_OFST 32 +#define MC_CMD_GET_VERSION_V5_OUT_EXTRA_LEN 16 +/* Flags indicating which extended fields are valid */ +#define MC_CMD_GET_VERSION_V5_OUT_FLAGS_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_FLAGS_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_MCFW_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_MCFW_EXT_INFO_PRESENT_LBN 0 +#define MC_CMD_GET_VERSION_V5_OUT_MCFW_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_EXT_INFO_PRESENT_LBN 1 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_CMC_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_CMC_EXT_INFO_PRESENT_LBN 2 +#define MC_CMD_GET_VERSION_V5_OUT_CMC_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_FPGA_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_FPGA_EXT_INFO_PRESENT_LBN 3 +#define MC_CMD_GET_VERSION_V5_OUT_FPGA_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_EXT_INFO_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_EXT_INFO_PRESENT_LBN 4 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_EXT_INFO_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_HW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_HW_VERSION_PRESENT_LBN 5 +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_HW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_FW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_FW_VERSION_PRESENT_LBN 6 +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_FW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_BOOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_BOOT_VERSION_PRESENT_LBN 7 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_BOOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_UBOOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_UBOOT_VERSION_PRESENT_LBN 8 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_UBOOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_LBN 9 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_MAIN_ROOTFS_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_LBN 10 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_RECOVERY_BUILDROOT_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_VERSION_PRESENT_LBN 11 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_VERSION_PRESENT_LBN 12 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_VERSION_PRESENT_WIDTH 1 +#define MC_CMD_GET_VERSION_V5_OUT_BUNDLE_VERSION_PRESENT_OFST 48 +#define MC_CMD_GET_VERSION_V5_OUT_BUNDLE_VERSION_PRESENT_LBN 13 +#define MC_CMD_GET_VERSION_V5_OUT_BUNDLE_VERSION_PRESENT_WIDTH 1 +/* MC firmware unique build ID (as binary SHA-1 value) */ +#define MC_CMD_GET_VERSION_V5_OUT_MCFW_BUILD_ID_OFST 52 +#define MC_CMD_GET_VERSION_V5_OUT_MCFW_BUILD_ID_LEN 20 +/* MC firmware security level */ +#define MC_CMD_GET_VERSION_V5_OUT_MCFW_SECURITY_LEVEL_OFST 72 +#define MC_CMD_GET_VERSION_V5_OUT_MCFW_SECURITY_LEVEL_LEN 4 +/* MC firmware build name (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V5_OUT_MCFW_BUILD_NAME_OFST 76 +#define MC_CMD_GET_VERSION_V5_OUT_MCFW_BUILD_NAME_LEN 64 +/* The SUC firmware version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_VERSION_OFST 140 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_VERSION_NUM 4 +/* SUC firmware build date (as 64-bit Unix timestamp) */ +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_BUILD_DATE_OFST 156 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_BUILD_DATE_LEN 8 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_BUILD_DATE_LO_OFST 156 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_BUILD_DATE_LO_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_BUILD_DATE_LO_LBN 1248 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_BUILD_DATE_LO_WIDTH 32 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_BUILD_DATE_HI_OFST 160 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_BUILD_DATE_HI_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_BUILD_DATE_HI_LBN 1280 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_BUILD_DATE_HI_WIDTH 32 +/* The ID of the SUC chip. This is specific to the platform but typically + * indicates family, memory sizes etc. See SF-116728-SW for further details. + */ +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_CHIP_ID_OFST 164 +#define MC_CMD_GET_VERSION_V5_OUT_SUCFW_CHIP_ID_LEN 4 +/* The CMC firmware version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_VERSION_OFST 168 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_VERSION_NUM 4 +/* CMC firmware build date (as 64-bit Unix timestamp) */ +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_BUILD_DATE_OFST 184 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_BUILD_DATE_LEN 8 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_BUILD_DATE_LO_OFST 184 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_BUILD_DATE_LO_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_BUILD_DATE_LO_LBN 1472 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_BUILD_DATE_LO_WIDTH 32 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_BUILD_DATE_HI_OFST 188 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_BUILD_DATE_HI_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_BUILD_DATE_HI_LBN 1504 +#define MC_CMD_GET_VERSION_V5_OUT_CMCFW_BUILD_DATE_HI_WIDTH 32 +/* FPGA version as three numbers. On Riverhead based systems this field uses + * the same encoding as hardware version ID registers (MC_FPGA_BUILD_HWRD_REG): + * FPGA_VERSION[0]: x => Image H{x} FPGA_VERSION[1]: Revision letter (0 => A, 1 + * => B, ...) FPGA_VERSION[2]: Sub-revision number + */ +#define MC_CMD_GET_VERSION_V5_OUT_FPGA_VERSION_OFST 192 +#define MC_CMD_GET_VERSION_V5_OUT_FPGA_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_FPGA_VERSION_NUM 3 +/* Extra FPGA revision information (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V5_OUT_FPGA_EXTRA_OFST 204 +#define MC_CMD_GET_VERSION_V5_OUT_FPGA_EXTRA_LEN 16 +/* Board name / adapter model (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_NAME_OFST 220 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_NAME_LEN 16 +/* Board revision number */ +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_REVISION_OFST 236 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_REVISION_LEN 4 +/* Board serial number (as null-terminated US-ASCII string) */ +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_SERIAL_OFST 240 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_SERIAL_LEN 64 +/* The version of the datapath hardware design as three number - a.b.c */ +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_HW_VERSION_OFST 304 +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_HW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_HW_VERSION_NUM 3 +/* The version of the firmware library used to control the datapath as three + * number - a.b.c + */ +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_FW_VERSION_OFST 316 +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_FW_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_DATAPATH_FW_VERSION_NUM 3 +/* The SOC boot version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V5_OUT_SOC_BOOT_VERSION_OFST 328 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_BOOT_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_BOOT_VERSION_NUM 4 +/* The SOC uboot version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V5_OUT_SOC_UBOOT_VERSION_OFST 344 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_UBOOT_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_UBOOT_VERSION_NUM 4 +/* The SOC main rootfs version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V5_OUT_SOC_MAIN_ROOTFS_VERSION_OFST 360 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_MAIN_ROOTFS_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_MAIN_ROOTFS_VERSION_NUM 4 +/* The SOC recovery buildroot version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V5_OUT_SOC_RECOVERY_BUILDROOT_VERSION_OFST 376 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_RECOVERY_BUILDROOT_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_SOC_RECOVERY_BUILDROOT_VERSION_NUM 4 +/* Board version as four numbers - a.b.c.d. BOARD_VERSION[0] duplicates the + * BOARD_REVISION field + */ +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_VERSION_OFST 392 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_BOARD_VERSION_NUM 4 +/* Bundle version as four numbers - a.b.c.d */ +#define MC_CMD_GET_VERSION_V5_OUT_BUNDLE_VERSION_OFST 408 +#define MC_CMD_GET_VERSION_V5_OUT_BUNDLE_VERSION_LEN 4 +#define MC_CMD_GET_VERSION_V5_OUT_BUNDLE_VERSION_NUM 4 + /***********************************/ /* MC_CMD_PTP @@ -1789,7 +3074,9 @@ #define MC_CMD_PTP_IN_CMD_LEN 4 #define MC_CMD_PTP_IN_PERIPH_ID_OFST 4 #define MC_CMD_PTP_IN_PERIPH_ID_LEN 4 -/* Not used. Events are always sent to function relative queue 0. */ +/* Not used, initialize to 0. Events are always sent to function relative queue + * 0. + */ #define MC_CMD_PTP_IN_ENABLE_QUEUE_OFST 8 #define MC_CMD_PTP_IN_ENABLE_QUEUE_LEN 4 /* PTP timestamping mode. Not used from Huntington onwards. */ @@ -1866,7 +3153,13 @@ #define MC_CMD_PTP_IN_ADJUST_FREQ_OFST 8 #define MC_CMD_PTP_IN_ADJUST_FREQ_LEN 8 #define MC_CMD_PTP_IN_ADJUST_FREQ_LO_OFST 8 +#define MC_CMD_PTP_IN_ADJUST_FREQ_LO_LEN 4 +#define MC_CMD_PTP_IN_ADJUST_FREQ_LO_LBN 64 +#define MC_CMD_PTP_IN_ADJUST_FREQ_LO_WIDTH 32 #define MC_CMD_PTP_IN_ADJUST_FREQ_HI_OFST 12 +#define MC_CMD_PTP_IN_ADJUST_FREQ_HI_LEN 4 +#define MC_CMD_PTP_IN_ADJUST_FREQ_HI_LBN 96 +#define MC_CMD_PTP_IN_ADJUST_FREQ_HI_WIDTH 32 /* enum: Number of fractional bits in frequency adjustment */ #define MC_CMD_PTP_IN_ADJUST_BITS 0x28 /* enum: Number of fractional bits in frequency adjustment when FP44_FREQ_ADJ @@ -1897,7 +3190,13 @@ #define MC_CMD_PTP_IN_ADJUST_V2_FREQ_OFST 8 #define MC_CMD_PTP_IN_ADJUST_V2_FREQ_LEN 8 #define MC_CMD_PTP_IN_ADJUST_V2_FREQ_LO_OFST 8 +#define MC_CMD_PTP_IN_ADJUST_V2_FREQ_LO_LEN 4 +#define MC_CMD_PTP_IN_ADJUST_V2_FREQ_LO_LBN 64 +#define MC_CMD_PTP_IN_ADJUST_V2_FREQ_LO_WIDTH 32 #define MC_CMD_PTP_IN_ADJUST_V2_FREQ_HI_OFST 12 +#define MC_CMD_PTP_IN_ADJUST_V2_FREQ_HI_LEN 4 +#define MC_CMD_PTP_IN_ADJUST_V2_FREQ_HI_LBN 96 +#define MC_CMD_PTP_IN_ADJUST_V2_FREQ_HI_WIDTH 32 /* enum: Number of fractional bits in frequency adjustment */ /* MC_CMD_PTP_IN_ADJUST_BITS 0x28 */ /* enum: Number of fractional bits in frequency adjustment when FP44_FREQ_ADJ @@ -1936,7 +3235,13 @@ #define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_OFST 12 #define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_LEN 8 #define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_LO_OFST 12 +#define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_LO_LEN 4 +#define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_LO_LBN 96 +#define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_LO_WIDTH 32 #define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_HI_OFST 16 +#define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_HI_LEN 4 +#define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_HI_LBN 128 +#define MC_CMD_PTP_IN_SYNCHRONIZE_START_ADDR_HI_WIDTH 32 /* MC_CMD_PTP_IN_MANFTEST_BASIC msgrequest */ #define MC_CMD_PTP_IN_MANFTEST_BASIC_LEN 8 @@ -2052,7 +3357,13 @@ #define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_OFST 8 #define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_LEN 8 #define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_LO_OFST 8 +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_LO_LEN 4 +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_LO_LBN 64 +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_LO_WIDTH 32 #define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_HI_OFST 12 +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_HI_LEN 4 +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_HI_LBN 96 +#define MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_HI_WIDTH 32 /* Enum values, see field(s): */ /* MC_CMD_PTP/MC_CMD_PTP_IN_ADJUST/FREQ */ @@ -2083,7 +3394,13 @@ #define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_OFST 12 #define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_LEN 8 #define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_LO_OFST 12 +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_LO_LEN 4 +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_LO_LBN 96 +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_LO_WIDTH 32 #define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_HI_OFST 16 +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_HI_LEN 4 +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_HI_LBN 128 +#define MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_HI_WIDTH 32 /* MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER msgrequest */ #define MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER_LEN 16 @@ -2130,7 +3447,9 @@ #define MC_CMD_PTP_ENABLE_PPS 0x0 /* enum: Disable */ #define MC_CMD_PTP_DISABLE_PPS 0x1 -/* Not used. Events are always sent to function relative queue 0. */ +/* Not used, initialize to 0. Events are always sent to function relative queue + * 0. + */ #define MC_CMD_PTP_IN_PPS_ENABLE_QUEUE_ID_OFST 8 #define MC_CMD_PTP_IN_PPS_ENABLE_QUEUE_ID_LEN 4 @@ -2492,6 +3811,87 @@ #define MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED2_OFST 20 #define MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED2_LEN 4 +/* MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2 msgresponse */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_LEN 40 +/* Time format required/used by for this NIC. Applies to all PTP MCDI + * operations that pass times between the host and firmware. If this operation + * is not supported (older firmware) a format of seconds and nanoseconds should + * be assumed. + */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_TIME_FORMAT_OFST 0 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_TIME_FORMAT_LEN 4 +/* enum: Times are in seconds and nanoseconds */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_SECONDS_NANOSECONDS 0x0 +/* enum: Major register has units of 16 second per tick, minor 8 ns per tick */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_16SECONDS_8NANOSECONDS 0x1 +/* enum: Major register has units of seconds, minor 2^-27s per tick */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_SECONDS_27FRACTION 0x2 +/* enum: Major register units are seconds, minor units are quarter nanoseconds + */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_SECONDS_QTR_NANOSECONDS 0x3 +/* Minimum acceptable value for a corrected synchronization timeset. When + * comparing host and NIC clock times, the MC returns a set of samples that + * contain the host start and end time, the MC time when the host start was + * detected and the time the MC waited between reading the time and detecting + * the host end. The corrected sync window is the difference between the host + * end and start times minus the time that the MC waited for host end. + */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_SYNC_WINDOW_MIN_OFST 4 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_SYNC_WINDOW_MIN_LEN 4 +/* Various PTP capabilities */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_CAPABILITIES_OFST 8 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_CAPABILITIES_LEN 4 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_REPORT_SYNC_STATUS_OFST 8 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_REPORT_SYNC_STATUS_LBN 0 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_REPORT_SYNC_STATUS_WIDTH 1 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_RX_TSTAMP_OOB_OFST 8 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_RX_TSTAMP_OOB_LBN 1 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_RX_TSTAMP_OOB_WIDTH 1 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_64BIT_SECONDS_OFST 8 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_64BIT_SECONDS_LBN 2 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_64BIT_SECONDS_WIDTH 1 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FP44_FREQ_ADJ_OFST 8 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FP44_FREQ_ADJ_LBN 3 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FP44_FREQ_ADJ_WIDTH 1 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_RESERVED0_OFST 12 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_RESERVED0_LEN 4 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_RESERVED1_OFST 16 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_RESERVED1_LEN 4 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_RESERVED2_OFST 20 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_RESERVED2_LEN 4 +/* Minimum supported value for the FREQ field in + * MC_CMD_PTP/MC_CMD_PTP_IN_ADJUST and + * MC_CMD_PTP/MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST message requests. If this message + * response is not supported a value of -0.1 ns should be assumed, which is + * equivalent to a -10% adjustment. + */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MIN_OFST 24 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MIN_LEN 8 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MIN_LO_OFST 24 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MIN_LO_LEN 4 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MIN_LO_LBN 192 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MIN_LO_WIDTH 32 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MIN_HI_OFST 28 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MIN_HI_LEN 4 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MIN_HI_LBN 224 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MIN_HI_WIDTH 32 +/* Maximum supported value for the FREQ field in + * MC_CMD_PTP/MC_CMD_PTP_IN_ADJUST and + * MC_CMD_PTP/MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST message requests. If this message + * response is not supported a value of 0.1 ns should be assumed, which is + * equivalent to a +10% adjustment. + */ +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MAX_OFST 32 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MAX_LEN 8 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MAX_LO_OFST 32 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MAX_LO_LEN 4 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MAX_LO_LBN 256 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MAX_LO_WIDTH 32 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MAX_HI_OFST 36 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MAX_HI_LEN 4 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MAX_HI_LBN 288 +#define MC_CMD_PTP_OUT_GET_ATTRIBUTES_V2_FREQ_ADJ_MAX_HI_WIDTH 32 + /* MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS msgresponse */ #define MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_LEN 16 /* Uncorrected error on PTP transmit timestamps in NIC clock format */ @@ -2634,7 +4034,13 @@ #define MC_CMD_HP_IN_OCSD_ADDR_OFST 4 #define MC_CMD_HP_IN_OCSD_ADDR_LEN 8 #define MC_CMD_HP_IN_OCSD_ADDR_LO_OFST 4 +#define MC_CMD_HP_IN_OCSD_ADDR_LO_LEN 4 +#define MC_CMD_HP_IN_OCSD_ADDR_LO_LBN 32 +#define MC_CMD_HP_IN_OCSD_ADDR_LO_WIDTH 32 #define MC_CMD_HP_IN_OCSD_ADDR_HI_OFST 8 +#define MC_CMD_HP_IN_OCSD_ADDR_HI_LEN 4 +#define MC_CMD_HP_IN_OCSD_ADDR_HI_LBN 64 +#define MC_CMD_HP_IN_OCSD_ADDR_HI_WIDTH 32 /* The requested update interval, in seconds. (Or the sub-command if ADDR is * NULL.) */ @@ -3039,7 +4445,13 @@ #define MC_CMD_DBI_READX_IN_DBIRDOP_OFST 0 #define MC_CMD_DBI_READX_IN_DBIRDOP_LEN 8 #define MC_CMD_DBI_READX_IN_DBIRDOP_LO_OFST 0 +#define MC_CMD_DBI_READX_IN_DBIRDOP_LO_LEN 4 +#define MC_CMD_DBI_READX_IN_DBIRDOP_LO_LBN 0 +#define MC_CMD_DBI_READX_IN_DBIRDOP_LO_WIDTH 32 #define MC_CMD_DBI_READX_IN_DBIRDOP_HI_OFST 4 +#define MC_CMD_DBI_READX_IN_DBIRDOP_HI_LEN 4 +#define MC_CMD_DBI_READX_IN_DBIRDOP_HI_LBN 32 +#define MC_CMD_DBI_READX_IN_DBIRDOP_HI_WIDTH 32 #define MC_CMD_DBI_READX_IN_DBIRDOP_MINNUM 1 #define MC_CMD_DBI_READX_IN_DBIRDOP_MAXNUM 31 #define MC_CMD_DBI_READX_IN_DBIRDOP_MAXNUM_MCDI2 127 @@ -3643,6 +5055,8 @@ #define MC_CMD_MEDIA_BASE_T 0x6 /* enum: QSFP+. */ #define MC_CMD_MEDIA_QSFP_PLUS 0x7 +/* enum: DSFP. */ +#define MC_CMD_MEDIA_DSFP 0x8 #define MC_CMD_GET_PHY_CFG_OUT_MMD_MASK_OFST 48 #define MC_CMD_GET_PHY_CFG_OUT_MMD_MASK_LEN 4 /* enum: Native clause 22 */ @@ -3912,7 +5326,13 @@ #define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_OFST 0 #define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_LO_OFST 0 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_LO_LBN 0 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_HI_OFST 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_HI_LBN 32 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_100M_HI_WIDTH 32 /* enum: None. */ #define MC_CMD_LOOPBACK_NONE 0x0 /* enum: Data. */ @@ -3995,28 +5415,52 @@ #define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_OFST 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_LO_OFST 8 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_LO_LBN 64 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_HI_OFST 12 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_HI_LBN 96 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_1G_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ /* Supported loopbacks. */ #define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_OFST 16 #define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_LO_OFST 16 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_LO_LBN 128 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_HI_OFST 20 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_HI_LBN 160 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_10G_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ /* Supported loopbacks. */ #define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_OFST 24 #define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LO_OFST 24 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LO_LBN 192 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_HI_OFST 28 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_HI_LBN 224 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_SUGGESTED_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ /* Supported loopbacks. */ #define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_OFST 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_LO_OFST 32 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_LO_LBN 256 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_HI_OFST 36 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_HI_LBN 288 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_40G_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ @@ -4028,7 +5472,13 @@ #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_OFST 0 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_LO_OFST 0 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_LO_LBN 0 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_HI_OFST 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_HI_LBN 32 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_HI_WIDTH 32 /* enum: None. */ /* MC_CMD_LOOPBACK_NONE 0x0 */ /* enum: Data. */ @@ -4111,49 +5561,91 @@ #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_OFST 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_LO_OFST 8 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_LO_LBN 64 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_HI_OFST 12 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_HI_LBN 96 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ /* Supported loopbacks. */ #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_OFST 16 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_LO_OFST 16 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_LO_LBN 128 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_HI_OFST 20 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_HI_LBN 160 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ /* Supported loopbacks. */ #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_OFST 24 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_LO_OFST 24 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_LO_LBN 192 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_HI_OFST 28 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_HI_LBN 224 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ /* Supported loopbacks. */ #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_OFST 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_LO_OFST 32 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_LO_LBN 256 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_HI_OFST 36 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_HI_LBN 288 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ /* Supported 25G loopbacks. */ #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_OFST 40 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_LO_OFST 40 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_LO_LBN 320 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_HI_OFST 44 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_HI_LBN 352 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ /* Supported 50 loopbacks. */ #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_OFST 48 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_LO_OFST 48 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_LO_LBN 384 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_HI_OFST 52 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_HI_LBN 416 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ /* Supported 100G loopbacks. */ #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_OFST 56 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_LEN 8 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_LO_OFST 56 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_LO_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_LO_LBN 448 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_LO_WIDTH 32 #define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_HI_OFST 60 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_HI_LEN 4 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_HI_LBN 480 +#define MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_HI_WIDTH 32 /* Enum values, see field(s): */ /* 100M */ @@ -4524,7 +6016,13 @@ #define MC_CMD_SET_MAC_IN_ADDR_OFST 8 #define MC_CMD_SET_MAC_IN_ADDR_LEN 8 #define MC_CMD_SET_MAC_IN_ADDR_LO_OFST 8 +#define MC_CMD_SET_MAC_IN_ADDR_LO_LEN 4 +#define MC_CMD_SET_MAC_IN_ADDR_LO_LBN 64 +#define MC_CMD_SET_MAC_IN_ADDR_LO_WIDTH 32 #define MC_CMD_SET_MAC_IN_ADDR_HI_OFST 12 +#define MC_CMD_SET_MAC_IN_ADDR_HI_LEN 4 +#define MC_CMD_SET_MAC_IN_ADDR_HI_LBN 96 +#define MC_CMD_SET_MAC_IN_ADDR_HI_WIDTH 32 #define MC_CMD_SET_MAC_IN_REJECT_OFST 16 #define MC_CMD_SET_MAC_IN_REJECT_LEN 4 #define MC_CMD_SET_MAC_IN_REJECT_UNCST_OFST 16 @@ -4565,7 +6063,13 @@ #define MC_CMD_SET_MAC_EXT_IN_ADDR_OFST 8 #define MC_CMD_SET_MAC_EXT_IN_ADDR_LEN 8 #define MC_CMD_SET_MAC_EXT_IN_ADDR_LO_OFST 8 +#define MC_CMD_SET_MAC_EXT_IN_ADDR_LO_LEN 4 +#define MC_CMD_SET_MAC_EXT_IN_ADDR_LO_LBN 64 +#define MC_CMD_SET_MAC_EXT_IN_ADDR_LO_WIDTH 32 #define MC_CMD_SET_MAC_EXT_IN_ADDR_HI_OFST 12 +#define MC_CMD_SET_MAC_EXT_IN_ADDR_HI_LEN 4 +#define MC_CMD_SET_MAC_EXT_IN_ADDR_HI_LBN 96 +#define MC_CMD_SET_MAC_EXT_IN_ADDR_HI_WIDTH 32 #define MC_CMD_SET_MAC_EXT_IN_REJECT_OFST 16 #define MC_CMD_SET_MAC_EXT_IN_REJECT_LEN 4 #define MC_CMD_SET_MAC_EXT_IN_REJECT_UNCST_OFST 16 @@ -4616,6 +6120,129 @@ #define MC_CMD_SET_MAC_EXT_IN_CFG_FCS_LBN 4 #define MC_CMD_SET_MAC_EXT_IN_CFG_FCS_WIDTH 1 +/* MC_CMD_SET_MAC_V3_IN msgrequest */ +#define MC_CMD_SET_MAC_V3_IN_LEN 40 +/* The MTU is the MTU programmed directly into the XMAC/GMAC (inclusive of + * EtherII, VLAN, bug16011 padding). + */ +#define MC_CMD_SET_MAC_V3_IN_MTU_OFST 0 +#define MC_CMD_SET_MAC_V3_IN_MTU_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_DRAIN_OFST 4 +#define MC_CMD_SET_MAC_V3_IN_DRAIN_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_ADDR_OFST 8 +#define MC_CMD_SET_MAC_V3_IN_ADDR_LEN 8 +#define MC_CMD_SET_MAC_V3_IN_ADDR_LO_OFST 8 +#define MC_CMD_SET_MAC_V3_IN_ADDR_LO_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_ADDR_LO_LBN 64 +#define MC_CMD_SET_MAC_V3_IN_ADDR_LO_WIDTH 32 +#define MC_CMD_SET_MAC_V3_IN_ADDR_HI_OFST 12 +#define MC_CMD_SET_MAC_V3_IN_ADDR_HI_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_ADDR_HI_LBN 96 +#define MC_CMD_SET_MAC_V3_IN_ADDR_HI_WIDTH 32 +#define MC_CMD_SET_MAC_V3_IN_REJECT_OFST 16 +#define MC_CMD_SET_MAC_V3_IN_REJECT_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_REJECT_UNCST_OFST 16 +#define MC_CMD_SET_MAC_V3_IN_REJECT_UNCST_LBN 0 +#define MC_CMD_SET_MAC_V3_IN_REJECT_UNCST_WIDTH 1 +#define MC_CMD_SET_MAC_V3_IN_REJECT_BRDCST_OFST 16 +#define MC_CMD_SET_MAC_V3_IN_REJECT_BRDCST_LBN 1 +#define MC_CMD_SET_MAC_V3_IN_REJECT_BRDCST_WIDTH 1 +#define MC_CMD_SET_MAC_V3_IN_FCNTL_OFST 20 +#define MC_CMD_SET_MAC_V3_IN_FCNTL_LEN 4 +/* enum: Flow control is off. */ +/* MC_CMD_FCNTL_OFF 0x0 */ +/* enum: Respond to flow control. */ +/* MC_CMD_FCNTL_RESPOND 0x1 */ +/* enum: Respond to and Issue flow control. */ +/* MC_CMD_FCNTL_BIDIR 0x2 */ +/* enum: Auto neg flow control. */ +/* MC_CMD_FCNTL_AUTO 0x3 */ +/* enum: Priority flow control (eftest builds only). */ +/* MC_CMD_FCNTL_QBB 0x4 */ +/* enum: Issue flow control. */ +/* MC_CMD_FCNTL_GENERATE 0x5 */ +#define MC_CMD_SET_MAC_V3_IN_FLAGS_OFST 24 +#define MC_CMD_SET_MAC_V3_IN_FLAGS_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_FLAG_INCLUDE_FCS_OFST 24 +#define MC_CMD_SET_MAC_V3_IN_FLAG_INCLUDE_FCS_LBN 0 +#define MC_CMD_SET_MAC_V3_IN_FLAG_INCLUDE_FCS_WIDTH 1 +/* Select which parameters to configure. A parameter will only be modified if + * the corresponding control flag is set. If SET_MAC_ENHANCED is not set in + * capabilities then this field is ignored (and all flags are assumed to be + * set). + */ +#define MC_CMD_SET_MAC_V3_IN_CONTROL_OFST 28 +#define MC_CMD_SET_MAC_V3_IN_CONTROL_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_CFG_MTU_OFST 28 +#define MC_CMD_SET_MAC_V3_IN_CFG_MTU_LBN 0 +#define MC_CMD_SET_MAC_V3_IN_CFG_MTU_WIDTH 1 +#define MC_CMD_SET_MAC_V3_IN_CFG_DRAIN_OFST 28 +#define MC_CMD_SET_MAC_V3_IN_CFG_DRAIN_LBN 1 +#define MC_CMD_SET_MAC_V3_IN_CFG_DRAIN_WIDTH 1 +#define MC_CMD_SET_MAC_V3_IN_CFG_REJECT_OFST 28 +#define MC_CMD_SET_MAC_V3_IN_CFG_REJECT_LBN 2 +#define MC_CMD_SET_MAC_V3_IN_CFG_REJECT_WIDTH 1 +#define MC_CMD_SET_MAC_V3_IN_CFG_FCNTL_OFST 28 +#define MC_CMD_SET_MAC_V3_IN_CFG_FCNTL_LBN 3 +#define MC_CMD_SET_MAC_V3_IN_CFG_FCNTL_WIDTH 1 +#define MC_CMD_SET_MAC_V3_IN_CFG_FCS_OFST 28 +#define MC_CMD_SET_MAC_V3_IN_CFG_FCS_LBN 4 +#define MC_CMD_SET_MAC_V3_IN_CFG_FCS_WIDTH 1 +/* Identifies the MAC to update by the specifying the end of a logical MAE + * link. Setting TARGET to MAE_LINK_ENDPOINT_COMPAT is equivalent to using the + * previous version of the command (MC_CMD_SET_MAC_EXT). Not all possible + * combinations of MPORT_END and MPORT_SELECTOR in TARGET will work in all + * circumstances. 1. Some will always work (e.g. a VF can always address its + * logical MAC using MPORT_SELECTOR=ASSIGNED,LINK_END=VNIC), 2. Some are not + * meaningful and will always fail with EINVAL (e.g. attempting to address the + * VNIC end of a link to a physical port), 3. Some are meaningful but require + * the MCDI client to have the required permission and fail with EPERM + * otherwise (e.g. trying to set the MAC on a VF the caller cannot administer), + * and 4. Some could be implementation-specific and fail with ENOTSUP if not + * available (no examples exist right now). See SF-123581-TC section 4.3 for + * more details. + */ +#define MC_CMD_SET_MAC_V3_IN_TARGET_OFST 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_LEN 8 +#define MC_CMD_SET_MAC_V3_IN_TARGET_LO_OFST 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_LO_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_TARGET_LO_LBN 256 +#define MC_CMD_SET_MAC_V3_IN_TARGET_LO_WIDTH 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_HI_OFST 36 +#define MC_CMD_SET_MAC_V3_IN_TARGET_HI_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_TARGET_HI_LBN 288 +#define MC_CMD_SET_MAC_V3_IN_TARGET_HI_WIDTH 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_OFST 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_FLAT_OFST 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_FLAT_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_TYPE_OFST 35 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_TYPE_LEN 1 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_MPORT_ID_OFST 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_MPORT_ID_LEN 3 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_PPORT_ID_LBN 256 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_PPORT_ID_WIDTH 4 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_FUNC_INTF_ID_LBN 276 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_FUNC_INTF_ID_WIDTH 4 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_FUNC_MH_PF_ID_LBN 272 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_FUNC_MH_PF_ID_WIDTH 4 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_FUNC_PF_ID_OFST 34 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_FUNC_PF_ID_LEN 1 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_FUNC_VF_ID_OFST 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_MPORT_SELECTOR_FUNC_VF_ID_LEN 2 +#define MC_CMD_SET_MAC_V3_IN_TARGET_LINK_END_OFST 36 +#define MC_CMD_SET_MAC_V3_IN_TARGET_LINK_END_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_TARGET_FLAT_OFST 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_FLAT_LEN 8 +#define MC_CMD_SET_MAC_V3_IN_TARGET_FLAT_LO_OFST 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_FLAT_LO_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_TARGET_FLAT_LO_LBN 256 +#define MC_CMD_SET_MAC_V3_IN_TARGET_FLAT_LO_WIDTH 32 +#define MC_CMD_SET_MAC_V3_IN_TARGET_FLAT_HI_OFST 36 +#define MC_CMD_SET_MAC_V3_IN_TARGET_FLAT_HI_LEN 4 +#define MC_CMD_SET_MAC_V3_IN_TARGET_FLAT_HI_LBN 288 +#define MC_CMD_SET_MAC_V3_IN_TARGET_FLAT_HI_WIDTH 32 + /* MC_CMD_SET_MAC_OUT msgresponse */ #define MC_CMD_SET_MAC_OUT_LEN 0 @@ -4649,7 +6276,13 @@ #define MC_CMD_PHY_STATS_IN_DMA_ADDR_OFST 0 #define MC_CMD_PHY_STATS_IN_DMA_ADDR_LEN 8 #define MC_CMD_PHY_STATS_IN_DMA_ADDR_LO_OFST 0 +#define MC_CMD_PHY_STATS_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_PHY_STATS_IN_DMA_ADDR_LO_LBN 0 +#define MC_CMD_PHY_STATS_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_PHY_STATS_IN_DMA_ADDR_HI_OFST 4 +#define MC_CMD_PHY_STATS_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_PHY_STATS_IN_DMA_ADDR_HI_LBN 32 +#define MC_CMD_PHY_STATS_IN_DMA_ADDR_HI_WIDTH 32 /* MC_CMD_PHY_STATS_OUT_DMA msgresponse */ #define MC_CMD_PHY_STATS_OUT_DMA_LEN 0 @@ -4731,7 +6364,13 @@ #define MC_CMD_MAC_STATS_IN_DMA_ADDR_OFST 0 #define MC_CMD_MAC_STATS_IN_DMA_ADDR_LEN 8 #define MC_CMD_MAC_STATS_IN_DMA_ADDR_LO_OFST 0 +#define MC_CMD_MAC_STATS_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_MAC_STATS_IN_DMA_ADDR_LO_LBN 0 +#define MC_CMD_MAC_STATS_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_MAC_STATS_IN_DMA_ADDR_HI_OFST 4 +#define MC_CMD_MAC_STATS_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_MAC_STATS_IN_DMA_ADDR_HI_LBN 32 +#define MC_CMD_MAC_STATS_IN_DMA_ADDR_HI_WIDTH 32 #define MC_CMD_MAC_STATS_IN_CMD_OFST 8 #define MC_CMD_MAC_STATS_IN_CMD_LEN 4 #define MC_CMD_MAC_STATS_IN_DMA_OFST 8 @@ -4774,7 +6413,13 @@ #define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_OFST 0 #define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_LEN 8 #define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_LO_OFST 0 +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_LO_LEN 4 +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_LO_LBN 0 +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_LO_WIDTH 32 #define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_HI_OFST 4 +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_HI_LEN 4 +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_HI_LBN 32 +#define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_HI_WIDTH 32 #define MC_CMD_MAC_STATS_OUT_NO_DMA_STATISTICS_NUM MC_CMD_MAC_NSTATS #define MC_CMD_MAC_GENERATION_START 0x0 /* enum */ #define MC_CMD_MAC_DMABUF_START 0x1 /* enum */ @@ -4930,7 +6575,13 @@ #define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_OFST 0 #define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_LEN 8 #define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_LO_OFST 0 +#define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_LO_LEN 4 +#define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_LO_LBN 0 +#define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_LO_WIDTH 32 #define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_HI_OFST 4 +#define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_HI_LEN 4 +#define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_HI_LBN 32 +#define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_HI_WIDTH 32 #define MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_NUM MC_CMD_MAC_NSTATS_V2 /* enum: Start of FEC stats buffer space, Medford2 and up */ #define MC_CMD_MAC_FEC_DMABUF_START 0x61 @@ -4963,7 +6614,13 @@ #define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_OFST 0 #define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_LEN 8 #define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_LO_OFST 0 +#define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_LO_LEN 4 +#define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_LO_LBN 0 +#define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_LO_WIDTH 32 #define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_HI_OFST 4 +#define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_HI_LEN 4 +#define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_HI_LBN 32 +#define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_HI_WIDTH 32 #define MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_NUM MC_CMD_MAC_NSTATS_V3 /* enum: Start of CTPIO stats buffer space, Medford2 and up */ #define MC_CMD_MAC_CTPIO_DMABUF_START 0x68 @@ -5037,7 +6694,13 @@ #define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_OFST 0 #define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_LEN 8 #define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_LO_OFST 0 +#define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_LO_LEN 4 +#define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_LO_LBN 0 +#define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_LO_WIDTH 32 #define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_HI_OFST 4 +#define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_HI_LEN 4 +#define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_HI_LBN 32 +#define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_HI_WIDTH 32 #define MC_CMD_MAC_STATS_V4_OUT_NO_DMA_STATISTICS_NUM MC_CMD_MAC_NSTATS_V4 /* enum: Start of V4 stats buffer space */ #define MC_CMD_MAC_V4_DMABUF_START 0x79 @@ -5097,7 +6760,13 @@ #define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_OFST 8 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LEN 8 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LO_OFST 8 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LO_LEN 4 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LO_LBN 64 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LO_WIDTH 32 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_HI_OFST 12 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_HI_LEN 4 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_HI_LBN 96 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_HI_WIDTH 32 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LBN 64 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_WIDTH 64 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_OFST 16 @@ -5108,7 +6777,13 @@ #define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_OFST 20 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LEN 8 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LO_OFST 20 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LO_LEN 4 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LO_LBN 160 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LO_WIDTH 32 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_HI_OFST 24 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_HI_LEN 4 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_HI_LBN 192 +#define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_HI_WIDTH 32 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LBN 160 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_WIDTH 64 #define MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_OFST 28 @@ -5201,7 +6876,13 @@ #define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_OFST 8 #define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LEN 8 #define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LO_OFST 8 +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LO_LEN 4 +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LO_LBN 64 +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LO_WIDTH 32 #define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_HI_OFST 12 +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_HI_LEN 4 +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_HI_LBN 96 +#define MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_HI_WIDTH 32 /* MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN msgrequest */ #define MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_LEN 20 @@ -5706,6 +7387,9 @@ #define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAG_POLL_VERIFY_RESULT_OFST 8 #define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAG_POLL_VERIFY_RESULT_LBN 2 #define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAG_POLL_VERIFY_RESULT_WIDTH 1 +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAG_ABORT_OFST 8 +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAG_ABORT_LBN 3 +#define MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAG_ABORT_WIDTH 1 /* MC_CMD_NVRAM_UPDATE_FINISH_OUT msgresponse: Legacy NVRAM_UPDATE_FINISH * response. Use NVRAM_UPDATE_FINISH_V2_OUT in new code @@ -6180,7 +7864,13 @@ #define MC_CMD_SENSOR_ENTRY_OFST 4 #define MC_CMD_SENSOR_ENTRY_LEN 8 #define MC_CMD_SENSOR_ENTRY_LO_OFST 4 +#define MC_CMD_SENSOR_ENTRY_LO_LEN 4 +#define MC_CMD_SENSOR_ENTRY_LO_LBN 32 +#define MC_CMD_SENSOR_ENTRY_LO_WIDTH 32 #define MC_CMD_SENSOR_ENTRY_HI_OFST 8 +#define MC_CMD_SENSOR_ENTRY_HI_LEN 4 +#define MC_CMD_SENSOR_ENTRY_HI_LBN 64 +#define MC_CMD_SENSOR_ENTRY_HI_WIDTH 32 #define MC_CMD_SENSOR_ENTRY_MINNUM 0 #define MC_CMD_SENSOR_ENTRY_MAXNUM 31 #define MC_CMD_SENSOR_ENTRY_MAXNUM_MCDI2 127 @@ -6202,7 +7892,13 @@ /* MC_CMD_SENSOR_ENTRY_OFST 4 */ /* MC_CMD_SENSOR_ENTRY_LEN 8 */ /* MC_CMD_SENSOR_ENTRY_LO_OFST 4 */ +/* MC_CMD_SENSOR_ENTRY_LO_LEN 4 */ +/* MC_CMD_SENSOR_ENTRY_LO_LBN 32 */ +/* MC_CMD_SENSOR_ENTRY_LO_WIDTH 32 */ /* MC_CMD_SENSOR_ENTRY_HI_OFST 8 */ +/* MC_CMD_SENSOR_ENTRY_HI_LEN 4 */ +/* MC_CMD_SENSOR_ENTRY_HI_LBN 64 */ +/* MC_CMD_SENSOR_ENTRY_HI_WIDTH 32 */ /* MC_CMD_SENSOR_ENTRY_MINNUM 0 */ /* MC_CMD_SENSOR_ENTRY_MAXNUM 31 */ /* MC_CMD_SENSOR_ENTRY_MAXNUM_MCDI2 127 */ @@ -6259,7 +7955,13 @@ #define MC_CMD_READ_SENSORS_IN_DMA_ADDR_OFST 0 #define MC_CMD_READ_SENSORS_IN_DMA_ADDR_LEN 8 #define MC_CMD_READ_SENSORS_IN_DMA_ADDR_LO_OFST 0 +#define MC_CMD_READ_SENSORS_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_READ_SENSORS_IN_DMA_ADDR_LO_LBN 0 +#define MC_CMD_READ_SENSORS_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_READ_SENSORS_IN_DMA_ADDR_HI_OFST 4 +#define MC_CMD_READ_SENSORS_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_READ_SENSORS_IN_DMA_ADDR_HI_LBN 32 +#define MC_CMD_READ_SENSORS_IN_DMA_ADDR_HI_WIDTH 32 /* MC_CMD_READ_SENSORS_EXT_IN msgrequest */ #define MC_CMD_READ_SENSORS_EXT_IN_LEN 12 @@ -6271,7 +7973,13 @@ #define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_OFST 0 #define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_LEN 8 #define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_LO_OFST 0 +#define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_LO_LBN 0 +#define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_HI_OFST 4 +#define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_HI_LBN 32 +#define MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_HI_WIDTH 32 /* Size in bytes of host buffer. */ #define MC_CMD_READ_SENSORS_EXT_IN_LENGTH_OFST 8 #define MC_CMD_READ_SENSORS_EXT_IN_LENGTH_LEN 4 @@ -6286,7 +7994,13 @@ #define MC_CMD_READ_SENSORS_EXT_IN_V2_DMA_ADDR_OFST 0 #define MC_CMD_READ_SENSORS_EXT_IN_V2_DMA_ADDR_LEN 8 #define MC_CMD_READ_SENSORS_EXT_IN_V2_DMA_ADDR_LO_OFST 0 +#define MC_CMD_READ_SENSORS_EXT_IN_V2_DMA_ADDR_LO_LEN 4 +#define MC_CMD_READ_SENSORS_EXT_IN_V2_DMA_ADDR_LO_LBN 0 +#define MC_CMD_READ_SENSORS_EXT_IN_V2_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_READ_SENSORS_EXT_IN_V2_DMA_ADDR_HI_OFST 4 +#define MC_CMD_READ_SENSORS_EXT_IN_V2_DMA_ADDR_HI_LEN 4 +#define MC_CMD_READ_SENSORS_EXT_IN_V2_DMA_ADDR_HI_LBN 32 +#define MC_CMD_READ_SENSORS_EXT_IN_V2_DMA_ADDR_HI_WIDTH 32 /* Size in bytes of host buffer. */ #define MC_CMD_READ_SENSORS_EXT_IN_V2_LENGTH_OFST 8 #define MC_CMD_READ_SENSORS_EXT_IN_V2_LENGTH_LEN 4 @@ -6583,11 +8297,16 @@ /***********************************/ /* MC_CMD_GET_PHY_MEDIA_INFO * Read media-specific data from PHY (e.g. SFP/SFP+ module ID information for - * SFP+ PHYs). The 'media type' can be found via GET_PHY_CFG - * (GET_PHY_CFG_OUT_MEDIA_TYPE); the valid 'page number' input values, and the - * output data, are interpreted on a per-type basis. For SFP+: PAGE=0 or 1 + * SFP+ PHYs). The "media type" can be found via GET_PHY_CFG + * (GET_PHY_CFG_OUT_MEDIA_TYPE); the valid "page number" input values, and the + * output data, are interpreted on a per-type basis. For SFP+, PAGE=0 or 1 * returns a 128-byte block read from module I2C address 0xA0 offset 0 or 0x80. - * Anything else: currently undefined. Locks required: None. Return code: 0. + * For QSFP, PAGE=-1 is the lower (unbanked) page. PAGE=2 is the EEPROM and + * PAGE=3 is the module limits. For DSFP, module addressing requires a + * "BANK:PAGE". Not every bank has the same number of pages. See the Common + * Management Interface Specification (CMIS) for further details. A BANK:PAGE + * of "0xffff:0xffff" retrieves the lower (unbanked) page. Locks required - + * None. Return code - 0. */ #define MC_CMD_GET_PHY_MEDIA_INFO 0x4b #undef MC_CMD_0x4b_PRIVILEGE_CTG @@ -6598,6 +8317,12 @@ #define MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN 4 #define MC_CMD_GET_PHY_MEDIA_INFO_IN_PAGE_OFST 0 #define MC_CMD_GET_PHY_MEDIA_INFO_IN_PAGE_LEN 4 +#define MC_CMD_GET_PHY_MEDIA_INFO_IN_DSFP_PAGE_OFST 0 +#define MC_CMD_GET_PHY_MEDIA_INFO_IN_DSFP_PAGE_LBN 0 +#define MC_CMD_GET_PHY_MEDIA_INFO_IN_DSFP_PAGE_WIDTH 16 +#define MC_CMD_GET_PHY_MEDIA_INFO_IN_DSFP_BANK_OFST 0 +#define MC_CMD_GET_PHY_MEDIA_INFO_IN_DSFP_BANK_LBN 16 +#define MC_CMD_GET_PHY_MEDIA_INFO_IN_DSFP_BANK_WIDTH 16 /* MC_CMD_GET_PHY_MEDIA_INFO_OUT msgresponse */ #define MC_CMD_GET_PHY_MEDIA_INFO_OUT_LENMIN 5 @@ -7404,7 +9129,13 @@ #define MC_CMD_MUM_OUT_GET_VERSION_VERSION_OFST 4 #define MC_CMD_MUM_OUT_GET_VERSION_VERSION_LEN 8 #define MC_CMD_MUM_OUT_GET_VERSION_VERSION_LO_OFST 4 +#define MC_CMD_MUM_OUT_GET_VERSION_VERSION_LO_LEN 4 +#define MC_CMD_MUM_OUT_GET_VERSION_VERSION_LO_LBN 32 +#define MC_CMD_MUM_OUT_GET_VERSION_VERSION_LO_WIDTH 32 #define MC_CMD_MUM_OUT_GET_VERSION_VERSION_HI_OFST 8 +#define MC_CMD_MUM_OUT_GET_VERSION_VERSION_HI_LEN 4 +#define MC_CMD_MUM_OUT_GET_VERSION_VERSION_HI_LBN 64 +#define MC_CMD_MUM_OUT_GET_VERSION_VERSION_HI_WIDTH 32 /* MC_CMD_MUM_OUT_RAW_CMD msgresponse */ #define MC_CMD_MUM_OUT_RAW_CMD_LENMIN 1 @@ -7589,7 +9320,13 @@ #define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_OFST 8 #define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_LEN 8 #define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_LO_OFST 8 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_LO_LEN 4 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_LO_LBN 64 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_LO_WIDTH 32 #define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_HI_OFST 12 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_HI_LEN 4 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_HI_LBN 96 +#define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_HI_WIDTH 32 #define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_MINNUM 2 #define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_MAXNUM 30 #define MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_MAXNUM_MCDI2 126 @@ -7997,7 +9734,13 @@ #define BUFTBL_ENTRY_RAWADDR_OFST 4 #define BUFTBL_ENTRY_RAWADDR_LEN 8 #define BUFTBL_ENTRY_RAWADDR_LO_OFST 4 +#define BUFTBL_ENTRY_RAWADDR_LO_LEN 4 +#define BUFTBL_ENTRY_RAWADDR_LO_LBN 32 +#define BUFTBL_ENTRY_RAWADDR_LO_WIDTH 32 #define BUFTBL_ENTRY_RAWADDR_HI_OFST 8 +#define BUFTBL_ENTRY_RAWADDR_HI_LEN 4 +#define BUFTBL_ENTRY_RAWADDR_HI_LBN 64 +#define BUFTBL_ENTRY_RAWADDR_HI_WIDTH 32 #define BUFTBL_ENTRY_RAWADDR_LBN 32 #define BUFTBL_ENTRY_RAWADDR_WIDTH 64 @@ -8007,14 +9750,25 @@ #define NVRAM_PARTITION_TYPE_ID_LEN 2 /* enum: Primary MC firmware partition */ #define NVRAM_PARTITION_TYPE_MC_FIRMWARE 0x100 +/* enum: NMC firmware partition (this is intentionally an alias of MC_FIRMWARE) + */ +#define NVRAM_PARTITION_TYPE_NMC_FIRMWARE 0x100 /* enum: Secondary MC firmware partition */ #define NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP 0x200 /* enum: Expansion ROM partition */ #define NVRAM_PARTITION_TYPE_EXPANSION_ROM 0x300 /* enum: Static configuration TLV partition */ #define NVRAM_PARTITION_TYPE_STATIC_CONFIG 0x400 +/* enum: Factory configuration TLV partition (this is intentionally an alias of + * STATIC_CONFIG) + */ +#define NVRAM_PARTITION_TYPE_FACTORY_CONFIG 0x400 /* enum: Dynamic configuration TLV partition */ #define NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG 0x500 +/* enum: User configuration TLV partition (this is intentionally an alias of + * DYNAMIC_CONFIG) + */ +#define NVRAM_PARTITION_TYPE_USER_CONFIG 0x500 /* enum: Expansion ROM configuration data for port 0 */ #define NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0 0x600 /* enum: Synonym for EXPROM_CONFIG_PORT0 as used in pmap files */ @@ -8027,10 +9781,16 @@ #define NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT3 0x603 /* enum: Non-volatile log output partition */ #define NVRAM_PARTITION_TYPE_LOG 0x700 +/* enum: Non-volatile log output partition for NMC firmware (this is + * intentionally an alias of LOG) + */ +#define NVRAM_PARTITION_TYPE_NMC_LOG 0x700 /* enum: Non-volatile log output of second core on dual-core device */ #define NVRAM_PARTITION_TYPE_LOG_SLAVE 0x701 /* enum: Device state dump output partition */ #define NVRAM_PARTITION_TYPE_DUMP 0x800 +/* enum: Crash log partition for NMC firmware */ +#define NVRAM_PARTITION_TYPE_NMC_CRASH_LOG 0x801 /* enum: Application license key storage partition */ #define NVRAM_PARTITION_TYPE_LICENSE 0x900 /* enum: Start of range used for PHY partitions (low 8 bits are the PHY ID) */ @@ -8047,6 +9807,22 @@ #define NVRAM_PARTITION_TYPE_FC_LICENSE 0xb03 /* enum: Non-volatile log output partition for FC */ #define NVRAM_PARTITION_TYPE_FC_LOG 0xb04 +/* enum: FPGA Stage 1 bitstream */ +#define NVRAM_PARTITION_TYPE_FPGA_STAGE1 0xb05 +/* enum: FPGA Stage 2 bitstream */ +#define NVRAM_PARTITION_TYPE_FPGA_STAGE2 0xb06 +/* enum: FPGA User XCLBIN / Programmable Region 0 bitstream */ +#define NVRAM_PARTITION_TYPE_FPGA_REGION0 0xb07 +/* enum: FPGA User XCLBIN (this is intentionally an alias of FPGA_REGION0) */ +#define NVRAM_PARTITION_TYPE_FPGA_XCLBIN_USER 0xb07 +/* enum: FPGA jump instruction (a.k.a. boot) partition to select Stage1 + * bitstream + */ +#define NVRAM_PARTITION_TYPE_FPGA_JUMP 0xb08 +/* enum: FPGA Validate XCLBIN */ +#define NVRAM_PARTITION_TYPE_FPGA_XCLBIN_VALIDATE 0xb09 +/* enum: FPGA XOCL Configuration information */ +#define NVRAM_PARTITION_TYPE_FPGA_XOCL_CONFIG 0xb0a /* enum: MUM firmware partition */ #define NVRAM_PARTITION_TYPE_MUM_FIRMWARE 0xc00 /* enum: SUC firmware partition (this is intentionally an alias of @@ -8055,6 +9831,10 @@ #define NVRAM_PARTITION_TYPE_SUC_FIRMWARE 0xc00 /* enum: MUM Non-volatile log output partition. */ #define NVRAM_PARTITION_TYPE_MUM_LOG 0xc01 +/* enum: SUC Non-volatile log output partition (this is intentionally an alias + * of MUM_LOG). + */ +#define NVRAM_PARTITION_TYPE_SUC_LOG 0xc01 /* enum: MUM Application table partition. */ #define NVRAM_PARTITION_TYPE_MUM_APPTABLE 0xc02 /* enum: MUM boot rom partition. */ @@ -8069,6 +9849,10 @@ #define NVRAM_PARTITION_TYPE_EXPANSION_UEFI 0xd00 /* enum: Used by the expansion ROM for logging */ #define NVRAM_PARTITION_TYPE_PXE_LOG 0x1000 +/* enum: Non-volatile log output partition for Expansion ROM (this is + * intentionally an alias of PXE_LOG). + */ +#define NVRAM_PARTITION_TYPE_EXPROM_LOG 0x1000 /* enum: Used for XIP code of shmbooted images */ #define NVRAM_PARTITION_TYPE_XIP_SCRATCH 0x1100 /* enum: Spare partition 2 */ @@ -8077,6 +9861,10 @@ * between XJTAG and Manftest. */ #define NVRAM_PARTITION_TYPE_MANUFACTURING 0x1300 +/* enum: Deployment configuration TLV partition (this is intentionally an alias + * of MANUFACTURING) + */ +#define NVRAM_PARTITION_TYPE_DEPLOYMENT_CONFIG 0x1300 /* enum: Spare partition 4 */ #define NVRAM_PARTITION_TYPE_SPARE_4 0x1400 /* enum: Spare partition 5 */ @@ -8112,14 +9900,45 @@ #define NVRAM_PARTITION_TYPE_BUNDLE_LOG 0x1e02 /* enum: Partition for Solarflare gPXE bootrom installed via Bundle update. */ #define NVRAM_PARTITION_TYPE_EXPANSION_ROM_INTERNAL 0x1e03 +/* enum: Partition to store ASN.1 format Bundle Signature for checking. */ +#define NVRAM_PARTITION_TYPE_BUNDLE_SIGNATURE 0x1e04 +/* enum: Test partition on SmartNIC system microcontroller (SUC) */ +#define NVRAM_PARTITION_TYPE_SUC_TEST 0x1f00 +/* enum: System microcontroller access to primary FPGA flash. */ +#define NVRAM_PARTITION_TYPE_SUC_FPGA_PRIMARY 0x1f01 +/* enum: System microcontroller access to secondary FPGA flash (if present) */ +#define NVRAM_PARTITION_TYPE_SUC_FPGA_SECONDARY 0x1f02 +/* enum: System microcontroller access to primary System-on-Chip flash */ +#define NVRAM_PARTITION_TYPE_SUC_SOC_PRIMARY 0x1f03 +/* enum: System microcontroller access to secondary System-on-Chip flash (if + * present) + */ +#define NVRAM_PARTITION_TYPE_SUC_SOC_SECONDARY 0x1f04 +/* enum: System microcontroller critical failure logs. Contains structured + * details of sensors leading up to a critical failure (where the board is shut + * down). + */ +#define NVRAM_PARTITION_TYPE_SUC_FAILURE_LOG 0x1f05 +/* enum: System-on-Chip configuration information (see XN-200467-PS). */ +#define NVRAM_PARTITION_TYPE_SUC_SOC_CONFIG 0x1f07 +/* enum: System-on-Chip update information. */ +#define NVRAM_PARTITION_TYPE_SOC_UPDATE 0x2003 /* enum: Start of reserved value range (firmware may use for any purpose) */ #define NVRAM_PARTITION_TYPE_RESERVED_VALUES_MIN 0xff00 /* enum: End of reserved value range (firmware may use for any purpose) */ #define NVRAM_PARTITION_TYPE_RESERVED_VALUES_MAX 0xfffd /* enum: Recovery partition map (provided if real map is missing or corrupt) */ #define NVRAM_PARTITION_TYPE_RECOVERY_MAP 0xfffe +/* enum: Recovery Flash Partition Table, see SF-122606-TC. (this is + * intentionally an alias of RECOVERY_MAP) + */ +#define NVRAM_PARTITION_TYPE_RECOVERY_FPT 0xfffe /* enum: Partition map (real map as stored in flash) */ #define NVRAM_PARTITION_TYPE_PARTITION_MAP 0xffff +/* enum: Flash Partition Table, see SF-122606-TC. (this is intentionally an + * alias of PARTITION_MAP) + */ +#define NVRAM_PARTITION_TYPE_FPT 0xffff #define NVRAM_PARTITION_TYPE_ID_LBN 0 #define NVRAM_PARTITION_TYPE_ID_WIDTH 16 @@ -8168,7 +9987,13 @@ #define LICENSED_FEATURES_MASK_OFST 0 #define LICENSED_FEATURES_MASK_LEN 8 #define LICENSED_FEATURES_MASK_LO_OFST 0 +#define LICENSED_FEATURES_MASK_LO_LEN 4 +#define LICENSED_FEATURES_MASK_LO_LBN 0 +#define LICENSED_FEATURES_MASK_LO_WIDTH 32 #define LICENSED_FEATURES_MASK_HI_OFST 4 +#define LICENSED_FEATURES_MASK_HI_LEN 4 +#define LICENSED_FEATURES_MASK_HI_LBN 32 +#define LICENSED_FEATURES_MASK_HI_WIDTH 32 #define LICENSED_FEATURES_RX_CUT_THROUGH_OFST 0 #define LICENSED_FEATURES_RX_CUT_THROUGH_LBN 0 #define LICENSED_FEATURES_RX_CUT_THROUGH_WIDTH 1 @@ -8208,7 +10033,13 @@ #define LICENSED_V3_APPS_MASK_OFST 0 #define LICENSED_V3_APPS_MASK_LEN 8 #define LICENSED_V3_APPS_MASK_LO_OFST 0 +#define LICENSED_V3_APPS_MASK_LO_LEN 4 +#define LICENSED_V3_APPS_MASK_LO_LBN 0 +#define LICENSED_V3_APPS_MASK_LO_WIDTH 32 #define LICENSED_V3_APPS_MASK_HI_OFST 4 +#define LICENSED_V3_APPS_MASK_HI_LEN 4 +#define LICENSED_V3_APPS_MASK_HI_LBN 32 +#define LICENSED_V3_APPS_MASK_HI_WIDTH 32 #define LICENSED_V3_APPS_ONLOAD_OFST 0 #define LICENSED_V3_APPS_ONLOAD_LBN 0 #define LICENSED_V3_APPS_ONLOAD_WIDTH 1 @@ -8266,7 +10097,13 @@ #define LICENSED_V3_FEATURES_MASK_OFST 0 #define LICENSED_V3_FEATURES_MASK_LEN 8 #define LICENSED_V3_FEATURES_MASK_LO_OFST 0 +#define LICENSED_V3_FEATURES_MASK_LO_LEN 4 +#define LICENSED_V3_FEATURES_MASK_LO_LBN 0 +#define LICENSED_V3_FEATURES_MASK_LO_WIDTH 32 #define LICENSED_V3_FEATURES_MASK_HI_OFST 4 +#define LICENSED_V3_FEATURES_MASK_HI_LEN 4 +#define LICENSED_V3_FEATURES_MASK_HI_LBN 32 +#define LICENSED_V3_FEATURES_MASK_HI_WIDTH 32 #define LICENSED_V3_FEATURES_RX_CUT_THROUGH_OFST 0 #define LICENSED_V3_FEATURES_RX_CUT_THROUGH_LBN 0 #define LICENSED_V3_FEATURES_RX_CUT_THROUGH_WIDTH 1 @@ -8421,7 +10258,8 @@ #define MC_CMD_INIT_EVQ_IN_SIZE_OFST 0 #define MC_CMD_INIT_EVQ_IN_SIZE_LEN 4 /* Desired instance. Must be set to a specific instance, which is a function - * local queue index. + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). */ #define MC_CMD_INIT_EVQ_IN_INSTANCE_OFST 4 #define MC_CMD_INIT_EVQ_IN_INSTANCE_LEN 4 @@ -8493,7 +10331,13 @@ #define MC_CMD_INIT_EVQ_IN_DMA_ADDR_OFST 36 #define MC_CMD_INIT_EVQ_IN_DMA_ADDR_LEN 8 #define MC_CMD_INIT_EVQ_IN_DMA_ADDR_LO_OFST 36 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_LO_LBN 288 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_INIT_EVQ_IN_DMA_ADDR_HI_OFST 40 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_HI_LBN 320 +#define MC_CMD_INIT_EVQ_IN_DMA_ADDR_HI_WIDTH 32 #define MC_CMD_INIT_EVQ_IN_DMA_ADDR_MINNUM 1 #define MC_CMD_INIT_EVQ_IN_DMA_ADDR_MAXNUM 64 #define MC_CMD_INIT_EVQ_IN_DMA_ADDR_MAXNUM_MCDI2 64 @@ -8514,7 +10358,8 @@ #define MC_CMD_INIT_EVQ_V2_IN_SIZE_OFST 0 #define MC_CMD_INIT_EVQ_V2_IN_SIZE_LEN 4 /* Desired instance. Must be set to a specific instance, which is a function - * local queue index. + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). */ #define MC_CMD_INIT_EVQ_V2_IN_INSTANCE_OFST 4 #define MC_CMD_INIT_EVQ_V2_IN_INSTANCE_LEN 4 @@ -8611,7 +10456,13 @@ #define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_OFST 36 #define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_LEN 8 #define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_LO_OFST 36 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_LO_LBN 288 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_HI_OFST 40 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_HI_LBN 320 +#define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_HI_WIDTH 32 #define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_MINNUM 1 #define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_MAXNUM 64 #define MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_MAXNUM_MCDI2 64 @@ -8637,6 +10488,158 @@ #define MC_CMD_INIT_EVQ_V2_OUT_FLAG_RXQ_FORCE_EV_MERGING_LBN 3 #define MC_CMD_INIT_EVQ_V2_OUT_FLAG_RXQ_FORCE_EV_MERGING_WIDTH 1 +/* MC_CMD_INIT_EVQ_V3_IN msgrequest: Extended request to specify per-queue + * event merge timeouts. + */ +#define MC_CMD_INIT_EVQ_V3_IN_LEN 556 +/* Size, in entries */ +#define MC_CMD_INIT_EVQ_V3_IN_SIZE_OFST 0 +#define MC_CMD_INIT_EVQ_V3_IN_SIZE_LEN 4 +/* Desired instance. Must be set to a specific instance, which is a function + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). + */ +#define MC_CMD_INIT_EVQ_V3_IN_INSTANCE_OFST 4 +#define MC_CMD_INIT_EVQ_V3_IN_INSTANCE_LEN 4 +/* The initial timer value. The load value is ignored if the timer mode is DIS. + */ +#define MC_CMD_INIT_EVQ_V3_IN_TMR_LOAD_OFST 8 +#define MC_CMD_INIT_EVQ_V3_IN_TMR_LOAD_LEN 4 +/* The reload value is ignored in one-shot modes */ +#define MC_CMD_INIT_EVQ_V3_IN_TMR_RELOAD_OFST 12 +#define MC_CMD_INIT_EVQ_V3_IN_TMR_RELOAD_LEN 4 +/* tbd */ +#define MC_CMD_INIT_EVQ_V3_IN_FLAGS_OFST 16 +#define MC_CMD_INIT_EVQ_V3_IN_FLAGS_LEN 4 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_INTERRUPTING_OFST 16 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_INTERRUPTING_LBN 0 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_INTERRUPTING_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_RPTR_DOS_OFST 16 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_RPTR_DOS_LBN 1 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_RPTR_DOS_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_INT_ARMD_OFST 16 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_INT_ARMD_LBN 2 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_INT_ARMD_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_CUT_THRU_OFST 16 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_CUT_THRU_LBN 3 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_CUT_THRU_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_RX_MERGE_OFST 16 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_RX_MERGE_LBN 4 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_RX_MERGE_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_TX_MERGE_OFST 16 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_TX_MERGE_LBN 5 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_TX_MERGE_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_USE_TIMER_OFST 16 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_USE_TIMER_LBN 6 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_USE_TIMER_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_TYPE_OFST 16 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_TYPE_LBN 7 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_TYPE_WIDTH 4 +/* enum: All initialisation flags specified by host. */ +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_TYPE_MANUAL 0x0 +/* enum: MEDFORD only. Certain initialisation flags specified by host may be + * over-ridden by firmware based on licenses and firmware variant in order to + * provide the lowest latency achievable. See + * MC_CMD_INIT_EVQ_V2/MC_CMD_INIT_EVQ_V2_OUT/FLAGS for list of affected flags. + */ +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_TYPE_LOW_LATENCY 0x1 +/* enum: MEDFORD only. Certain initialisation flags specified by host may be + * over-ridden by firmware based on licenses and firmware variant in order to + * provide the best throughput achievable. See + * MC_CMD_INIT_EVQ_V2/MC_CMD_INIT_EVQ_V2_OUT/FLAGS for list of affected flags. + */ +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_TYPE_THROUGHPUT 0x2 +/* enum: MEDFORD only. Certain initialisation flags may be over-ridden by + * firmware based on licenses and firmware variant. See + * MC_CMD_INIT_EVQ_V2/MC_CMD_INIT_EVQ_V2_OUT/FLAGS for list of affected flags. + */ +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_TYPE_AUTO 0x3 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_EXT_WIDTH_OFST 16 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_EXT_WIDTH_LBN 11 +#define MC_CMD_INIT_EVQ_V3_IN_FLAG_EXT_WIDTH_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_IN_TMR_MODE_OFST 20 +#define MC_CMD_INIT_EVQ_V3_IN_TMR_MODE_LEN 4 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_V3_IN_TMR_MODE_DIS 0x0 +/* enum: Immediate */ +#define MC_CMD_INIT_EVQ_V3_IN_TMR_IMMED_START 0x1 +/* enum: Triggered */ +#define MC_CMD_INIT_EVQ_V3_IN_TMR_TRIG_START 0x2 +/* enum: Hold-off */ +#define MC_CMD_INIT_EVQ_V3_IN_TMR_INT_HLDOFF 0x3 +/* Target EVQ for wakeups if in wakeup mode. */ +#define MC_CMD_INIT_EVQ_V3_IN_TARGET_EVQ_OFST 24 +#define MC_CMD_INIT_EVQ_V3_IN_TARGET_EVQ_LEN 4 +/* Target interrupt if in interrupting mode (note union with target EVQ). Use + * MC_CMD_RESOURCE_INSTANCE_ANY unless a specific one required for test + * purposes. + */ +#define MC_CMD_INIT_EVQ_V3_IN_IRQ_NUM_OFST 24 +#define MC_CMD_INIT_EVQ_V3_IN_IRQ_NUM_LEN 4 +/* Event Counter Mode. */ +#define MC_CMD_INIT_EVQ_V3_IN_COUNT_MODE_OFST 28 +#define MC_CMD_INIT_EVQ_V3_IN_COUNT_MODE_LEN 4 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_V3_IN_COUNT_MODE_DIS 0x0 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_V3_IN_COUNT_MODE_RX 0x1 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_V3_IN_COUNT_MODE_TX 0x2 +/* enum: Disabled */ +#define MC_CMD_INIT_EVQ_V3_IN_COUNT_MODE_RXTX 0x3 +/* Event queue packet count threshold. */ +#define MC_CMD_INIT_EVQ_V3_IN_COUNT_THRSHLD_OFST 32 +#define MC_CMD_INIT_EVQ_V3_IN_COUNT_THRSHLD_LEN 4 +/* 64-bit address of 4k of 4k-aligned host memory buffer */ +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_OFST 36 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_LEN 8 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_LO_OFST 36 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_LO_LBN 288 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_LO_WIDTH 32 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_HI_OFST 40 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_HI_LBN 320 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_HI_WIDTH 32 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_MINNUM 1 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_MAXNUM 64 +#define MC_CMD_INIT_EVQ_V3_IN_DMA_ADDR_MAXNUM_MCDI2 64 +/* Receive event merge timeout to configure, in nanoseconds. The valid range + * and granularity are device specific. Specify 0 to use the firmware's default + * value. This field is ignored and per-queue merging is disabled if + * MC_CMD_INIT_EVQ/MC_CMD_INIT_EVQ_IN/FLAG_RX_MERGE is not set. + */ +#define MC_CMD_INIT_EVQ_V3_IN_RX_MERGE_TIMEOUT_NS_OFST 548 +#define MC_CMD_INIT_EVQ_V3_IN_RX_MERGE_TIMEOUT_NS_LEN 4 +/* Transmit event merge timeout to configure, in nanoseconds. The valid range + * and granularity are device specific. Specify 0 to use the firmware's default + * value. This field is ignored and per-queue merging is disabled if + * MC_CMD_INIT_EVQ/MC_CMD_INIT_EVQ_IN/FLAG_TX_MERGE is not set. + */ +#define MC_CMD_INIT_EVQ_V3_IN_TX_MERGE_TIMEOUT_NS_OFST 552 +#define MC_CMD_INIT_EVQ_V3_IN_TX_MERGE_TIMEOUT_NS_LEN 4 + +/* MC_CMD_INIT_EVQ_V3_OUT msgresponse */ +#define MC_CMD_INIT_EVQ_V3_OUT_LEN 8 +/* Only valid if INTRFLAG was true */ +#define MC_CMD_INIT_EVQ_V3_OUT_IRQ_OFST 0 +#define MC_CMD_INIT_EVQ_V3_OUT_IRQ_LEN 4 +/* Actual configuration applied on the card */ +#define MC_CMD_INIT_EVQ_V3_OUT_FLAGS_OFST 4 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAGS_LEN 4 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_CUT_THRU_OFST 4 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_CUT_THRU_LBN 0 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_CUT_THRU_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_RX_MERGE_OFST 4 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_RX_MERGE_LBN 1 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_RX_MERGE_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_TX_MERGE_OFST 4 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_TX_MERGE_LBN 2 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_TX_MERGE_WIDTH 1 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_RXQ_FORCE_EV_MERGING_OFST 4 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_RXQ_FORCE_EV_MERGING_LBN 3 +#define MC_CMD_INIT_EVQ_V3_OUT_FLAG_RXQ_FORCE_EV_MERGING_WIDTH 1 + /* QUEUE_CRC_MODE structuredef */ #define QUEUE_CRC_MODE_LEN 1 #define QUEUE_CRC_MODE_MODE_LBN 0 @@ -8687,7 +10690,8 @@ #define MC_CMD_INIT_RXQ_IN_LABEL_OFST 8 #define MC_CMD_INIT_RXQ_IN_LABEL_LEN 4 /* Desired instance. Must be set to a specific instance, which is a function - * local queue index. + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). */ #define MC_CMD_INIT_RXQ_IN_INSTANCE_OFST 12 #define MC_CMD_INIT_RXQ_IN_INSTANCE_LEN 4 @@ -8728,7 +10732,13 @@ #define MC_CMD_INIT_RXQ_IN_DMA_ADDR_OFST 28 #define MC_CMD_INIT_RXQ_IN_DMA_ADDR_LEN 8 #define MC_CMD_INIT_RXQ_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_LO_LBN 224 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_INIT_RXQ_IN_DMA_ADDR_HI_OFST 32 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_HI_LBN 256 +#define MC_CMD_INIT_RXQ_IN_DMA_ADDR_HI_WIDTH 32 #define MC_CMD_INIT_RXQ_IN_DMA_ADDR_MINNUM 1 #define MC_CMD_INIT_RXQ_IN_DMA_ADDR_MAXNUM 28 #define MC_CMD_INIT_RXQ_IN_DMA_ADDR_MAXNUM_MCDI2 124 @@ -8752,7 +10762,8 @@ #define MC_CMD_INIT_RXQ_EXT_IN_LABEL_OFST 8 #define MC_CMD_INIT_RXQ_EXT_IN_LABEL_LEN 4 /* Desired instance. Must be set to a specific instance, which is a function - * local queue index. + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). */ #define MC_CMD_INIT_RXQ_EXT_IN_INSTANCE_OFST 12 #define MC_CMD_INIT_RXQ_EXT_IN_INSTANCE_LEN 4 @@ -8826,8 +10837,16 @@ #define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_OFST 28 #define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_LEN 8 #define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_LO_LBN 224 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_HI_OFST 32 -#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_NUM 64 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_HI_LBN 256 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_HI_WIDTH 32 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_MINNUM 0 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_MAXNUM 64 +#define MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_MAXNUM_MCDI2 64 /* Maximum length of packet to receive, if SNAPSHOT_MODE flag is set */ #define MC_CMD_INIT_RXQ_EXT_IN_SNAPSHOT_LENGTH_OFST 540 #define MC_CMD_INIT_RXQ_EXT_IN_SNAPSHOT_LENGTH_LEN 4 @@ -8849,7 +10868,8 @@ #define MC_CMD_INIT_RXQ_V3_IN_LABEL_OFST 8 #define MC_CMD_INIT_RXQ_V3_IN_LABEL_LEN 4 /* Desired instance. Must be set to a specific instance, which is a function - * local queue index. + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). */ #define MC_CMD_INIT_RXQ_V3_IN_INSTANCE_OFST 12 #define MC_CMD_INIT_RXQ_V3_IN_INSTANCE_LEN 4 @@ -8923,8 +10943,16 @@ #define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_OFST 28 #define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_LEN 8 #define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_LO_LBN 224 +#define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_HI_OFST 32 -#define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_NUM 64 +#define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_HI_LBN 256 +#define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_HI_WIDTH 32 +#define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_MINNUM 0 +#define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_MAXNUM 64 +#define MC_CMD_INIT_RXQ_V3_IN_DMA_ADDR_MAXNUM_MCDI2 64 /* Maximum length of packet to receive, if SNAPSHOT_MODE flag is set */ #define MC_CMD_INIT_RXQ_V3_IN_SNAPSHOT_LENGTH_OFST 540 #define MC_CMD_INIT_RXQ_V3_IN_SNAPSHOT_LENGTH_LEN 4 @@ -8975,7 +11003,8 @@ #define MC_CMD_INIT_RXQ_V4_IN_LABEL_OFST 8 #define MC_CMD_INIT_RXQ_V4_IN_LABEL_LEN 4 /* Desired instance. Must be set to a specific instance, which is a function - * local queue index. + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). */ #define MC_CMD_INIT_RXQ_V4_IN_INSTANCE_OFST 12 #define MC_CMD_INIT_RXQ_V4_IN_INSTANCE_LEN 4 @@ -9049,8 +11078,16 @@ #define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_OFST 28 #define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_LEN 8 #define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_LO_LBN 224 +#define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_HI_OFST 32 -#define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_NUM 64 +#define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_HI_LBN 256 +#define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_HI_WIDTH 32 +#define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_MINNUM 0 +#define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_MAXNUM 64 +#define MC_CMD_INIT_RXQ_V4_IN_DMA_ADDR_MAXNUM_MCDI2 64 /* Maximum length of packet to receive, if SNAPSHOT_MODE flag is set */ #define MC_CMD_INIT_RXQ_V4_IN_SNAPSHOT_LENGTH_OFST 540 #define MC_CMD_INIT_RXQ_V4_IN_SNAPSHOT_LENGTH_LEN 4 @@ -9114,7 +11151,8 @@ #define MC_CMD_INIT_RXQ_V5_IN_LABEL_OFST 8 #define MC_CMD_INIT_RXQ_V5_IN_LABEL_LEN 4 /* Desired instance. Must be set to a specific instance, which is a function - * local queue index. + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). */ #define MC_CMD_INIT_RXQ_V5_IN_INSTANCE_OFST 12 #define MC_CMD_INIT_RXQ_V5_IN_INSTANCE_LEN 4 @@ -9188,8 +11226,16 @@ #define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_OFST 28 #define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_LEN 8 #define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_LO_LBN 224 +#define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_HI_OFST 32 -#define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_NUM 64 +#define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_HI_LBN 256 +#define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_HI_WIDTH 32 +#define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_MINNUM 0 +#define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_MAXNUM 64 +#define MC_CMD_INIT_RXQ_V5_IN_DMA_ADDR_MAXNUM_MCDI2 64 /* Maximum length of packet to receive, if SNAPSHOT_MODE flag is set */ #define MC_CMD_INIT_RXQ_V5_IN_SNAPSHOT_LENGTH_OFST 540 #define MC_CMD_INIT_RXQ_V5_IN_SNAPSHOT_LENGTH_LEN 4 @@ -9285,7 +11331,8 @@ #define MC_CMD_INIT_TXQ_IN_LABEL_OFST 8 #define MC_CMD_INIT_TXQ_IN_LABEL_LEN 4 /* Desired instance. Must be set to a specific instance, which is a function - * local queue index. + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). */ #define MC_CMD_INIT_TXQ_IN_INSTANCE_OFST 12 #define MC_CMD_INIT_TXQ_IN_INSTANCE_LEN 4 @@ -9329,7 +11376,13 @@ #define MC_CMD_INIT_TXQ_IN_DMA_ADDR_OFST 28 #define MC_CMD_INIT_TXQ_IN_DMA_ADDR_LEN 8 #define MC_CMD_INIT_TXQ_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_LO_LBN 224 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_INIT_TXQ_IN_DMA_ADDR_HI_OFST 32 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_HI_LBN 256 +#define MC_CMD_INIT_TXQ_IN_DMA_ADDR_HI_WIDTH 32 #define MC_CMD_INIT_TXQ_IN_DMA_ADDR_MINNUM 1 #define MC_CMD_INIT_TXQ_IN_DMA_ADDR_MAXNUM 28 #define MC_CMD_INIT_TXQ_IN_DMA_ADDR_MAXNUM_MCDI2 124 @@ -9350,7 +11403,8 @@ #define MC_CMD_INIT_TXQ_EXT_IN_LABEL_OFST 8 #define MC_CMD_INIT_TXQ_EXT_IN_LABEL_LEN 4 /* Desired instance. Must be set to a specific instance, which is a function - * local queue index. + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). */ #define MC_CMD_INIT_TXQ_EXT_IN_INSTANCE_OFST 12 #define MC_CMD_INIT_TXQ_EXT_IN_INSTANCE_LEN 4 @@ -9399,6 +11453,9 @@ #define MC_CMD_INIT_TXQ_EXT_IN_FLAG_DESC_PROXY_OFST 16 #define MC_CMD_INIT_TXQ_EXT_IN_FLAG_DESC_PROXY_LBN 16 #define MC_CMD_INIT_TXQ_EXT_IN_FLAG_DESC_PROXY_WIDTH 1 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_ABS_TARGET_EVQ_OFST 16 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_ABS_TARGET_EVQ_LBN 17 +#define MC_CMD_INIT_TXQ_EXT_IN_FLAG_ABS_TARGET_EVQ_WIDTH 1 /* Owner ID to use if in buffer mode (zero if physical) */ #define MC_CMD_INIT_TXQ_EXT_IN_OWNER_ID_OFST 20 #define MC_CMD_INIT_TXQ_EXT_IN_OWNER_ID_LEN 4 @@ -9409,8 +11466,14 @@ #define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_OFST 28 #define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_LEN 8 #define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_LO_OFST 28 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_LO_LEN 4 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_LO_LBN 224 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_LO_WIDTH 32 #define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_HI_OFST 32 -#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_MINNUM 1 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_HI_LEN 4 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_HI_LBN 256 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_HI_WIDTH 32 +#define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_MINNUM 0 #define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_MAXNUM 64 #define MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_MAXNUM_MCDI2 64 /* Flags related to Qbb flow control mode. */ @@ -9507,7 +11570,13 @@ #define MC_CMD_DRIVER_EVENT_IN_DATA_OFST 4 #define MC_CMD_DRIVER_EVENT_IN_DATA_LEN 8 #define MC_CMD_DRIVER_EVENT_IN_DATA_LO_OFST 4 +#define MC_CMD_DRIVER_EVENT_IN_DATA_LO_LEN 4 +#define MC_CMD_DRIVER_EVENT_IN_DATA_LO_LBN 32 +#define MC_CMD_DRIVER_EVENT_IN_DATA_LO_WIDTH 32 #define MC_CMD_DRIVER_EVENT_IN_DATA_HI_OFST 8 +#define MC_CMD_DRIVER_EVENT_IN_DATA_HI_LEN 4 +#define MC_CMD_DRIVER_EVENT_IN_DATA_HI_LBN 64 +#define MC_CMD_DRIVER_EVENT_IN_DATA_HI_WIDTH 32 /* MC_CMD_DRIVER_EVENT_OUT msgresponse */ #define MC_CMD_DRIVER_EVENT_OUT_LEN 0 @@ -9606,7 +11675,13 @@ #define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_OFST 4 #define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_LEN 8 #define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_LO_OFST 4 +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_LO_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_LO_LBN 32 +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_LO_WIDTH 32 #define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_HI_OFST 8 +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_HI_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_HI_LBN 64 +#define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_HI_WIDTH 32 /* Must be a power of 2 */ #define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BLOCK_SIZE_OFST 12 #define MC_CMD_PROXY_CONFIGURE_IN_STATUS_BLOCK_SIZE_LEN 4 @@ -9616,7 +11691,13 @@ #define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_OFST 16 #define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_LEN 8 #define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_LO_OFST 16 +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_LO_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_LO_LBN 128 +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_LO_WIDTH 32 #define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_HI_OFST 20 +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_HI_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_HI_LBN 160 +#define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_HI_WIDTH 32 /* Must be a power of 2 */ #define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BLOCK_SIZE_OFST 24 #define MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BLOCK_SIZE_LEN 4 @@ -9627,7 +11708,13 @@ #define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_OFST 28 #define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_LEN 8 #define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_LO_OFST 28 +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_LO_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_LO_LBN 224 +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_LO_WIDTH 32 #define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_HI_OFST 32 +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_HI_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_HI_LBN 256 +#define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_HI_WIDTH 32 /* Must be a power of 2, or zero if this buffer is not provided */ #define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BLOCK_SIZE_OFST 36 #define MC_CMD_PROXY_CONFIGURE_IN_REPLY_BLOCK_SIZE_LEN 4 @@ -9651,7 +11738,13 @@ #define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_OFST 4 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_LEN 8 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_LO_OFST 4 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_LO_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_LO_LBN 32 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_LO_WIDTH 32 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_HI_OFST 8 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_HI_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_HI_LBN 64 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_HI_WIDTH 32 /* Must be a power of 2 */ #define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BLOCK_SIZE_OFST 12 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BLOCK_SIZE_LEN 4 @@ -9661,7 +11754,13 @@ #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_OFST 16 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_LEN 8 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_LO_OFST 16 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_LO_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_LO_LBN 128 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_LO_WIDTH 32 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_HI_OFST 20 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_HI_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_HI_LBN 160 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_HI_WIDTH 32 /* Must be a power of 2 */ #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BLOCK_SIZE_OFST 24 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BLOCK_SIZE_LEN 4 @@ -9672,7 +11771,13 @@ #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_OFST 28 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_LEN 8 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_LO_OFST 28 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_LO_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_LO_LBN 224 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_LO_WIDTH 32 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_HI_OFST 32 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_HI_LEN 4 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_HI_LBN 256 +#define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_HI_WIDTH 32 /* Must be a power of 2, or zero if this buffer is not provided */ #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BLOCK_SIZE_OFST 36 #define MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BLOCK_SIZE_LEN 4 @@ -9788,7 +11893,13 @@ #define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_OFST 12 #define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_LEN 8 #define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_LO_OFST 12 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_LO_LEN 4 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_LO_LBN 96 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_LO_WIDTH 32 #define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_HI_OFST 16 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_HI_LEN 4 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_HI_LBN 128 +#define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_HI_WIDTH 32 #define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_MINNUM 1 #define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_MAXNUM 32 #define MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_MAXNUM_MCDI2 32 @@ -9844,7 +11955,13 @@ #define MC_CMD_FILTER_OP_IN_HANDLE_OFST 4 #define MC_CMD_FILTER_OP_IN_HANDLE_LEN 8 #define MC_CMD_FILTER_OP_IN_HANDLE_LO_OFST 4 +#define MC_CMD_FILTER_OP_IN_HANDLE_LO_LEN 4 +#define MC_CMD_FILTER_OP_IN_HANDLE_LO_LBN 32 +#define MC_CMD_FILTER_OP_IN_HANDLE_LO_WIDTH 32 #define MC_CMD_FILTER_OP_IN_HANDLE_HI_OFST 8 +#define MC_CMD_FILTER_OP_IN_HANDLE_HI_LEN 4 +#define MC_CMD_FILTER_OP_IN_HANDLE_HI_LBN 64 +#define MC_CMD_FILTER_OP_IN_HANDLE_HI_WIDTH 32 /* The port ID associated with the v-adaptor which should contain this filter. */ #define MC_CMD_FILTER_OP_IN_PORT_ID_OFST 12 @@ -9888,6 +12005,9 @@ #define MC_CMD_FILTER_OP_IN_MATCH_FWDEF1_OFST 16 #define MC_CMD_FILTER_OP_IN_MATCH_FWDEF1_LBN 11 #define MC_CMD_FILTER_OP_IN_MATCH_FWDEF1_WIDTH 1 +#define MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_IPV4_MCAST_DST_OFST 16 +#define MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_IPV4_MCAST_DST_LBN 29 +#define MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_IPV4_MCAST_DST_WIDTH 1 #define MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_MCAST_DST_OFST 16 #define MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_MCAST_DST_LBN 30 #define MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_MCAST_DST_WIDTH 1 @@ -10000,7 +12120,13 @@ #define MC_CMD_FILTER_OP_EXT_IN_HANDLE_OFST 4 #define MC_CMD_FILTER_OP_EXT_IN_HANDLE_LEN 8 #define MC_CMD_FILTER_OP_EXT_IN_HANDLE_LO_OFST 4 +#define MC_CMD_FILTER_OP_EXT_IN_HANDLE_LO_LEN 4 +#define MC_CMD_FILTER_OP_EXT_IN_HANDLE_LO_LBN 32 +#define MC_CMD_FILTER_OP_EXT_IN_HANDLE_LO_WIDTH 32 #define MC_CMD_FILTER_OP_EXT_IN_HANDLE_HI_OFST 8 +#define MC_CMD_FILTER_OP_EXT_IN_HANDLE_HI_LEN 4 +#define MC_CMD_FILTER_OP_EXT_IN_HANDLE_HI_LBN 64 +#define MC_CMD_FILTER_OP_EXT_IN_HANDLE_HI_WIDTH 32 /* The port ID associated with the v-adaptor which should contain this filter. */ #define MC_CMD_FILTER_OP_EXT_IN_PORT_ID_OFST 12 @@ -10086,6 +12212,9 @@ #define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_OFST 16 #define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN 25 #define MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_WIDTH 1 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_IPV4_MCAST_DST_OFST 16 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_IPV4_MCAST_DST_LBN 29 +#define MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_IPV4_MCAST_DST_WIDTH 1 #define MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_OFST 16 #define MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN 30 #define MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_WIDTH 1 @@ -10263,9 +12392,10 @@ #define MC_CMD_FILTER_OP_EXT_IN_IFRM_DST_IP_LEN 16 /* MC_CMD_FILTER_OP_V3_IN msgrequest: FILTER_OP extension to support additional - * filter actions for Intel's DPDK (Data Plane Development Kit, dpdk.org) via - * its rte_flow API. This extension is only useful with the sfc_efx driver - * included as part of DPDK, used in conjunction with the dpdk datapath + * filter actions for EF100. Some of these actions are also supported on EF10, + * for Intel's DPDK (Data Plane Development Kit, dpdk.org) via its rte_flow + * API. In the latter case, this extension is only useful with the sfc_efx + * driver included as part of DPDK, used in conjunction with the dpdk datapath * firmware variant. */ #define MC_CMD_FILTER_OP_V3_IN_LEN 180 @@ -10278,7 +12408,13 @@ #define MC_CMD_FILTER_OP_V3_IN_HANDLE_OFST 4 #define MC_CMD_FILTER_OP_V3_IN_HANDLE_LEN 8 #define MC_CMD_FILTER_OP_V3_IN_HANDLE_LO_OFST 4 +#define MC_CMD_FILTER_OP_V3_IN_HANDLE_LO_LEN 4 +#define MC_CMD_FILTER_OP_V3_IN_HANDLE_LO_LBN 32 +#define MC_CMD_FILTER_OP_V3_IN_HANDLE_LO_WIDTH 32 #define MC_CMD_FILTER_OP_V3_IN_HANDLE_HI_OFST 8 +#define MC_CMD_FILTER_OP_V3_IN_HANDLE_HI_LEN 4 +#define MC_CMD_FILTER_OP_V3_IN_HANDLE_HI_LBN 64 +#define MC_CMD_FILTER_OP_V3_IN_HANDLE_HI_WIDTH 32 /* The port ID associated with the v-adaptor which should contain this filter. */ #define MC_CMD_FILTER_OP_V3_IN_PORT_ID_OFST 12 @@ -10364,6 +12500,9 @@ #define MC_CMD_FILTER_OP_V3_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_OFST 16 #define MC_CMD_FILTER_OP_V3_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN 25 #define MC_CMD_FILTER_OP_V3_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_WIDTH 1 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_UNKNOWN_IPV4_MCAST_DST_OFST 16 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_UNKNOWN_IPV4_MCAST_DST_LBN 29 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_UNKNOWN_IPV4_MCAST_DST_WIDTH 1 #define MC_CMD_FILTER_OP_V3_IN_MATCH_UNKNOWN_MCAST_DST_OFST 16 #define MC_CMD_FILTER_OP_V3_IN_MATCH_UNKNOWN_MCAST_DST_LBN 30 #define MC_CMD_FILTER_OP_V3_IN_MATCH_UNKNOWN_MCAST_DST_WIDTH 1 @@ -10539,11 +12678,42 @@ */ #define MC_CMD_FILTER_OP_V3_IN_IFRM_DST_IP_OFST 156 #define MC_CMD_FILTER_OP_V3_IN_IFRM_DST_IP_LEN 16 -/* Set an action for all packets matching this filter. The DPDK driver and dpdk - * f/w variant use their own specific delivery structures, which are documented - * in the DPDK Firmware Driver Interface (SF-119419-TC). Requesting anything - * other than MATCH_ACTION_NONE when the NIC is running another f/w variant - * will cause the filter insertion to fail with ENOTSUP. +/* Flags controlling mutations of the packet and/or metadata when the filter is + * matched. The user_mark and user_flag fields' logic is as follows: if + * (req.MATCH_BITOR_FLAG == 1) user_flag = req.MATCH_SET_FLAG bit_or user_flag; + * else user_flag = req.MATCH_SET_FLAG; if (req.MATCH_SET_MARK == 0) user_mark + * = 0; else if (req.MATCH_BITOR_MARK == 1) user_mark = req.MATCH_SET_MARK + * bit_or user_mark; else user_mark = req.MATCH_SET_MARK; N.B. These flags + * overlap with the MATCH_ACTION field, which is deprecated in favour of this + * field. For the cases where these flags induce a valid encoding of the + * MATCH_ACTION field, the semantics agree. + */ +#define MC_CMD_FILTER_OP_V3_IN_MATCH_ACTION_FLAGS_OFST 172 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_ACTION_FLAGS_LEN 4 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_SET_FLAG_OFST 172 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_SET_FLAG_LBN 0 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_SET_FLAG_WIDTH 1 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_SET_MARK_OFST 172 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_SET_MARK_LBN 1 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_SET_MARK_WIDTH 1 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_BITOR_FLAG_OFST 172 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_BITOR_FLAG_LBN 2 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_BITOR_FLAG_WIDTH 1 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_BITOR_MARK_OFST 172 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_BITOR_MARK_LBN 3 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_BITOR_MARK_WIDTH 1 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_STRIP_VLAN_OFST 172 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_STRIP_VLAN_LBN 4 +#define MC_CMD_FILTER_OP_V3_IN_MATCH_STRIP_VLAN_WIDTH 1 +/* Deprecated: the overlapping MATCH_ACTION_FLAGS field exposes all of the + * functionality of this field in an ABI-backwards-compatible manner, and + * should be used instead. Any future extensions should be made to the + * MATCH_ACTION_FLAGS field, and not to this field. Set an action for all + * packets matching this filter. The DPDK driver and (on EF10) dpdk f/w variant + * use their own specific delivery structures, which are documented in the DPDK + * Firmware Driver Interface (SF-119419-TC). Requesting anything other than + * MATCH_ACTION_NONE on an EF10 NIC running another f/w variant will cause the + * filter insertion to fail with ENOTSUP. */ #define MC_CMD_FILTER_OP_V3_IN_MATCH_ACTION_OFST 172 #define MC_CMD_FILTER_OP_V3_IN_MATCH_ACTION_LEN 4 @@ -10580,7 +12750,13 @@ #define MC_CMD_FILTER_OP_OUT_HANDLE_OFST 4 #define MC_CMD_FILTER_OP_OUT_HANDLE_LEN 8 #define MC_CMD_FILTER_OP_OUT_HANDLE_LO_OFST 4 +#define MC_CMD_FILTER_OP_OUT_HANDLE_LO_LEN 4 +#define MC_CMD_FILTER_OP_OUT_HANDLE_LO_LBN 32 +#define MC_CMD_FILTER_OP_OUT_HANDLE_LO_WIDTH 32 #define MC_CMD_FILTER_OP_OUT_HANDLE_HI_OFST 8 +#define MC_CMD_FILTER_OP_OUT_HANDLE_HI_LEN 4 +#define MC_CMD_FILTER_OP_OUT_HANDLE_HI_LBN 64 +#define MC_CMD_FILTER_OP_OUT_HANDLE_HI_WIDTH 32 /* enum: guaranteed invalid filter handle (low 32 bits) */ #define MC_CMD_FILTER_OP_OUT_HANDLE_LO_INVALID 0xffffffff /* enum: guaranteed invalid filter handle (high 32 bits) */ @@ -10600,7 +12776,13 @@ #define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_OFST 4 #define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_LEN 8 #define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_LO_OFST 4 +#define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_LO_LEN 4 +#define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_LO_LBN 32 +#define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_LO_WIDTH 32 #define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_HI_OFST 8 +#define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_HI_LEN 4 +#define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_HI_LBN 64 +#define MC_CMD_FILTER_OP_EXT_OUT_HANDLE_HI_WIDTH 32 /* Enum values, see field(s): */ /* MC_CMD_FILTER_OP_OUT/HANDLE */ @@ -10638,6 +12820,8 @@ * rules inserted by MC_CMD_VNIC_ENCAP_RULE_ADD. (ef100 and later) */ #define MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_VNIC_ENCAP_MATCHES 0x5 +/* enum: read the supported encapsulation types for the VNIC */ +#define MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_VNIC_ENCAP_TYPES 0x6 /* MC_CMD_GET_PARSER_DISP_INFO_OUT msgresponse */ #define MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMIN 8 @@ -10704,6 +12888,30 @@ #define MC_CMD_GET_PARSER_DISP_VNIC_ENCAP_MATCHES_OUT_SUPPORTED_MATCHES_MAXNUM 61 #define MC_CMD_GET_PARSER_DISP_VNIC_ENCAP_MATCHES_OUT_SUPPORTED_MATCHES_MAXNUM_MCDI2 253 +/* MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT msgresponse: Returns + * the supported encapsulation types for the VNIC + */ +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_LEN 8 +/* The op code OP_GET_SUPPORTED_VNIC_ENCAP_TYPES is returned */ +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_OP_OFST 0 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_OP_LEN 4 +/* Enum values, see field(s): */ +/* MC_CMD_GET_PARSER_DISP_INFO_IN/OP */ +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPES_SUPPORTED_OFST 4 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPES_SUPPORTED_LEN 4 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_VXLAN_OFST 4 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_VXLAN_LBN 0 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_VXLAN_WIDTH 1 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_NVGRE_OFST 4 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_NVGRE_LBN 1 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_NVGRE_WIDTH 1 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_GENEVE_OFST 4 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_GENEVE_LBN 2 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_GENEVE_WIDTH 1 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_L2GRE_OFST 4 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_L2GRE_LBN 3 +#define MC_CMD_GET_PARSER_DISP_SUPPORTED_VNIC_ENCAP_TYPES_OUT_ENCAP_TYPE_L2GRE_WIDTH 1 + /***********************************/ /* MC_CMD_PARSER_DISP_RW @@ -10849,9 +13057,15 @@ /* MC_CMD_GET_PORT_ASSIGNMENT_OUT msgresponse */ #define MC_CMD_GET_PORT_ASSIGNMENT_OUT_LEN 4 -/* Identifies the port assignment for this function. */ +/* Identifies the port assignment for this function. On EF100, it is possible + * for the function to have no network port assigned (either because it is not + * yet configured, or assigning a port to a given function personality makes no + * sense - e.g. virtio-blk), in which case the return value is NULL_PORT. + */ #define MC_CMD_GET_PORT_ASSIGNMENT_OUT_PORT_OFST 0 #define MC_CMD_GET_PORT_ASSIGNMENT_OUT_PORT_LEN 4 +/* enum: Special value to indicate no port is assigned to a function. */ +#define MC_CMD_GET_PORT_ASSIGNMENT_OUT_NULL_PORT 0xffffffff /***********************************/ @@ -11009,7 +13223,8 @@ /***********************************/ /* MC_CMD_GET_VI_ALLOC_INFO * Get information about number of VI's and base VI number allocated to this - * function. + * function. This message is not available to dynamic clients created by + * MC_CMD_CLIENT_ALLOC. */ #define MC_CMD_GET_VI_ALLOC_INFO 0x8d #undef MC_CMD_0x8d_PRIVILEGE_CTG @@ -11036,7 +13251,9 @@ /***********************************/ /* MC_CMD_DUMP_VI_STATE - * For CmdClient use. Dump pertinent information on a specific absolute VI. + * For CmdClient use. Dump pertinent information on a specific absolute VI. The + * VI must be owned by the calling client or one of its ancestors; usership of + * the VI (as set by MC_CMD_SET_VI_USER) is not sufficient. */ #define MC_CMD_DUMP_VI_STATE 0x8e #undef MC_CMD_0x8e_PRIVILEGE_CTG @@ -11050,7 +13267,7 @@ #define MC_CMD_DUMP_VI_STATE_IN_VI_NUMBER_LEN 4 /* MC_CMD_DUMP_VI_STATE_OUT msgresponse */ -#define MC_CMD_DUMP_VI_STATE_OUT_LEN 96 +#define MC_CMD_DUMP_VI_STATE_OUT_LEN 100 /* The PF part of the function owning this VI. */ #define MC_CMD_DUMP_VI_STATE_OUT_OWNER_PF_OFST 0 #define MC_CMD_DUMP_VI_STATE_OUT_OWNER_PF_LEN 2 @@ -11073,12 +13290,24 @@ #define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_OFST 12 #define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_LEN 8 #define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_LO_OFST 12 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_LO_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_LO_LBN 96 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_LO_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_HI_OFST 16 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_HI_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_HI_LBN 128 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EVQ_PTR_RAW_HI_WIDTH 32 /* Raw evq timer table data. */ #define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_OFST 20 #define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_LEN 8 #define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_LO_OFST 20 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_LO_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_LO_LBN 160 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_LO_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_HI_OFST 24 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_HI_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_HI_LBN 192 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_HI_WIDTH 32 /* Combined metadata field. */ #define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_OFST 28 #define MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_LEN 4 @@ -11095,22 +13324,46 @@ #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_OFST 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_LEN 8 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_LO_OFST 32 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_LO_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_LO_LBN 256 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_LO_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_HI_OFST 36 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_HI_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_HI_LBN 288 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_0_HI_WIDTH 32 /* TXDPCPU raw table data for queue. */ #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_OFST 40 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_LEN 8 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_LO_OFST 40 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_LO_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_LO_LBN 320 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_LO_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_HI_OFST 44 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_HI_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_HI_LBN 352 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_1_HI_WIDTH 32 /* TXDPCPU raw table data for queue. */ #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_OFST 48 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_LEN 8 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_LO_OFST 48 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_LO_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_LO_LBN 384 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_LO_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_HI_OFST 52 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_HI_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_HI_LBN 416 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_RAW_TBL_2_HI_WIDTH 32 /* Combined metadata field. */ #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_OFST 56 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_LEN 8 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_LO_OFST 56 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_LO_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_LO_LBN 448 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_LO_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_HI_OFST 60 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_HI_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_HI_LBN 480 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_HI_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_BUFS_BASE_OFST 56 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_BUFS_BASE_LBN 0 #define MC_CMD_DUMP_VI_STATE_OUT_VI_TX_META_BUFS_BASE_WIDTH 16 @@ -11130,22 +13383,46 @@ #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_OFST 64 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_LEN 8 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_LO_OFST 64 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_LO_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_LO_LBN 512 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_LO_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_HI_OFST 68 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_HI_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_HI_LBN 544 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_0_HI_WIDTH 32 /* RXDPCPU raw table data for queue. */ #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_OFST 72 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_LEN 8 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_LO_OFST 72 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_LO_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_LO_LBN 576 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_LO_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_HI_OFST 76 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_HI_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_HI_LBN 608 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_1_HI_WIDTH 32 /* Reserved, currently 0. */ #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_OFST 80 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_LEN 8 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_LO_OFST 80 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_LO_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_LO_LBN 640 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_LO_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_HI_OFST 84 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_HI_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_HI_LBN 672 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_RAW_TBL_2_HI_WIDTH 32 /* Combined metadata field. */ #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_OFST 88 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_LEN 8 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_LO_OFST 88 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_LO_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_LO_LBN 704 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_LO_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_HI_OFST 92 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_HI_LEN 4 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_HI_LBN 736 +#define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_HI_WIDTH 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_BUFS_BASE_OFST 88 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_BUFS_BASE_LBN 0 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_BUFS_BASE_WIDTH 16 @@ -11158,6 +13435,9 @@ #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_WAITCOUNT_OFST 88 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_WAITCOUNT_LBN 32 #define MC_CMD_DUMP_VI_STATE_OUT_VI_RX_META_WAITCOUNT_WIDTH 8 +/* Current user, as assigned by MC_CMD_SET_VI_USER. */ +#define MC_CMD_DUMP_VI_STATE_OUT_USER_CLIENT_ID_OFST 96 +#define MC_CMD_DUMP_VI_STATE_OUT_USER_CLIENT_ID_LEN 4 /***********************************/ @@ -11200,7 +13480,9 @@ /***********************************/ /* MC_CMD_GET_VI_TLP_PROCESSING - * Get TLP steering and ordering information for a VI. + * Get TLP steering and ordering information for a VI. The caller must have the + * GRP_FUNC_DMA privilege and must be the currently-assigned user of this VI or + * an ancestor of the current user (see MC_CMD_SET_VI_USER). */ #define MC_CMD_GET_VI_TLP_PROCESSING 0xb0 #undef MC_CMD_0xb0_PRIVILEGE_CTG @@ -11239,7 +13521,9 @@ /***********************************/ /* MC_CMD_SET_VI_TLP_PROCESSING - * Set TLP steering and ordering information for a VI. + * Set TLP steering and ordering information for a VI. The caller must have the + * GRP_FUNC_DMA privilege and must be the currently-assigned user of this VI or + * an ancestor of the current user (see MC_CMD_SET_VI_USER). */ #define MC_CMD_SET_VI_TLP_PROCESSING 0xb1 #undef MC_CMD_0xb1_PRIVILEGE_CTG @@ -14497,6 +16781,24 @@ #define MC_CMD_GET_CAPABILITIES_V7_OUT_UNSOL_EV_CREDIT_SUPPORTED_OFST 148 #define MC_CMD_GET_CAPABILITIES_V7_OUT_UNSOL_EV_CREDIT_SUPPORTED_LBN 7 #define MC_CMD_GET_CAPABILITIES_V7_OUT_UNSOL_EV_CREDIT_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_ENCAPSULATED_MCDI_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_ENCAPSULATED_MCDI_SUPPORTED_LBN 8 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_ENCAPSULATED_MCDI_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_EXTERNAL_MAE_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_EXTERNAL_MAE_SUPPORTED_LBN 9 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_EXTERNAL_MAE_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_LBN 10 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_LBN 11 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_RSS_STEER_ON_OUTER_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_RSS_STEER_ON_OUTER_SUPPORTED_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_RSS_STEER_ON_OUTER_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14 +#define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1 /* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */ #define MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160 @@ -14983,6 +17285,24 @@ #define MC_CMD_GET_CAPABILITIES_V8_OUT_UNSOL_EV_CREDIT_SUPPORTED_OFST 148 #define MC_CMD_GET_CAPABILITIES_V8_OUT_UNSOL_EV_CREDIT_SUPPORTED_LBN 7 #define MC_CMD_GET_CAPABILITIES_V8_OUT_UNSOL_EV_CREDIT_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_ENCAPSULATED_MCDI_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_ENCAPSULATED_MCDI_SUPPORTED_LBN 8 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_ENCAPSULATED_MCDI_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_EXTERNAL_MAE_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_EXTERNAL_MAE_SUPPORTED_LBN 9 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_EXTERNAL_MAE_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_LBN 10 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_LBN 11 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_RSS_STEER_ON_OUTER_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_RSS_STEER_ON_OUTER_SUPPORTED_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_RSS_STEER_ON_OUTER_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1 /* These bits are reserved for communicating test-specific capabilities to * host-side test software. All production drivers should treat this field as * opaque. @@ -14990,7 +17310,13 @@ #define MC_CMD_GET_CAPABILITIES_V8_OUT_TEST_RESERVED_OFST 152 #define MC_CMD_GET_CAPABILITIES_V8_OUT_TEST_RESERVED_LEN 8 #define MC_CMD_GET_CAPABILITIES_V8_OUT_TEST_RESERVED_LO_OFST 152 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_TEST_RESERVED_LO_LEN 4 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_TEST_RESERVED_LO_LBN 1216 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_TEST_RESERVED_LO_WIDTH 32 #define MC_CMD_GET_CAPABILITIES_V8_OUT_TEST_RESERVED_HI_OFST 156 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_TEST_RESERVED_HI_LEN 4 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_TEST_RESERVED_HI_LBN 1248 +#define MC_CMD_GET_CAPABILITIES_V8_OUT_TEST_RESERVED_HI_WIDTH 32 /* MC_CMD_GET_CAPABILITIES_V9_OUT msgresponse */ #define MC_CMD_GET_CAPABILITIES_V9_OUT_LEN 184 @@ -15477,6 +17803,24 @@ #define MC_CMD_GET_CAPABILITIES_V9_OUT_UNSOL_EV_CREDIT_SUPPORTED_OFST 148 #define MC_CMD_GET_CAPABILITIES_V9_OUT_UNSOL_EV_CREDIT_SUPPORTED_LBN 7 #define MC_CMD_GET_CAPABILITIES_V9_OUT_UNSOL_EV_CREDIT_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_ENCAPSULATED_MCDI_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_ENCAPSULATED_MCDI_SUPPORTED_LBN 8 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_ENCAPSULATED_MCDI_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_EXTERNAL_MAE_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_EXTERNAL_MAE_SUPPORTED_LBN 9 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_EXTERNAL_MAE_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_LBN 10 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_LBN 11 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_RSS_STEER_ON_OUTER_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_RSS_STEER_ON_OUTER_SUPPORTED_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_RSS_STEER_ON_OUTER_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1 /* These bits are reserved for communicating test-specific capabilities to * host-side test software. All production drivers should treat this field as * opaque. @@ -15484,7 +17828,13 @@ #define MC_CMD_GET_CAPABILITIES_V9_OUT_TEST_RESERVED_OFST 152 #define MC_CMD_GET_CAPABILITIES_V9_OUT_TEST_RESERVED_LEN 8 #define MC_CMD_GET_CAPABILITIES_V9_OUT_TEST_RESERVED_LO_OFST 152 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_TEST_RESERVED_LO_LEN 4 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_TEST_RESERVED_LO_LBN 1216 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_TEST_RESERVED_LO_WIDTH 32 #define MC_CMD_GET_CAPABILITIES_V9_OUT_TEST_RESERVED_HI_OFST 156 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_TEST_RESERVED_HI_LEN 4 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_TEST_RESERVED_HI_LBN 1248 +#define MC_CMD_GET_CAPABILITIES_V9_OUT_TEST_RESERVED_HI_WIDTH 32 /* The minimum size (in table entries) of indirection table to be allocated * from the pool for an RSS context. Note that the table size used must be a * power of 2. @@ -15521,6 +17871,573 @@ #define MC_CMD_GET_CAPABILITIES_V9_OUT_RSS_TABLE_POOL_SIZE_OFST 180 #define MC_CMD_GET_CAPABILITIES_V9_OUT_RSS_TABLE_POOL_SIZE_LEN 4 +/* MC_CMD_GET_CAPABILITIES_V10_OUT msgresponse */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_LEN 192 +/* First word of flags. */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FLAGS1_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FLAGS1_LEN 4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VPORT_RECONFIGURE_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VPORT_RECONFIGURE_LBN 3 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VPORT_RECONFIGURE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_STRIPING_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_STRIPING_LBN 4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_STRIPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VADAPTOR_QUERY_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VADAPTOR_QUERY_LBN 5 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VADAPTOR_QUERY_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVB_PORT_VLAN_RESTRICT_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVB_PORT_VLAN_RESTRICT_LBN 6 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVB_PORT_VLAN_RESTRICT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_DRV_ATTACH_PREBOOT_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_DRV_ATTACH_PREBOOT_LBN 7 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_DRV_ATTACH_PREBOOT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_FORCE_EVENT_MERGING_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_FORCE_EVENT_MERGING_LBN 8 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_FORCE_EVENT_MERGING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_SET_MAC_ENHANCED_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_SET_MAC_ENHANCED_LBN 9 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_SET_MAC_ENHANCED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_LBN 10 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 11 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_MAC_SECURITY_FILTERING_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_MAC_SECURITY_FILTERING_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_MAC_SECURITY_FILTERING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_ADDITIONAL_RSS_MODES_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_ADDITIONAL_RSS_MODES_LBN 13 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_ADDITIONAL_RSS_MODES_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_QBB_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_QBB_LBN 14 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_QBB_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PACKED_STREAM_VAR_BUFFERS_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PACKED_STREAM_VAR_BUFFERS_LBN 15 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PACKED_STREAM_VAR_BUFFERS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_RSS_LIMITED_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_RSS_LIMITED_LBN 16 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_RSS_LIMITED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PACKED_STREAM_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PACKED_STREAM_LBN 17 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PACKED_STREAM_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_INCLUDE_FCS_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_INCLUDE_FCS_LBN 18 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_INCLUDE_FCS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_VLAN_INSERTION_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_VLAN_INSERTION_LBN 19 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_VLAN_INSERTION_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_VLAN_STRIPPING_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_VLAN_STRIPPING_LBN 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_VLAN_STRIPPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_LBN 21 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PREFIX_LEN_0_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PREFIX_LEN_0_LBN 22 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PREFIX_LEN_0_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PREFIX_LEN_14_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PREFIX_LEN_14_LBN 23 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_PREFIX_LEN_14_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_TIMESTAMP_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_TIMESTAMP_LBN 24 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_TIMESTAMP_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_BATCHING_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_BATCHING_LBN 25 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_BATCHING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MCAST_FILTER_CHAINING_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MCAST_FILTER_CHAINING_LBN 26 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MCAST_FILTER_CHAINING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_PM_AND_RXDP_COUNTERS_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_PM_AND_RXDP_COUNTERS_LBN 27 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_PM_AND_RXDP_COUNTERS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_DISABLE_SCATTER_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_DISABLE_SCATTER_LBN 28 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_DISABLE_SCATTER_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_MCAST_UDP_LOOPBACK_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_MCAST_UDP_LOOPBACK_LBN 29 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_MCAST_UDP_LOOPBACK_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVB_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVB_LBN 30 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVB_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VXLAN_NVGRE_OFST 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VXLAN_NVGRE_LBN 31 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VXLAN_NVGRE_WIDTH 1 +/* RxDPCPU firmware id. */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_DPCPU_FW_ID_OFST 4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_DPCPU_FW_ID_LEN 2 +/* enum: Standard RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP 0x0 +/* enum: Low latency RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_LOW_LATENCY 0x1 +/* enum: Packed stream RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_PACKED_STREAM 0x2 +/* enum: Rules engine RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_RULES_ENGINE 0x5 +/* enum: DPDK RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_DPDK 0x6 +/* enum: BIST RXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_BIST 0x10a +/* enum: RXDP Test firmware image 1 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_TEST_FW_TO_MC_CUT_THROUGH 0x101 +/* enum: RXDP Test firmware image 2 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD 0x102 +/* enum: RXDP Test firmware image 3 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD_FIRST 0x103 +/* enum: RXDP Test firmware image 4 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_TEST_EVERY_EVENT_BATCHABLE 0x104 +/* enum: RXDP Test firmware image 5 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_TEST_BACKPRESSURE 0x105 +/* enum: RXDP Test firmware image 6 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_TEST_FW_PACKET_EDITS 0x106 +/* enum: RXDP Test firmware image 7 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_TEST_FW_RX_HDR_SPLIT 0x107 +/* enum: RXDP Test firmware image 8 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_TEST_FW_DISABLE_DL 0x108 +/* enum: RXDP Test firmware image 9 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_TEST_FW_DOORBELL_DELAY 0x10b +/* enum: RXDP Test firmware image 10 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_TEST_FW_SLOW 0x10c +/* TxDPCPU firmware id. */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_DPCPU_FW_ID_OFST 6 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_DPCPU_FW_ID_LEN 2 +/* enum: Standard TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXDP 0x0 +/* enum: Low latency TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXDP_LOW_LATENCY 0x1 +/* enum: High packet rate TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXDP_HIGH_PACKET_RATE 0x3 +/* enum: Rules engine TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXDP_RULES_ENGINE 0x5 +/* enum: DPDK TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXDP_DPDK 0x6 +/* enum: BIST TXDP firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXDP_BIST 0x12d +/* enum: TXDP Test firmware image 1 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXDP_TEST_FW_TSO_EDIT 0x101 +/* enum: TXDP Test firmware image 2 */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXDP_TEST_FW_PACKET_EDITS 0x102 +/* enum: TXDP CSR bus test firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXDP_TEST_FW_CSR 0x103 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_VERSION_OFST 8 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_VERSION_LEN 2 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_VERSION_REV_OFST 8 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_VERSION_REV_LBN 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_VERSION_REV_WIDTH 12 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_VERSION_TYPE_OFST 8 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_VERSION_TYPE_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_VERSION_TYPE_WIDTH 4 +/* enum: reserved value - do not use (may indicate alternative interpretation + * of REV field in future) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_RESERVED 0x0 +/* enum: Trivial RX PD firmware for early Huntington development (Huntington + * development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_FIRST_PKT 0x1 +/* enum: RX PD firmware for telemetry prototyping (Medford2 development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_TESTFW_TELEMETRY 0x1 +/* enum: RX PD firmware with approximately Siena-compatible behaviour + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_SIENA_COMPAT 0x2 +/* enum: Full featured RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_FULL_FEATURED 0x3 +/* enum: (deprecated original name for the FULL_FEATURED variant) */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_VSWITCH 0x3 +/* enum: siena_compat variant RX PD firmware using PM rather than MAC + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_SIENA_COMPAT_PM 0x4 +/* enum: Low latency RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_LOW_LATENCY 0x5 +/* enum: Packed stream RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_PACKED_STREAM 0x6 +/* enum: RX PD firmware handling layer 2 only for high packet rate performance + * tests (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_LAYER2_PERF 0x7 +/* enum: Rules engine RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_RULES_ENGINE 0x8 +/* enum: Custom firmware variant (see SF-119495-PD and bug69716) */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_L3XUDP 0x9 +/* enum: DPDK RX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_DPDK 0xa +/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE 0xe +/* enum: RX PD firmware parsing but not filtering network overlay tunnel + * encapsulations (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXPD_FW_TYPE_TESTFW_ENCAP_PARSING_ONLY 0xf +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_VERSION_OFST 10 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_VERSION_LEN 2 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_VERSION_REV_OFST 10 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_VERSION_REV_LBN 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_VERSION_REV_WIDTH 12 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_VERSION_TYPE_OFST 10 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_VERSION_TYPE_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_VERSION_TYPE_WIDTH 4 +/* enum: reserved value - do not use (may indicate alternative interpretation + * of REV field in future) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_RESERVED 0x0 +/* enum: Trivial TX PD firmware for early Huntington development (Huntington + * development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_FIRST_PKT 0x1 +/* enum: TX PD firmware for telemetry prototyping (Medford2 development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_TESTFW_TELEMETRY 0x1 +/* enum: TX PD firmware with approximately Siena-compatible behaviour + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_SIENA_COMPAT 0x2 +/* enum: Full featured TX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_FULL_FEATURED 0x3 +/* enum: (deprecated original name for the FULL_FEATURED variant) */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_VSWITCH 0x3 +/* enum: siena_compat variant TX PD firmware using PM rather than MAC + * (Huntington development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_SIENA_COMPAT_PM 0x4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_LOW_LATENCY 0x5 /* enum */ +/* enum: TX PD firmware handling layer 2 only for high packet rate performance + * tests (Medford development only) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_LAYER2_PERF 0x7 +/* enum: Rules engine TX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_RULES_ENGINE 0x8 +/* enum: Custom firmware variant (see SF-119495-PD and bug69716) */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_L3XUDP 0x9 +/* enum: DPDK TX PD production firmware */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_DPDK 0xa +/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE 0xe +/* Hardware capabilities of NIC */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_HW_CAPABILITIES_OFST 12 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_HW_CAPABILITIES_LEN 4 +/* Licensed capabilities */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_LICENSE_CAPABILITIES_OFST 16 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_LICENSE_CAPABILITIES_LEN 4 +/* Second word of flags. Not present on older firmware (check the length). */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FLAGS2_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FLAGS2_LEN 4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V2_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V2_LBN 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V2_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V2_ENCAP_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V2_ENCAP_LBN 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V2_ENCAP_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVQ_TIMER_CTRL_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVQ_TIMER_CTRL_LBN 2 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVQ_TIMER_CTRL_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVENT_CUT_THROUGH_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVENT_CUT_THROUGH_LBN 3 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EVENT_CUT_THROUGH_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_CUT_THROUGH_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_CUT_THROUGH_LBN 4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_CUT_THROUGH_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_VFIFO_ULL_MODE_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_VFIFO_ULL_MODE_LBN 5 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_VFIFO_ULL_MODE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAC_STATS_40G_TX_SIZE_BINS_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAC_STATS_40G_TX_SIZE_BINS_LBN 6 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAC_STATS_40G_TX_SIZE_BINS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_EVQ_TYPE_SUPPORTED_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_EVQ_TYPE_SUPPORTED_LBN 7 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_EVQ_TYPE_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_EVQ_V2_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_EVQ_V2_LBN 7 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_EVQ_V2_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_MAC_TIMESTAMPING_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_MAC_TIMESTAMPING_LBN 8 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_MAC_TIMESTAMPING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TIMESTAMP_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TIMESTAMP_LBN 9 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TIMESTAMP_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_SNIFF_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_SNIFF_LBN 10 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_SNIFF_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_SNIFF_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_SNIFF_LBN 11 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_SNIFF_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MCDI_BACKGROUND_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MCDI_BACKGROUND_LBN 13 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MCDI_BACKGROUND_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MCDI_DB_RETURN_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MCDI_DB_RETURN_LBN 14 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MCDI_DB_RETURN_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CTPIO_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CTPIO_LBN 15 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CTPIO_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TSA_SUPPORT_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TSA_SUPPORT_LBN 16 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TSA_SUPPORT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TSA_BOUND_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TSA_BOUND_LBN 17 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TSA_BOUND_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_SF_ADAPTER_AUTHENTICATION_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_SF_ADAPTER_AUTHENTICATION_LBN 18 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_SF_ADAPTER_AUTHENTICATION_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FILTER_ACTION_FLAG_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FILTER_ACTION_FLAG_LBN 19 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FILTER_ACTION_FLAG_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FILTER_ACTION_MARK_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FILTER_ACTION_MARK_LBN 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FILTER_ACTION_MARK_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EQUAL_STRIDE_SUPER_BUFFER_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EQUAL_STRIDE_SUPER_BUFFER_LBN 21 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EQUAL_STRIDE_SUPER_BUFFER_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EQUAL_STRIDE_PACKED_STREAM_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EQUAL_STRIDE_PACKED_STREAM_LBN 21 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EQUAL_STRIDE_PACKED_STREAM_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_L3XUDP_SUPPORT_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_L3XUDP_SUPPORT_LBN 22 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_L3XUDP_SUPPORT_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FW_SUBVARIANT_NO_TX_CSUM_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FW_SUBVARIANT_NO_TX_CSUM_LBN 23 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FW_SUBVARIANT_NO_TX_CSUM_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VI_SPREADING_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VI_SPREADING_LBN 24 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VI_SPREADING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_HLB_IDLE_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_HLB_IDLE_LBN 25 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RXDP_HLB_IDLE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_RXQ_NO_CONT_EV_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_RXQ_NO_CONT_EV_LBN 26 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_RXQ_NO_CONT_EV_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_RXQ_WITH_BUFFER_SIZE_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_RXQ_WITH_BUFFER_SIZE_LBN 27 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INIT_RXQ_WITH_BUFFER_SIZE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_BUNDLE_UPDATE_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_BUNDLE_UPDATE_LBN 28 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_BUNDLE_UPDATE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V3_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V3_LBN 29 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V3_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_SENSORS_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_SENSORS_LBN 30 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_SENSORS_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NVRAM_UPDATE_POLL_VERIFY_RESULT_OFST 20 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NVRAM_UPDATE_POLL_VERIFY_RESULT_LBN 31 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NVRAM_UPDATE_POLL_VERIFY_RESULT_WIDTH 1 +/* Number of FATSOv2 contexts per datapath supported by this NIC (when + * TX_TSO_V2 == 1). Not present on older firmware (check the length). + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V2_N_CONTEXTS_OFST 24 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_TSO_V2_N_CONTEXTS_LEN 2 +/* One byte per PF containing the number of the external port assigned to this + * PF, indexed by PF number. Special values indicate that a PF is either not + * present or not assigned. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_PFS_TO_PORTS_ASSIGNMENT_OFST 26 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_PFS_TO_PORTS_ASSIGNMENT_LEN 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_PFS_TO_PORTS_ASSIGNMENT_NUM 16 +/* enum: The caller is not permitted to access information on this PF. */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_ACCESS_NOT_PERMITTED 0xff +/* enum: PF does not exist. */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_PF_NOT_PRESENT 0xfe +/* enum: PF does exist but is not assigned to any external port. */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_PF_NOT_ASSIGNED 0xfd +/* enum: This value indicates that PF is assigned, but it cannot be expressed + * in this field. It is intended for a possible future situation where a more + * complex scheme of PFs to ports mapping is being used. The future driver + * should look for a new field supporting the new scheme. The current/old + * driver should treat this value as PF_NOT_ASSIGNED. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_INCOMPATIBLE_ASSIGNMENT 0xfc +/* One byte per PF containing the number of its VFs, indexed by PF number. A + * special value indicates that a PF is not present. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NUM_VFS_PER_PF_OFST 42 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NUM_VFS_PER_PF_LEN 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NUM_VFS_PER_PF_NUM 16 +/* enum: The caller is not permitted to access information on this PF. */ +/* MC_CMD_GET_CAPABILITIES_V10_OUT_ACCESS_NOT_PERMITTED 0xff */ +/* enum: PF does not exist. */ +/* MC_CMD_GET_CAPABILITIES_V10_OUT_PF_NOT_PRESENT 0xfe */ +/* Number of VIs available for each external port */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NUM_VIS_PER_PORT_OFST 58 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NUM_VIS_PER_PORT_LEN 2 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NUM_VIS_PER_PORT_NUM 4 +/* Size of RX descriptor cache expressed as binary logarithm The actual size + * equals (2 ^ RX_DESC_CACHE_SIZE) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_DESC_CACHE_SIZE_OFST 66 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_DESC_CACHE_SIZE_LEN 1 +/* Size of TX descriptor cache expressed as binary logarithm The actual size + * equals (2 ^ TX_DESC_CACHE_SIZE) + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_DESC_CACHE_SIZE_OFST 67 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TX_DESC_CACHE_SIZE_LEN 1 +/* Total number of available PIO buffers */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NUM_PIO_BUFFS_OFST 68 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NUM_PIO_BUFFS_LEN 2 +/* Size of a single PIO buffer */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_SIZE_PIO_BUFF_OFST 70 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_SIZE_PIO_BUFF_LEN 2 +/* On chips later than Medford the amount of address space assigned to each VI + * is configurable. This is a global setting that the driver must query to + * discover the VI to address mapping. Cut-through PIO (CTPIO) is not available + * with 8k VI windows. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VI_WINDOW_MODE_OFST 72 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VI_WINDOW_MODE_LEN 1 +/* enum: Each VI occupies 8k as on Huntington and Medford. PIO is at offset 4k. + * CTPIO is not mapped. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VI_WINDOW_MODE_8K 0x0 +/* enum: Each VI occupies 16k. PIO is at offset 4k. CTPIO is at offset 12k. */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VI_WINDOW_MODE_16K 0x1 +/* enum: Each VI occupies 64k. PIO is at offset 4k. CTPIO is at offset 12k. */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VI_WINDOW_MODE_64K 0x2 +/* Number of vFIFOs per adapter that can be used for VFIFO Stuffing + * (SF-115995-SW) in the present configuration of firmware and port mode. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VFIFO_STUFFING_NUM_VFIFOS_OFST 73 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VFIFO_STUFFING_NUM_VFIFOS_LEN 1 +/* Number of buffers per adapter that can be used for VFIFO Stuffing + * (SF-115995-SW) in the present configuration of firmware and port mode. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VFIFO_STUFFING_NUM_CP_BUFFERS_OFST 74 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VFIFO_STUFFING_NUM_CP_BUFFERS_LEN 2 +/* Entry count in the MAC stats array, including the final GENERATION_END + * entry. For MAC stats DMA, drivers should allocate a buffer large enough to + * hold at least this many 64-bit stats values, if they wish to receive all + * available stats. If the buffer is shorter than MAC_STATS_NUM_STATS * 8, the + * stats array returned will be truncated. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAC_STATS_NUM_STATS_OFST 76 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAC_STATS_NUM_STATS_LEN 2 +/* Maximum supported value for MC_CMD_FILTER_OP_V3/MATCH_MARK_VALUE. This field + * will only be non-zero if MC_CMD_GET_CAPABILITIES/FILTER_ACTION_MARK is set. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FILTER_ACTION_MARK_MAX_OFST 80 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FILTER_ACTION_MARK_MAX_LEN 4 +/* On devices where the INIT_RXQ_WITH_BUFFER_SIZE flag (in + * GET_CAPABILITIES_OUT_V2) is set, drivers have to specify a buffer size when + * they create an RX queue. Due to hardware limitations, only a small number of + * different buffer sizes may be available concurrently. Nonzero entries in + * this array are the sizes of buffers which the system guarantees will be + * available for use. If the list is empty, there are no limitations on + * concurrent buffer sizes. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_GUARANTEED_RX_BUFFER_SIZES_OFST 84 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_GUARANTEED_RX_BUFFER_SIZES_LEN 4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_GUARANTEED_RX_BUFFER_SIZES_NUM 16 +/* Third word of flags. Not present on older firmware (check the length). */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FLAGS3_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_FLAGS3_LEN 4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_WOL_ETHERWAKE_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_WOL_ETHERWAKE_LBN 0 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_WOL_ETHERWAKE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_EVEN_SPREADING_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_EVEN_SPREADING_LBN 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_EVEN_SPREADING_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_SELECTABLE_TABLE_SIZE_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_SELECTABLE_TABLE_SIZE_LBN 2 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_SELECTABLE_TABLE_SIZE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAE_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAE_SUPPORTED_LBN 3 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAE_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VDPA_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VDPA_SUPPORTED_LBN 4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_VDPA_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_VLAN_STRIPPING_PER_ENCAP_RULE_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_VLAN_STRIPPING_PER_ENCAP_RULE_LBN 5 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RX_VLAN_STRIPPING_PER_ENCAP_RULE_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EXTENDED_WIDTH_EVQS_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EXTENDED_WIDTH_EVQS_SUPPORTED_LBN 6 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EXTENDED_WIDTH_EVQS_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_UNSOL_EV_CREDIT_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_UNSOL_EV_CREDIT_SUPPORTED_LBN 7 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_UNSOL_EV_CREDIT_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_ENCAPSULATED_MCDI_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_ENCAPSULATED_MCDI_SUPPORTED_LBN 8 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_ENCAPSULATED_MCDI_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EXTERNAL_MAE_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EXTERNAL_MAE_SUPPORTED_LBN 9 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_EXTERNAL_MAE_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_LBN 10 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_NVRAM_UPDATE_ABORT_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_LBN 11 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_MAE_ACTION_SET_ALLOC_V2_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_STEER_ON_OUTER_SUPPORTED_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_STEER_ON_OUTER_SUPPORTED_LBN 12 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_STEER_ON_OUTER_SUPPORTED_WIDTH 1 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1 +/* These bits are reserved for communicating test-specific capabilities to + * host-side test software. All production drivers should treat this field as + * opaque. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TEST_RESERVED_OFST 152 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TEST_RESERVED_LEN 8 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TEST_RESERVED_LO_OFST 152 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TEST_RESERVED_LO_LEN 4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TEST_RESERVED_LO_LBN 1216 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TEST_RESERVED_LO_WIDTH 32 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TEST_RESERVED_HI_OFST 156 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TEST_RESERVED_HI_LEN 4 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TEST_RESERVED_HI_LBN 1248 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_TEST_RESERVED_HI_WIDTH 32 +/* The minimum size (in table entries) of indirection table to be allocated + * from the pool for an RSS context. Note that the table size used must be a + * power of 2. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_MIN_INDIRECTION_TABLE_SIZE_OFST 160 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_MIN_INDIRECTION_TABLE_SIZE_LEN 4 +/* The maximum size (in table entries) of indirection table to be allocated + * from the pool for an RSS context. Note that the table size used must be a + * power of 2. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_MAX_INDIRECTION_TABLE_SIZE_OFST 164 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_MAX_INDIRECTION_TABLE_SIZE_LEN 4 +/* The maximum number of queues that can be used by an RSS context in exclusive + * mode. In exclusive mode the context has a configurable indirection table and + * a configurable RSS key. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_MAX_INDIRECTION_QUEUES_OFST 168 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_MAX_INDIRECTION_QUEUES_LEN 4 +/* The maximum number of queues that can be used by an RSS context in even- + * spreading mode. In even-spreading mode the context has no indirection table + * but it does have a configurable RSS key. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_MAX_EVEN_SPREADING_QUEUES_OFST 172 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_MAX_EVEN_SPREADING_QUEUES_LEN 4 +/* The total number of RSS contexts supported. Note that the number of + * available contexts using indirection tables is also limited by the + * availability of indirection table space allocated from a common pool. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_NUM_CONTEXTS_OFST 176 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_NUM_CONTEXTS_LEN 4 +/* The total amount of indirection table space that can be shared between RSS + * contexts. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_TABLE_POOL_SIZE_OFST 180 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_RSS_TABLE_POOL_SIZE_LEN 4 +/* A bitmap of the queue sizes the device can provide, where bit N being set + * indicates that 2**N is a valid size. The device may be limited in the number + * of different queue sizes that can exist simultaneously, so a bit being set + * here does not guarantee that an attempt to create a queue of that size will + * succeed. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_SUPPORTED_QUEUE_SIZES_OFST 184 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_SUPPORTED_QUEUE_SIZES_LEN 4 +/* A bitmap of queue sizes that are always available, in the same format as + * SUPPORTED_QUEUE_SIZES. Attempting to create a queue with one of these sizes + * will never fail due to unavailability of the requested size. + */ +#define MC_CMD_GET_CAPABILITIES_V10_OUT_GUARANTEED_QUEUE_SIZES_OFST 188 +#define MC_CMD_GET_CAPABILITIES_V10_OUT_GUARANTEED_QUEUE_SIZES_LEN 4 + /***********************************/ /* MC_CMD_V2_EXTN @@ -15729,7 +18646,7 @@ /* Handle for allocated push I/O buffer. */ #define MC_CMD_LINK_PIOBUF_IN_PIOBUF_HANDLE_OFST 0 #define MC_CMD_LINK_PIOBUF_IN_PIOBUF_HANDLE_LEN 4 -/* Function Local Instance (VI) number. */ +/* Function Local Instance (VI) number which has a TxQ allocated to it. */ #define MC_CMD_LINK_PIOBUF_IN_TXQ_INSTANCE_OFST 4 #define MC_CMD_LINK_PIOBUF_IN_TXQ_INSTANCE_LEN 4 @@ -17303,7 +20220,13 @@ #define MC_CMD_CAP_BLK_READ_OUT_BUFFER_OFST 0 #define MC_CMD_CAP_BLK_READ_OUT_BUFFER_LEN 8 #define MC_CMD_CAP_BLK_READ_OUT_BUFFER_LO_OFST 0 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_LO_LEN 4 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_LO_LBN 0 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_LO_WIDTH 32 #define MC_CMD_CAP_BLK_READ_OUT_BUFFER_HI_OFST 4 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_HI_LEN 4 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_HI_LBN 32 +#define MC_CMD_CAP_BLK_READ_OUT_BUFFER_HI_WIDTH 32 #define MC_CMD_CAP_BLK_READ_OUT_BUFFER_MINNUM 1 #define MC_CMD_CAP_BLK_READ_OUT_BUFFER_MAXNUM 31 #define MC_CMD_CAP_BLK_READ_OUT_BUFFER_MAXNUM_MCDI2 127 @@ -17503,6 +20426,18 @@ #define MC_CMD_GET_FUNCTION_INFO_OUT_VF_OFST 4 #define MC_CMD_GET_FUNCTION_INFO_OUT_VF_LEN 4 +/* MC_CMD_GET_FUNCTION_INFO_OUT_V2 msgresponse */ +#define MC_CMD_GET_FUNCTION_INFO_OUT_V2_LEN 12 +#define MC_CMD_GET_FUNCTION_INFO_OUT_V2_PF_OFST 0 +#define MC_CMD_GET_FUNCTION_INFO_OUT_V2_PF_LEN 4 +#define MC_CMD_GET_FUNCTION_INFO_OUT_V2_VF_OFST 4 +#define MC_CMD_GET_FUNCTION_INFO_OUT_V2_VF_LEN 4 +/* Values from PCIE_INTERFACE enumeration. For NICs with a single interface, or + * in the case of a V1 response, this should be HOST_PRIMARY. + */ +#define MC_CMD_GET_FUNCTION_INFO_OUT_V2_INTF_OFST 8 +#define MC_CMD_GET_FUNCTION_INFO_OUT_V2_INTF_LEN 4 + /***********************************/ /* MC_CMD_ENABLE_OFFLINE_BIST @@ -18570,7 +21505,13 @@ #define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_OFST 24 #define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_LEN 8 #define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_LO_OFST 24 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_LO_LEN 4 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_LO_LBN 192 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_LO_WIDTH 32 #define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_HI_OFST 28 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_HI_LEN 4 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_HI_LBN 224 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_APPS_HI_WIDTH 32 /* reserved for future use */ #define MC_CMD_LICENSING_V3_OUT_RESERVED_0_OFST 32 #define MC_CMD_LICENSING_V3_OUT_RESERVED_0_LEN 24 @@ -18578,7 +21519,13 @@ #define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_OFST 56 #define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_LEN 8 #define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_LO_OFST 56 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_LO_LEN 4 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_LO_LBN 448 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_LO_WIDTH 32 #define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_HI_OFST 60 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_HI_LEN 4 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_HI_LBN 480 +#define MC_CMD_LICENSING_V3_OUT_LICENSED_FEATURES_HI_WIDTH 32 /* reserved for future use */ #define MC_CMD_LICENSING_V3_OUT_RESERVED_1_OFST 64 #define MC_CMD_LICENSING_V3_OUT_RESERVED_1_LEN 24 @@ -18681,7 +21628,13 @@ #define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_OFST 0 #define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_LEN 8 #define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_LO_OFST 0 +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_LO_LEN 4 +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_LO_LBN 0 +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_LO_WIDTH 32 #define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_HI_OFST 4 +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_HI_LEN 4 +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_HI_LBN 32 +#define MC_CMD_GET_LICENSED_V3_APP_STATE_IN_APP_ID_HI_WIDTH 32 /* MC_CMD_GET_LICENSED_V3_APP_STATE_OUT msgresponse */ #define MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LEN 4 @@ -18713,7 +21666,13 @@ #define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_OFST 0 #define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_LEN 8 #define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_LO_OFST 0 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_LO_LEN 4 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_LO_LBN 0 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_LO_WIDTH 32 #define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_HI_OFST 4 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_HI_LEN 4 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_HI_LBN 32 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_IN_FEATURES_HI_WIDTH 32 /* MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT msgresponse */ #define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_LEN 8 @@ -18721,7 +21680,13 @@ #define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_OFST 0 #define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_LEN 8 #define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_LO_OFST 0 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_LO_LEN 4 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_LO_LBN 0 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_LO_WIDTH 32 #define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_HI_OFST 4 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_HI_LEN 4 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_HI_LBN 32 +#define MC_CMD_GET_LICENSED_V3_FEATURE_STATES_OUT_STATES_HI_WIDTH 32 /***********************************/ @@ -18826,7 +21791,13 @@ #define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_OFST 48 #define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_LEN 8 #define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_LO_OFST 48 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_LO_LEN 4 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_LO_LBN 384 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_LO_WIDTH 32 #define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_HI_OFST 52 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_HI_LEN 4 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_HI_LBN 416 +#define MC_CMD_LICENSED_V3_VALIDATE_APP_IN_APP_ID_HI_WIDTH 32 /* MC_CMD_LICENSED_V3_VALIDATE_APP_OUT msgresponse */ #define MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_LEN 116 @@ -18876,7 +21847,13 @@ #define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_OFST 0 #define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_LEN 8 #define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_LO_OFST 0 +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_LO_LEN 4 +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_LO_LBN 0 +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_LO_WIDTH 32 #define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_HI_OFST 4 +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_HI_LEN 4 +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_HI_LBN 32 +#define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_HI_WIDTH 32 /* whether to turn on or turn off the masked features */ #define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_FLAG_OFST 8 #define MC_CMD_LICENSED_V3_MASK_FEATURES_IN_FLAG_LEN 4 @@ -18956,7 +21933,13 @@ #define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_OFST 4 #define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_LEN 8 #define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_LO_OFST 4 +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_LO_LEN 4 +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_LO_LBN 32 +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_LO_WIDTH 32 #define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_HI_OFST 8 +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_HI_LEN 4 +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_HI_LBN 64 +#define MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LICENSED_FEATURES_HI_WIDTH 32 /***********************************/ @@ -19490,6 +22473,22 @@ * SF-117064-DG for background). */ #define MC_CMD_PRIVILEGE_MASK_IN_GRP_ADMIN_TSA_UNBOUND 0x8000 +/* enum: Control the Match-Action Engine if present. See mcdi_mae.yml. */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_MAE 0x10000 +/* enum: This Function/client may call MC_CMD_CLIENT_ALLOC to create new + * dynamic client children of itself. + */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_ALLOC_CLIENT 0x20000 +/* enum: A dynamic client with this privilege may perform all the same DMA + * operations as the function client from which it is descended. + */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_FUNC_DMA 0x40000 +/* enum: A client with this privilege may perform DMA as any PCIe function on + * the device and to on-device DDR. It allows clients to use TX-DESC2CMPT-DESC + * descriptors, and to use TX-SEG-DESC and TX-MEM2MEM-DESC with an address + * space override (i.e. with the ADDR_SPC_EN bit set). + */ +#define MC_CMD_PRIVILEGE_MASK_IN_GRP_ARBITRARY_DMA 0x80000 /* enum: Set this bit to indicate that a new privilege mask is to be set, * otherwise the command will only read the existing mask. */ @@ -20277,7 +23276,8 @@ /* MC_CMD_ALLOCATE_TX_VFIFO_CP_IN msgrequest */ #define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_LEN 20 /* Desired instance. Must be set to a specific instance, which is a function - * local queue index. + * local queue index. The calling client must be the currently-assigned user of + * this VI (see MC_CMD_SET_VI_USER). */ #define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_INSTANCE_OFST 0 #define MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_INSTANCE_LEN 4 @@ -20499,7 +23499,13 @@ #define MC_CMD_GET_RX_PREFIX_ID_IN_FIELDS_OFST 0 #define MC_CMD_GET_RX_PREFIX_ID_IN_FIELDS_LEN 8 #define MC_CMD_GET_RX_PREFIX_ID_IN_FIELDS_LO_OFST 0 +#define MC_CMD_GET_RX_PREFIX_ID_IN_FIELDS_LO_LEN 4 +#define MC_CMD_GET_RX_PREFIX_ID_IN_FIELDS_LO_LBN 0 +#define MC_CMD_GET_RX_PREFIX_ID_IN_FIELDS_LO_WIDTH 32 #define MC_CMD_GET_RX_PREFIX_ID_IN_FIELDS_HI_OFST 4 +#define MC_CMD_GET_RX_PREFIX_ID_IN_FIELDS_HI_LEN 4 +#define MC_CMD_GET_RX_PREFIX_ID_IN_FIELDS_HI_LBN 32 +#define MC_CMD_GET_RX_PREFIX_ID_IN_FIELDS_HI_WIDTH 32 #define MC_CMD_GET_RX_PREFIX_ID_IN_LENGTH_OFST 0 #define MC_CMD_GET_RX_PREFIX_ID_IN_LENGTH_LBN 0 #define MC_CMD_GET_RX_PREFIX_ID_IN_LENGTH_WIDTH 1 @@ -20521,6 +23527,9 @@ #define MC_CMD_GET_RX_PREFIX_ID_IN_USER_MARK_OFST 0 #define MC_CMD_GET_RX_PREFIX_ID_IN_USER_MARK_LBN 6 #define MC_CMD_GET_RX_PREFIX_ID_IN_USER_MARK_WIDTH 1 +#define MC_CMD_GET_RX_PREFIX_ID_IN_INGRESS_MPORT_OFST 0 +#define MC_CMD_GET_RX_PREFIX_ID_IN_INGRESS_MPORT_LBN 7 +#define MC_CMD_GET_RX_PREFIX_ID_IN_INGRESS_MPORT_WIDTH 1 #define MC_CMD_GET_RX_PREFIX_ID_IN_INGRESS_VPORT_OFST 0 #define MC_CMD_GET_RX_PREFIX_ID_IN_INGRESS_VPORT_LBN 7 #define MC_CMD_GET_RX_PREFIX_ID_IN_INGRESS_VPORT_WIDTH 1 @@ -20530,6 +23539,12 @@ #define MC_CMD_GET_RX_PREFIX_ID_IN_VLAN_STRIP_TCI_OFST 0 #define MC_CMD_GET_RX_PREFIX_ID_IN_VLAN_STRIP_TCI_LBN 9 #define MC_CMD_GET_RX_PREFIX_ID_IN_VLAN_STRIP_TCI_WIDTH 1 +#define MC_CMD_GET_RX_PREFIX_ID_IN_VLAN_STRIPPED_OFST 0 +#define MC_CMD_GET_RX_PREFIX_ID_IN_VLAN_STRIPPED_LBN 10 +#define MC_CMD_GET_RX_PREFIX_ID_IN_VLAN_STRIPPED_WIDTH 1 +#define MC_CMD_GET_RX_PREFIX_ID_IN_VSWITCH_STATUS_OFST 0 +#define MC_CMD_GET_RX_PREFIX_ID_IN_VSWITCH_STATUS_LBN 11 +#define MC_CMD_GET_RX_PREFIX_ID_IN_VSWITCH_STATUS_WIDTH 1 /* MC_CMD_GET_RX_PREFIX_ID_OUT msgresponse */ #define MC_CMD_GET_RX_PREFIX_ID_OUT_LENMIN 8 @@ -20575,9 +23590,12 @@ #define RX_PREFIX_FIELD_INFO_PARTIAL_TSTAMP 0x4 /* enum */ #define RX_PREFIX_FIELD_INFO_RSS_HASH 0x5 /* enum */ #define RX_PREFIX_FIELD_INFO_USER_MARK 0x6 /* enum */ +#define RX_PREFIX_FIELD_INFO_INGRESS_MPORT 0x7 /* enum */ #define RX_PREFIX_FIELD_INFO_INGRESS_VPORT 0x7 /* enum */ #define RX_PREFIX_FIELD_INFO_CSUM_FRAME 0x8 /* enum */ #define RX_PREFIX_FIELD_INFO_VLAN_STRIP_TCI 0x9 /* enum */ +#define RX_PREFIX_FIELD_INFO_VLAN_STRIPPED 0xa /* enum */ +#define RX_PREFIX_FIELD_INFO_VSWITCH_STATUS 0xb /* enum */ #define RX_PREFIX_FIELD_INFO_TYPE_LBN 24 #define RX_PREFIX_FIELD_INFO_TYPE_WIDTH 8 @@ -20814,6 +23832,21 @@ #define MC_CMD_GET_NCSI_INFO_STATISTICS_OUT_AENS_SENT_OFST 24 #define MC_CMD_GET_NCSI_INFO_STATISTICS_OUT_AENS_SENT_LEN 4 +/* CLIENT_HANDLE structuredef: A client is an abstract entity that can make + * requests of the device and that can own resources managed by the device. + * Examples of clients include PCIe functions and dynamic clients. A client + * handle is a 32b opaque value used to refer to a client. Further details can + * be found within XN-200418-TC. + */ +#define CLIENT_HANDLE_LEN 4 +#define CLIENT_HANDLE_OPAQUE_OFST 0 +#define CLIENT_HANDLE_OPAQUE_LEN 4 +/* enum: A client handle guaranteed never to refer to a real client. */ +#define CLIENT_HANDLE_NULL 0xffffffff +/* enum: Used to refer to the calling client. */ +#define CLIENT_HANDLE_SELF 0xfffffffe +#define CLIENT_HANDLE_OPAQUE_LBN 0 +#define CLIENT_HANDLE_OPAQUE_WIDTH 32 /* CLOCK_INFO structuredef: Information about a single hardware clock */ #define CLOCK_INFO_LEN 28 @@ -20848,7 +23881,13 @@ #define CLOCK_INFO_FREQUENCY_OFST 4 #define CLOCK_INFO_FREQUENCY_LEN 8 #define CLOCK_INFO_FREQUENCY_LO_OFST 4 +#define CLOCK_INFO_FREQUENCY_LO_LEN 4 +#define CLOCK_INFO_FREQUENCY_LO_LBN 32 +#define CLOCK_INFO_FREQUENCY_LO_WIDTH 32 #define CLOCK_INFO_FREQUENCY_HI_OFST 8 +#define CLOCK_INFO_FREQUENCY_HI_LEN 4 +#define CLOCK_INFO_FREQUENCY_HI_LBN 64 +#define CLOCK_INFO_FREQUENCY_HI_WIDTH 32 #define CLOCK_INFO_FREQUENCY_LBN 32 #define CLOCK_INFO_FREQUENCY_WIDTH 64 /* Human-readable ASCII name for clock, with NUL termination */ @@ -20858,6 +23897,57 @@ #define CLOCK_INFO_NAME_LBN 96 #define CLOCK_INFO_NAME_WIDTH 8 +/* SCHED_CREDIT_CHECK_RESULT structuredef */ +#define SCHED_CREDIT_CHECK_RESULT_LEN 16 +/* The instance of the scheduler. Refer to XN-200389-AW for the location of + * these schedulers in the hardware. + */ +#define SCHED_CREDIT_CHECK_RESULT_SCHED_INSTANCE_OFST 0 +#define SCHED_CREDIT_CHECK_RESULT_SCHED_INSTANCE_LEN 1 +#define SCHED_CREDIT_CHECK_RESULT_HUB_HOST_A 0x0 /* enum */ +#define SCHED_CREDIT_CHECK_RESULT_HUB_NET_A 0x1 /* enum */ +#define SCHED_CREDIT_CHECK_RESULT_HUB_B 0x2 /* enum */ +#define SCHED_CREDIT_CHECK_RESULT_HUB_HOST_C 0x3 /* enum */ +#define SCHED_CREDIT_CHECK_RESULT_HUB_NET_TX 0x4 /* enum */ +#define SCHED_CREDIT_CHECK_RESULT_HUB_HOST_D 0x5 /* enum */ +#define SCHED_CREDIT_CHECK_RESULT_HUB_REPLAY 0x6 /* enum */ +#define SCHED_CREDIT_CHECK_RESULT_DMAC_H2C 0x7 /* enum */ +#define SCHED_CREDIT_CHECK_RESULT_HUB_NET_B 0x8 /* enum */ +#define SCHED_CREDIT_CHECK_RESULT_HUB_NET_REPLAY 0x9 /* enum */ +#define SCHED_CREDIT_CHECK_RESULT_SCHED_INSTANCE_LBN 0 +#define SCHED_CREDIT_CHECK_RESULT_SCHED_INSTANCE_WIDTH 8 +/* The type of node that this result refers to. */ +#define SCHED_CREDIT_CHECK_RESULT_NODE_TYPE_OFST 1 +#define SCHED_CREDIT_CHECK_RESULT_NODE_TYPE_LEN 1 +/* enum: Destination node */ +#define SCHED_CREDIT_CHECK_RESULT_DEST 0x0 +/* enum: Source node */ +#define SCHED_CREDIT_CHECK_RESULT_SOURCE 0x1 +#define SCHED_CREDIT_CHECK_RESULT_NODE_TYPE_LBN 8 +#define SCHED_CREDIT_CHECK_RESULT_NODE_TYPE_WIDTH 8 +/* Level of node in scheduler hierarchy (level 0 is the bottom of the + * hierarchy, increasing towards the root node). + */ +#define SCHED_CREDIT_CHECK_RESULT_NODE_LEVEL_OFST 2 +#define SCHED_CREDIT_CHECK_RESULT_NODE_LEVEL_LEN 2 +#define SCHED_CREDIT_CHECK_RESULT_NODE_LEVEL_LBN 16 +#define SCHED_CREDIT_CHECK_RESULT_NODE_LEVEL_WIDTH 16 +/* Node index */ +#define SCHED_CREDIT_CHECK_RESULT_NODE_INDEX_OFST 4 +#define SCHED_CREDIT_CHECK_RESULT_NODE_INDEX_LEN 4 +#define SCHED_CREDIT_CHECK_RESULT_NODE_INDEX_LBN 32 +#define SCHED_CREDIT_CHECK_RESULT_NODE_INDEX_WIDTH 32 +/* The number of credits the node is expected to have. */ +#define SCHED_CREDIT_CHECK_RESULT_EXPECTED_CREDITS_OFST 8 +#define SCHED_CREDIT_CHECK_RESULT_EXPECTED_CREDITS_LEN 4 +#define SCHED_CREDIT_CHECK_RESULT_EXPECTED_CREDITS_LBN 64 +#define SCHED_CREDIT_CHECK_RESULT_EXPECTED_CREDITS_WIDTH 32 +/* The number of credits the node actually had. */ +#define SCHED_CREDIT_CHECK_RESULT_ACTUAL_CREDITS_OFST 12 +#define SCHED_CREDIT_CHECK_RESULT_ACTUAL_CREDITS_LEN 4 +#define SCHED_CREDIT_CHECK_RESULT_ACTUAL_CREDITS_LBN 96 +#define SCHED_CREDIT_CHECK_RESULT_ACTUAL_CREDITS_WIDTH 32 + /***********************************/ /* MC_CMD_GET_CLOCKS_INFO @@ -20887,7 +23977,19 @@ /***********************************/ /* MC_CMD_VNIC_ENCAP_RULE_ADD - * Add a rule for detecting encapsulations in the VNIC stage. Currently this only affects checksum validation in VNIC RX - on TX the send descriptor explicitly specifies encapsulation. These rules are per-VNIC, i.e. only apply to the current driver. If a rule matches, then the packet is considered to have the corresponding encapsulation type, and the inner packet is parsed. It is up to the driver to ensure that overlapping rules are not inserted. (If a packet would match multiple rules, a random one of them will be used.) A rule with the exact same match criteria may not be inserted twice (EALREADY). Only a limited number MATCH_FLAGS values are supported, use MC_CMD_GET_PARSER_DISP_INFO with OP OP_GET_SUPPORTED_VNIC_ENCAP_RULE_MATCHES to get a list of supported combinations. Each driver may only have a limited set of active rules - returns ENOSPC if the caller's table is full. + * Add a rule for detecting encapsulations in the VNIC stage. Currently this + * only affects checksum validation in VNIC RX - on TX the send descriptor + * explicitly specifies encapsulation. These rules are per-VNIC, i.e. only + * apply to the current driver. If a rule matches, then the packet is + * considered to have the corresponding encapsulation type, and the inner + * packet is parsed. It is up to the driver to ensure that overlapping rules + * are not inserted. (If a packet would match multiple rules, a random one of + * them will be used.) A rule with the exact same match criteria may not be + * inserted twice (EALREADY). Only a limited number MATCH_FLAGS values are + * supported, use MC_CMD_GET_PARSER_DISP_INFO with OP + * OP_GET_SUPPORTED_VNIC_ENCAP_RULE_MATCHES to get a list of supported + * combinations. Each driver may only have a limited set of active rules - + * returns ENOSPC if the caller's table is full. */ #define MC_CMD_VNIC_ENCAP_RULE_ADD 0x16d #undef MC_CMD_0x16d_PRIVILEGE_CTG @@ -20951,6 +24053,12 @@ #define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_STRIP_OUTER_VLAN_OFST 29 #define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_STRIP_OUTER_VLAN_LBN 0 #define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_STRIP_OUTER_VLAN_WIDTH 1 +#define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_RSS_ON_OUTER_OFST 29 +#define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_RSS_ON_OUTER_LBN 1 +#define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_RSS_ON_OUTER_WIDTH 1 +#define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_STEER_ON_OUTER_OFST 29 +#define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_STEER_ON_OUTER_LBN 2 +#define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_STEER_ON_OUTER_WIDTH 1 /* Only if MATCH_DST_PORT is set. Port number as bytes in network order. */ #define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_DST_PORT_OFST 30 #define MC_CMD_VNIC_ENCAP_RULE_ADD_IN_DST_PORT_LEN 2 @@ -20967,7 +24075,9 @@ /***********************************/ /* MC_CMD_VNIC_ENCAP_RULE_REMOVE - * Remove a VNIC encapsulation rule. Packets which would have previously matched the rule will then be considered as unencapsulated. Returns EALREADY if the input HANDLE doesn't correspond to an existing rule. + * Remove a VNIC encapsulation rule. Packets which would have previously + * matched the rule will then be considered as unencapsulated. Returns EALREADY + * if the input HANDLE doesn't correspond to an existing rule. */ #define MC_CMD_VNIC_ENCAP_RULE_REMOVE 0x16e #undef MC_CMD_0x16e_PRIVILEGE_CTG @@ -20983,6 +24093,964 @@ /* MC_CMD_VNIC_ENCAP_RULE_REMOVE_OUT msgresponse */ #define MC_CMD_VNIC_ENCAP_RULE_REMOVE_OUT_LEN 0 +/* UUID structuredef: An RFC4122 standard UUID. The values here are stored in + * the endianness specified by the RFC; users should ignore the broken-out + * fields and instead do straight memory copies to ensure correct ordering. + */ +#define UUID_LEN 16 +#define UUID_TIME_LOW_OFST 0 +#define UUID_TIME_LOW_LEN 4 +#define UUID_TIME_LOW_LBN 0 +#define UUID_TIME_LOW_WIDTH 32 +#define UUID_TIME_MID_OFST 4 +#define UUID_TIME_MID_LEN 2 +#define UUID_TIME_MID_LBN 32 +#define UUID_TIME_MID_WIDTH 16 +#define UUID_TIME_HI_LBN 52 +#define UUID_TIME_HI_WIDTH 12 +#define UUID_VERSION_LBN 48 +#define UUID_VERSION_WIDTH 4 +#define UUID_RESERVED_LBN 64 +#define UUID_RESERVED_WIDTH 2 +#define UUID_CLK_SEQ_LBN 66 +#define UUID_CLK_SEQ_WIDTH 14 +#define UUID_NODE_OFST 10 +#define UUID_NODE_LEN 6 +#define UUID_NODE_LBN 80 +#define UUID_NODE_WIDTH 48 + + +/***********************************/ +/* MC_CMD_PLUGIN_ALLOC + * Create a handle to a datapath plugin's extension. This involves finding a + * currently-loaded plugin offering the given functionality (as identified by + * the UUID) and allocating a handle to track the usage of it. Plugin + * functionality is identified by 'extension' rather than any other identifier + * so that a single plugin bitfile may offer more than one piece of independent + * functionality. If two bitfiles are loaded which both offer the same + * extension, then the metadata is interrogated further to determine which is + * the newest and that is the one opened. See SF-123625-SW for architectural + * detail on datapath plugins. + */ +#define MC_CMD_PLUGIN_ALLOC 0x1ad +#undef MC_CMD_0x1ad_PRIVILEGE_CTG + +#define MC_CMD_0x1ad_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_PLUGIN_ALLOC_IN msgrequest */ +#define MC_CMD_PLUGIN_ALLOC_IN_LEN 24 +/* The functionality requested of the plugin, as a UUID structure */ +#define MC_CMD_PLUGIN_ALLOC_IN_UUID_OFST 0 +#define MC_CMD_PLUGIN_ALLOC_IN_UUID_LEN 16 +/* Additional options for opening the handle */ +#define MC_CMD_PLUGIN_ALLOC_IN_FLAGS_OFST 16 +#define MC_CMD_PLUGIN_ALLOC_IN_FLAGS_LEN 4 +#define MC_CMD_PLUGIN_ALLOC_IN_FLAG_INFO_ONLY_OFST 16 +#define MC_CMD_PLUGIN_ALLOC_IN_FLAG_INFO_ONLY_LBN 0 +#define MC_CMD_PLUGIN_ALLOC_IN_FLAG_INFO_ONLY_WIDTH 1 +#define MC_CMD_PLUGIN_ALLOC_IN_FLAG_ALLOW_DISABLED_OFST 16 +#define MC_CMD_PLUGIN_ALLOC_IN_FLAG_ALLOW_DISABLED_LBN 1 +#define MC_CMD_PLUGIN_ALLOC_IN_FLAG_ALLOW_DISABLED_WIDTH 1 +/* Load the extension only if it is in the specified administrative group. + * Specify ANY to load the extension wherever it is found (if there are + * multiple choices then the extension with the highest MINOR_VER/PATCH_VER + * will be loaded). See MC_CMD_PLUGIN_GET_META_GLOBAL for a description of + * administrative groups. + */ +#define MC_CMD_PLUGIN_ALLOC_IN_ADMIN_GROUP_OFST 20 +#define MC_CMD_PLUGIN_ALLOC_IN_ADMIN_GROUP_LEN 2 +/* enum: Load the extension from any ADMIN_GROUP. */ +#define MC_CMD_PLUGIN_ALLOC_IN_ANY 0xffff +/* Reserved */ +#define MC_CMD_PLUGIN_ALLOC_IN_RESERVED_OFST 22 +#define MC_CMD_PLUGIN_ALLOC_IN_RESERVED_LEN 2 + +/* MC_CMD_PLUGIN_ALLOC_OUT msgresponse */ +#define MC_CMD_PLUGIN_ALLOC_OUT_LEN 4 +/* Unique identifier of this usage */ +#define MC_CMD_PLUGIN_ALLOC_OUT_HANDLE_OFST 0 +#define MC_CMD_PLUGIN_ALLOC_OUT_HANDLE_LEN 4 + + +/***********************************/ +/* MC_CMD_PLUGIN_FREE + * Delete a handle to a plugin's extension. + */ +#define MC_CMD_PLUGIN_FREE 0x1ae +#undef MC_CMD_0x1ae_PRIVILEGE_CTG + +#define MC_CMD_0x1ae_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_PLUGIN_FREE_IN msgrequest */ +#define MC_CMD_PLUGIN_FREE_IN_LEN 4 +/* Handle returned by MC_CMD_PLUGIN_ALLOC_OUT */ +#define MC_CMD_PLUGIN_FREE_IN_HANDLE_OFST 0 +#define MC_CMD_PLUGIN_FREE_IN_HANDLE_LEN 4 + +/* MC_CMD_PLUGIN_FREE_OUT msgresponse */ +#define MC_CMD_PLUGIN_FREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_PLUGIN_GET_META_GLOBAL + * Returns the global metadata applying to the whole plugin extension. See the + * other metadata calls for subtypes of data. + */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL 0x1af +#undef MC_CMD_0x1af_PRIVILEGE_CTG + +#define MC_CMD_0x1af_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_PLUGIN_GET_META_GLOBAL_IN msgrequest */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_IN_LEN 4 +/* Handle returned by MC_CMD_PLUGIN_ALLOC_OUT */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_IN_HANDLE_OFST 0 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_IN_HANDLE_LEN 4 + +/* MC_CMD_PLUGIN_GET_META_GLOBAL_OUT msgresponse */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_LEN 36 +/* Unique identifier of this plugin extension. This is identical to the value + * which was requested when the handle was allocated. + */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_UUID_OFST 0 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_UUID_LEN 16 +/* semver sub-version of this plugin extension */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MINOR_VER_OFST 16 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MINOR_VER_LEN 2 +/* semver micro-version of this plugin extension */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_PATCH_VER_OFST 18 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_PATCH_VER_LEN 2 +/* Number of different messages which can be sent to this extension */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_NUM_MSGS_OFST 20 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_NUM_MSGS_LEN 4 +/* Byte offset within the VI window of the plugin's mapped CSR window. */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_OFFSET_OFST 24 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_OFFSET_LEN 2 +/* Number of bytes mapped through to the plugin's CSRs. 0 if that feature was + * not requested by the plugin (in which case MAPPED_CSR_OFFSET and + * MAPPED_CSR_FLAGS are ignored). + */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_SIZE_OFST 26 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_SIZE_LEN 2 +/* Flags indicating how to perform the CSR window mapping. */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_FLAGS_OFST 28 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_FLAGS_LEN 4 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_FLAG_READ_OFST 28 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_FLAG_READ_LBN 0 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_FLAG_READ_WIDTH 1 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_FLAG_WRITE_OFST 28 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_FLAG_WRITE_LBN 1 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_MAPPED_CSR_FLAG_WRITE_WIDTH 1 +/* Identifier of the set of extensions which all change state together. + * Extensions having the same ADMIN_GROUP will always load and unload at the + * same time. ADMIN_GROUP values themselves are arbitrary (but they contain a + * generation number as an implementation detail to ensure that they're not + * reused rapidly). + */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_ADMIN_GROUP_OFST 32 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_ADMIN_GROUP_LEN 1 +/* Bitshift in MC_CMD_DEVEL_CLIENT_PRIVILEGE_MODIFY's MASK parameters + * corresponding to this extension, i.e. set the bit 1<<PRIVILEGE_BIT to permit + * access to this extension. + */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_PRIVILEGE_BIT_OFST 33 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_PRIVILEGE_BIT_LEN 1 +/* Reserved */ +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_RESERVED_OFST 34 +#define MC_CMD_PLUGIN_GET_META_GLOBAL_OUT_RESERVED_LEN 2 + + +/***********************************/ +/* MC_CMD_PLUGIN_GET_META_PUBLISHER + * Returns metadata supplied by the plugin author which describes this + * extension in a human-readable way. Contrast with + * MC_CMD_PLUGIN_GET_META_GLOBAL, which returns information needed for software + * to operate. + */ +#define MC_CMD_PLUGIN_GET_META_PUBLISHER 0x1b0 +#undef MC_CMD_0x1b0_PRIVILEGE_CTG + +#define MC_CMD_0x1b0_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_PLUGIN_GET_META_PUBLISHER_IN msgrequest */ +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_IN_LEN 12 +/* Handle returned by MC_CMD_PLUGIN_ALLOC_OUT */ +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_IN_HANDLE_OFST 0 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_IN_HANDLE_LEN 4 +/* Category of data to return */ +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_IN_SUBTYPE_OFST 4 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_IN_SUBTYPE_LEN 4 +/* enum: Top-level information about the extension. The returned data is an + * array of key/value pairs using the keys in RFC5013 (Dublin Core) to describe + * the extension. The data is a back-to-back list of zero-terminated strings; + * the even-numbered fields (0,2,4,...) are keys and their following odd- + * numbered fields are the corresponding values. Both keys and values are + * nominally UTF-8. Per RFC5013, the same key may be repeated any number of + * times. Note that all information (including the key/value structure itself + * and the UTF-8 encoding) may have been provided by the plugin author, so + * callers must be cautious about parsing it. Callers should parse only the + * top-level structure to separate out the keys and values; the contents of the + * values is not expected to be machine-readable. + */ +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_IN_EXTENSION_KVS 0x0 +/* Byte position of the data to be returned within the full data block of the + * given SUBTYPE. + */ +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_IN_OFFSET_OFST 8 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_IN_OFFSET_LEN 4 + +/* MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT msgresponse */ +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_LENMIN 4 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_LENMAX 252 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_LEN(num) (4+1*(num)) +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_DATA_NUM(len) (((len)-4)/1) +/* Full length of the data block of the requested SUBTYPE, in bytes. */ +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_TOTAL_SIZE_OFST 0 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_TOTAL_SIZE_LEN 4 +/* The information requested by SUBTYPE. */ +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_DATA_OFST 4 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_DATA_LEN 1 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_DATA_MINNUM 0 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_DATA_MAXNUM 248 +#define MC_CMD_PLUGIN_GET_META_PUBLISHER_OUT_DATA_MAXNUM_MCDI2 1016 + + +/***********************************/ +/* MC_CMD_PLUGIN_GET_META_MSG + * Returns the simple metadata for a specific plugin request message. This + * supplies information necessary for the host to know how to build an + * MC_CMD_PLUGIN_REQ request. + */ +#define MC_CMD_PLUGIN_GET_META_MSG 0x1b1 +#undef MC_CMD_0x1b1_PRIVILEGE_CTG + +#define MC_CMD_0x1b1_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_PLUGIN_GET_META_MSG_IN msgrequest */ +#define MC_CMD_PLUGIN_GET_META_MSG_IN_LEN 8 +/* Handle returned by MC_CMD_PLUGIN_ALLOC_OUT */ +#define MC_CMD_PLUGIN_GET_META_MSG_IN_HANDLE_OFST 0 +#define MC_CMD_PLUGIN_GET_META_MSG_IN_HANDLE_LEN 4 +/* Unique message ID to obtain */ +#define MC_CMD_PLUGIN_GET_META_MSG_IN_ID_OFST 4 +#define MC_CMD_PLUGIN_GET_META_MSG_IN_ID_LEN 4 + +/* MC_CMD_PLUGIN_GET_META_MSG_OUT msgresponse */ +#define MC_CMD_PLUGIN_GET_META_MSG_OUT_LEN 44 +/* Unique message ID. This is the same value as the input parameter; it exists + * to allow future MCDI extensions which enumerate all messages. + */ +#define MC_CMD_PLUGIN_GET_META_MSG_OUT_ID_OFST 0 +#define MC_CMD_PLUGIN_GET_META_MSG_OUT_ID_LEN 4 +/* Packed index number of this message, assigned by the MC to give each message + * a unique ID in an array to allow for more efficient storage/management. + */ +#define MC_CMD_PLUGIN_GET_META_MSG_OUT_INDEX_OFST 4 +#define MC_CMD_PLUGIN_GET_META_MSG_OUT_INDEX_LEN 4 +/* Short human-readable codename for this message. This is conventionally + * formatted as a C identifier in the basic ASCII character set with any spare + * bytes at the end set to 0, however this convention is not enforced by the MC + * so consumers must check for all potential malformations before using it for + * a trusted purpose. + */ +#define MC_CMD_PLUGIN_GET_META_MSG_OUT_NAME_OFST 8 +#define MC_CMD_PLUGIN_GET_META_MSG_OUT_NAME_LEN 32 +/* Number of bytes of data which must be passed from the host kernel to the MC + * for this message's payload, and which are passed back again in the response. + * The MC's plugin metadata loader will have validated that the number of bytes + * specified here will fit in to MC_CMD_PLUGIN_REQ_IN_DATA in a single MCDI + * message. + */ +#define MC_CMD_PLUGIN_GET_META_MSG_OUT_DATA_SIZE_OFST 40 +#define MC_CMD_PLUGIN_GET_META_MSG_OUT_DATA_SIZE_LEN 4 + +/* PLUGIN_EXTENSION structuredef: Used within MC_CMD_PLUGIN_GET_ALL to describe + * an individual extension. + */ +#define PLUGIN_EXTENSION_LEN 20 +#define PLUGIN_EXTENSION_UUID_OFST 0 +#define PLUGIN_EXTENSION_UUID_LEN 16 +#define PLUGIN_EXTENSION_UUID_LBN 0 +#define PLUGIN_EXTENSION_UUID_WIDTH 128 +#define PLUGIN_EXTENSION_ADMIN_GROUP_OFST 16 +#define PLUGIN_EXTENSION_ADMIN_GROUP_LEN 1 +#define PLUGIN_EXTENSION_ADMIN_GROUP_LBN 128 +#define PLUGIN_EXTENSION_ADMIN_GROUP_WIDTH 8 +#define PLUGIN_EXTENSION_FLAG_ENABLED_LBN 136 +#define PLUGIN_EXTENSION_FLAG_ENABLED_WIDTH 1 +#define PLUGIN_EXTENSION_RESERVED_LBN 137 +#define PLUGIN_EXTENSION_RESERVED_WIDTH 23 + + +/***********************************/ +/* MC_CMD_PLUGIN_GET_ALL + * Returns a list of all plugin extensions currently loaded and available. The + * UUIDs returned can be passed to MC_CMD_PLUGIN_ALLOC in order to obtain more + * detailed metadata via the MC_CMD_PLUGIN_GET_META_* family of requests. The + * ADMIN_GROUP field collects how extensions are grouped in to units which are + * loaded/unloaded together; extensions with the same value are in the same + * group. + */ +#define MC_CMD_PLUGIN_GET_ALL 0x1b2 +#undef MC_CMD_0x1b2_PRIVILEGE_CTG + +#define MC_CMD_0x1b2_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_PLUGIN_GET_ALL_IN msgrequest */ +#define MC_CMD_PLUGIN_GET_ALL_IN_LEN 4 +/* Additional options for querying. Note that if neither FLAG_INCLUDE_ENABLED + * nor FLAG_INCLUDE_DISABLED are specified then the result set will be empty. + */ +#define MC_CMD_PLUGIN_GET_ALL_IN_FLAGS_OFST 0 +#define MC_CMD_PLUGIN_GET_ALL_IN_FLAGS_LEN 4 +#define MC_CMD_PLUGIN_GET_ALL_IN_FLAG_INCLUDE_ENABLED_OFST 0 +#define MC_CMD_PLUGIN_GET_ALL_IN_FLAG_INCLUDE_ENABLED_LBN 0 +#define MC_CMD_PLUGIN_GET_ALL_IN_FLAG_INCLUDE_ENABLED_WIDTH 1 +#define MC_CMD_PLUGIN_GET_ALL_IN_FLAG_INCLUDE_DISABLED_OFST 0 +#define MC_CMD_PLUGIN_GET_ALL_IN_FLAG_INCLUDE_DISABLED_LBN 1 +#define MC_CMD_PLUGIN_GET_ALL_IN_FLAG_INCLUDE_DISABLED_WIDTH 1 + +/* MC_CMD_PLUGIN_GET_ALL_OUT msgresponse */ +#define MC_CMD_PLUGIN_GET_ALL_OUT_LENMIN 0 +#define MC_CMD_PLUGIN_GET_ALL_OUT_LENMAX 240 +#define MC_CMD_PLUGIN_GET_ALL_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_PLUGIN_GET_ALL_OUT_LEN(num) (0+20*(num)) +#define MC_CMD_PLUGIN_GET_ALL_OUT_EXTENSIONS_NUM(len) (((len)-0)/20) +/* The list of available plugin extensions, as an array of PLUGIN_EXTENSION + * structs. + */ +#define MC_CMD_PLUGIN_GET_ALL_OUT_EXTENSIONS_OFST 0 +#define MC_CMD_PLUGIN_GET_ALL_OUT_EXTENSIONS_LEN 20 +#define MC_CMD_PLUGIN_GET_ALL_OUT_EXTENSIONS_MINNUM 0 +#define MC_CMD_PLUGIN_GET_ALL_OUT_EXTENSIONS_MAXNUM 12 +#define MC_CMD_PLUGIN_GET_ALL_OUT_EXTENSIONS_MAXNUM_MCDI2 51 + + +/***********************************/ +/* MC_CMD_PLUGIN_REQ + * Send a command to a plugin. A plugin may define an arbitrary number of + * 'messages' which it allows applications on the host system to send, each + * identified by a 32-bit ID. + */ +#define MC_CMD_PLUGIN_REQ 0x1b3 +#undef MC_CMD_0x1b3_PRIVILEGE_CTG + +#define MC_CMD_0x1b3_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_PLUGIN_REQ_IN msgrequest */ +#define MC_CMD_PLUGIN_REQ_IN_LENMIN 8 +#define MC_CMD_PLUGIN_REQ_IN_LENMAX 252 +#define MC_CMD_PLUGIN_REQ_IN_LENMAX_MCDI2 1020 +#define MC_CMD_PLUGIN_REQ_IN_LEN(num) (8+1*(num)) +#define MC_CMD_PLUGIN_REQ_IN_DATA_NUM(len) (((len)-8)/1) +/* Handle returned by MC_CMD_PLUGIN_ALLOC_OUT */ +#define MC_CMD_PLUGIN_REQ_IN_HANDLE_OFST 0 +#define MC_CMD_PLUGIN_REQ_IN_HANDLE_LEN 4 +/* Message ID defined by the plugin author */ +#define MC_CMD_PLUGIN_REQ_IN_ID_OFST 4 +#define MC_CMD_PLUGIN_REQ_IN_ID_LEN 4 +/* Data blob being the parameter to the message. This must be of the length + * specified by MC_CMD_PLUGIN_GET_META_MSG_IN_MCDI_PARAM_SIZE. + */ +#define MC_CMD_PLUGIN_REQ_IN_DATA_OFST 8 +#define MC_CMD_PLUGIN_REQ_IN_DATA_LEN 1 +#define MC_CMD_PLUGIN_REQ_IN_DATA_MINNUM 0 +#define MC_CMD_PLUGIN_REQ_IN_DATA_MAXNUM 244 +#define MC_CMD_PLUGIN_REQ_IN_DATA_MAXNUM_MCDI2 1012 + +/* MC_CMD_PLUGIN_REQ_OUT msgresponse */ +#define MC_CMD_PLUGIN_REQ_OUT_LENMIN 0 +#define MC_CMD_PLUGIN_REQ_OUT_LENMAX 252 +#define MC_CMD_PLUGIN_REQ_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_PLUGIN_REQ_OUT_LEN(num) (0+1*(num)) +#define MC_CMD_PLUGIN_REQ_OUT_DATA_NUM(len) (((len)-0)/1) +/* The input data, as transformed and/or updated by the plugin's eBPF. Will be + * the same size as the input DATA parameter. + */ +#define MC_CMD_PLUGIN_REQ_OUT_DATA_OFST 0 +#define MC_CMD_PLUGIN_REQ_OUT_DATA_LEN 1 +#define MC_CMD_PLUGIN_REQ_OUT_DATA_MINNUM 0 +#define MC_CMD_PLUGIN_REQ_OUT_DATA_MAXNUM 252 +#define MC_CMD_PLUGIN_REQ_OUT_DATA_MAXNUM_MCDI2 1020 + +/* DESC_ADDR_REGION structuredef: Describes a contiguous region of DESC_ADDR + * space that maps to a contiguous region of TRGT_ADDR space. Addresses + * DESC_ADDR in the range [DESC_ADDR_BASE:DESC_ADDR_BASE + 1 << + * WINDOW_SIZE_LOG2) map to TRGT_ADDR = DESC_ADDR - DESC_ADDR_BASE + + * TRGT_ADDR_BASE. + */ +#define DESC_ADDR_REGION_LEN 32 +/* The start of the region in DESC_ADDR space. */ +#define DESC_ADDR_REGION_DESC_ADDR_BASE_OFST 0 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_LEN 8 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_LO_OFST 0 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_LO_LEN 4 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_LO_LBN 0 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_LO_WIDTH 32 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_HI_OFST 4 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_HI_LEN 4 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_HI_LBN 32 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_HI_WIDTH 32 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_LBN 0 +#define DESC_ADDR_REGION_DESC_ADDR_BASE_WIDTH 64 +/* The start of the region in TRGT_ADDR space. Drivers can set this via + * MC_CMD_SET_DESC_ADDR_REGIONS. + */ +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_OFST 8 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_LEN 8 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_LO_OFST 8 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_LO_LEN 4 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_LO_LBN 64 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_LO_WIDTH 32 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_HI_OFST 12 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_HI_LEN 4 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_HI_LBN 96 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_HI_WIDTH 32 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_LBN 64 +#define DESC_ADDR_REGION_TRGT_ADDR_BASE_WIDTH 64 +/* The size of the region. */ +#define DESC_ADDR_REGION_WINDOW_SIZE_LOG2_OFST 16 +#define DESC_ADDR_REGION_WINDOW_SIZE_LOG2_LEN 4 +#define DESC_ADDR_REGION_WINDOW_SIZE_LOG2_LBN 128 +#define DESC_ADDR_REGION_WINDOW_SIZE_LOG2_WIDTH 32 +/* The alignment restriction on TRGT_ADDR. TRGT_ADDR values set by the driver + * must be a multiple of 1 << TRGT_ADDR_ALIGN_LOG2. + */ +#define DESC_ADDR_REGION_TRGT_ADDR_ALIGN_LOG2_OFST 20 +#define DESC_ADDR_REGION_TRGT_ADDR_ALIGN_LOG2_LEN 4 +#define DESC_ADDR_REGION_TRGT_ADDR_ALIGN_LOG2_LBN 160 +#define DESC_ADDR_REGION_TRGT_ADDR_ALIGN_LOG2_WIDTH 32 +#define DESC_ADDR_REGION_RSVD_OFST 24 +#define DESC_ADDR_REGION_RSVD_LEN 8 +#define DESC_ADDR_REGION_RSVD_LO_OFST 24 +#define DESC_ADDR_REGION_RSVD_LO_LEN 4 +#define DESC_ADDR_REGION_RSVD_LO_LBN 192 +#define DESC_ADDR_REGION_RSVD_LO_WIDTH 32 +#define DESC_ADDR_REGION_RSVD_HI_OFST 28 +#define DESC_ADDR_REGION_RSVD_HI_LEN 4 +#define DESC_ADDR_REGION_RSVD_HI_LBN 224 +#define DESC_ADDR_REGION_RSVD_HI_WIDTH 32 +#define DESC_ADDR_REGION_RSVD_LBN 192 +#define DESC_ADDR_REGION_RSVD_WIDTH 64 + + +/***********************************/ +/* MC_CMD_GET_DESC_ADDR_INFO + * Returns a description of the mapping from DESC_ADDR to TRGT_ADDR for the calling function's address space. + */ +#define MC_CMD_GET_DESC_ADDR_INFO 0x1b7 +#undef MC_CMD_0x1b7_PRIVILEGE_CTG + +#define MC_CMD_0x1b7_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_DESC_ADDR_INFO_IN msgrequest */ +#define MC_CMD_GET_DESC_ADDR_INFO_IN_LEN 0 + +/* MC_CMD_GET_DESC_ADDR_INFO_OUT msgresponse */ +#define MC_CMD_GET_DESC_ADDR_INFO_OUT_LEN 4 +/* The type of mapping; see SF-nnnnnn-xx (EF100 driver writer's guide, once + * written) for details of each type. + */ +#define MC_CMD_GET_DESC_ADDR_INFO_OUT_MAPPING_TYPE_OFST 0 +#define MC_CMD_GET_DESC_ADDR_INFO_OUT_MAPPING_TYPE_LEN 4 +/* enum: TRGT_ADDR = DESC_ADDR */ +#define MC_CMD_GET_DESC_ADDR_INFO_OUT_MAPPING_FLAT 0x0 +/* enum: DESC_ADDR has one or more regions that map into TRGT_ADDR. The base + * TRGT_ADDR for each region is programmable via MCDI. + */ +#define MC_CMD_GET_DESC_ADDR_INFO_OUT_MAPPING_REGIONED 0x1 + + +/***********************************/ +/* MC_CMD_GET_DESC_ADDR_REGIONS + * Returns a list of the DESC_ADDR regions for the calling function's address space. Only valid if that function's address space has the REGIONED mapping from DESC_ADDR to TRGT_ADDR. + */ +#define MC_CMD_GET_DESC_ADDR_REGIONS 0x1b8 +#undef MC_CMD_0x1b8_PRIVILEGE_CTG + +#define MC_CMD_0x1b8_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_DESC_ADDR_REGIONS_IN msgrequest */ +#define MC_CMD_GET_DESC_ADDR_REGIONS_IN_LEN 0 + +/* MC_CMD_GET_DESC_ADDR_REGIONS_OUT msgresponse */ +#define MC_CMD_GET_DESC_ADDR_REGIONS_OUT_LENMIN 32 +#define MC_CMD_GET_DESC_ADDR_REGIONS_OUT_LENMAX 224 +#define MC_CMD_GET_DESC_ADDR_REGIONS_OUT_LENMAX_MCDI2 992 +#define MC_CMD_GET_DESC_ADDR_REGIONS_OUT_LEN(num) (0+32*(num)) +#define MC_CMD_GET_DESC_ADDR_REGIONS_OUT_REGIONS_NUM(len) (((len)-0)/32) +/* An array of DESC_ADDR_REGION strutures. The number of entries in the array + * indicates the number of available regions. + */ +#define MC_CMD_GET_DESC_ADDR_REGIONS_OUT_REGIONS_OFST 0 +#define MC_CMD_GET_DESC_ADDR_REGIONS_OUT_REGIONS_LEN 32 +#define MC_CMD_GET_DESC_ADDR_REGIONS_OUT_REGIONS_MINNUM 1 +#define MC_CMD_GET_DESC_ADDR_REGIONS_OUT_REGIONS_MAXNUM 7 +#define MC_CMD_GET_DESC_ADDR_REGIONS_OUT_REGIONS_MAXNUM_MCDI2 31 + + +/***********************************/ +/* MC_CMD_SET_DESC_ADDR_REGIONS + * Set the base TRGT_ADDR for a set of DESC_ADDR regions for the calling function's address space. Only valid if that function's address space had the REGIONED mapping from DESC_ADDR to TRGT_ADDR. + */ +#define MC_CMD_SET_DESC_ADDR_REGIONS 0x1b9 +#undef MC_CMD_0x1b9_PRIVILEGE_CTG + +#define MC_CMD_0x1b9_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_SET_DESC_ADDR_REGIONS_IN msgrequest */ +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_LENMIN 16 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_LENMAX 248 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_LENMAX_MCDI2 1016 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_LEN(num) (8+8*(num)) +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_NUM(len) (((len)-8)/8) +/* A bitmask indicating which regions should have their base TRGT_ADDR updated. + * To update the base TRGR_ADDR for a DESC_ADDR region, the corresponding bit + * should be set to 1. + */ +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_SET_REGION_MASK_OFST 0 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_SET_REGION_MASK_LEN 4 +/* Reserved field; must be set to zero. */ +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_RSVD_OFST 4 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_RSVD_LEN 4 +/* An array of values used to updated the base TRGT_ADDR for DESC_ADDR regions. + * Array indices corresponding to region numbers (i.e. the array is sparse, and + * included entries for regions even if the corresponding SET_REGION_MASK bit + * is zero). + */ +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_OFST 8 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_LEN 8 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_LO_OFST 8 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_LO_LEN 4 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_LO_LBN 64 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_LO_WIDTH 32 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_HI_OFST 12 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_HI_LEN 4 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_HI_LBN 96 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_HI_WIDTH 32 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_MINNUM 1 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_MAXNUM 30 +#define MC_CMD_SET_DESC_ADDR_REGIONS_IN_TRGT_ADDR_BASE_MAXNUM_MCDI2 126 + +/* MC_CMD_SET_DESC_ADDR_REGIONS_OUT msgresponse */ +#define MC_CMD_SET_DESC_ADDR_REGIONS_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_CLIENT_CMD + * Execute an arbitrary MCDI command on behalf of a different client. The + * consequences of the command (e.g. ownership of any resources created) apply + * to the indicated client rather than the function client which actually sent + * this command. All inherent permission checks are also performed on the + * indicated client. The given client must be a descendant of the requestor. + * The command to be proxied follows immediately afterward in the host buffer + * (or on the UART). Chaining multiple MC_CMD_CLIENT_CMD is unnecessary and not + * supported. New dynamic clients may be created with MC_CMD_CLIENT_ALLOC. + */ +#define MC_CMD_CLIENT_CMD 0x1ba +#undef MC_CMD_0x1ba_PRIVILEGE_CTG + +#define MC_CMD_0x1ba_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_CLIENT_CMD_IN msgrequest */ +#define MC_CMD_CLIENT_CMD_IN_LEN 4 +/* The client as which to execute the following command. */ +#define MC_CMD_CLIENT_CMD_IN_CLIENT_ID_OFST 0 +#define MC_CMD_CLIENT_CMD_IN_CLIENT_ID_LEN 4 + +/* MC_CMD_CLIENT_CMD_OUT msgresponse */ +#define MC_CMD_CLIENT_CMD_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_CLIENT_ALLOC + * Create a new client object. Clients are a system for delineating NIC + * resource ownership, such that groups of resources may be torn down as a + * unit. See also MC_CMD_CLIENT_CMD. See XN-200265-TC for background, concepts + * and a glossary. Clients created by this command are known as "dynamic + * clients". The newly-created client is a child of the client which sent this + * command. The caller must have the GRP_ALLOC_CLIENT privilege. The new client + * initially has no permission to do anything; see + * MC_CMD_DEVEL_CLIENT_PRIVILEGE_MODIFY. + */ +#define MC_CMD_CLIENT_ALLOC 0x1bb +#undef MC_CMD_0x1bb_PRIVILEGE_CTG + +#define MC_CMD_0x1bb_PRIVILEGE_CTG SRIOV_CTG_ALLOC_CLIENT + +/* MC_CMD_CLIENT_ALLOC_IN msgrequest */ +#define MC_CMD_CLIENT_ALLOC_IN_LEN 0 + +/* MC_CMD_CLIENT_ALLOC_OUT msgresponse */ +#define MC_CMD_CLIENT_ALLOC_OUT_LEN 4 +/* The ID of the new client object which has been created. */ +#define MC_CMD_CLIENT_ALLOC_OUT_CLIENT_ID_OFST 0 +#define MC_CMD_CLIENT_ALLOC_OUT_CLIENT_ID_LEN 4 + + +/***********************************/ +/* MC_CMD_CLIENT_FREE + * Destroy and release an existing client object. All resources owned by that + * client (including its child clients, and thus all resources owned by the + * entire family tree) are freed. + */ +#define MC_CMD_CLIENT_FREE 0x1bc +#undef MC_CMD_0x1bc_PRIVILEGE_CTG + +#define MC_CMD_0x1bc_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_CLIENT_FREE_IN msgrequest */ +#define MC_CMD_CLIENT_FREE_IN_LEN 4 +/* The ID of the client to be freed. This client must be a descendant of the + * requestor. A client cannot free itself. + */ +#define MC_CMD_CLIENT_FREE_IN_CLIENT_ID_OFST 0 +#define MC_CMD_CLIENT_FREE_IN_CLIENT_ID_LEN 4 + +/* MC_CMD_CLIENT_FREE_OUT msgresponse */ +#define MC_CMD_CLIENT_FREE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_SET_VI_USER + * Assign partial rights over this VI to another client. VIs have an 'owner' + * and a 'user'. The owner is the client which allocated the VI + * (MC_CMD_ALLOC_VIS) and cannot be changed. The user is the client which has + * permission to create queues and other resources on that VI. Initially + * user==owner, but the user can be changed by this command; the resources thus + * created are then owned by the user-client. Only the VI owner can call this + * command, and the request will fail if there are any outstanding child + * resources (e.g. queues) currently allocated from this VI. + */ +#define MC_CMD_SET_VI_USER 0x1be +#undef MC_CMD_0x1be_PRIVILEGE_CTG + +#define MC_CMD_0x1be_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_SET_VI_USER_IN msgrequest */ +#define MC_CMD_SET_VI_USER_IN_LEN 8 +/* Function-relative VI number to modify. */ +#define MC_CMD_SET_VI_USER_IN_INSTANCE_OFST 0 +#define MC_CMD_SET_VI_USER_IN_INSTANCE_LEN 4 +/* Client ID to become the new user. This must be a descendant of the owning + * client, the owning client itself, or the special value MC_CMD_CLIENT_ID_SELF + * which is synonymous with the owning client. + */ +#define MC_CMD_SET_VI_USER_IN_CLIENT_ID_OFST 4 +#define MC_CMD_SET_VI_USER_IN_CLIENT_ID_LEN 4 + +/* MC_CMD_SET_VI_USER_OUT msgresponse */ +#define MC_CMD_SET_VI_USER_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_CLIENT_MAC_ADDRESSES + * A device reports a set of MAC addresses for each client to use, known as the + * "permanent MAC addresses". Those MAC addresses are provided by the client's + * administrator, e.g. via MC_CMD_SET_CLIENT_MAC_ADDRESSES, and are intended as + * a hint to that client which MAC address its administrator would like to use + * to identity itself. This API exists solely to allow communication of MAC + * address from administrator to adminstree, and has no inherent interaction + * with switching within the device. There is no guarantee that a client will + * be able to send traffic with a source MAC address taken from the list of MAC + * address reported, nor is there a guarantee that a client will be able to + * resource traffic with a destination MAC taken from the list of MAC + * addresses. Likewise, there is no guarantee that a client will not be able to + * use a MAC address not present in the list. Restrictions on switching are + * controlled either through the EVB API if operating in EVB mode, or via MAE + * rules if host software is directly managing the MAE. In order to allow + * tenants to use this API whilst a provider is using the EVB API, the MAC + * addresses reported by MC_CMD_GET_CLIENT_MAC_ADDRESSES will be augmented with + * any MAC addresses associated with the vPort assigned to the caller. In order + * to allow tenants to use the EVB API whilst a provider is using this API, if + * a client queries the MAC addresses for a vPort using the host_evb_port_id + * EVB_PORT_ASSIGNED, that list of MAC addresses will be augmented with the MAC + * addresses assigned to the calling client. This query can either be explicit + * (i.e. MC_CMD_VPORT_GET_MAC_ADDRESSES) or implicit (e.g. creation of a + * vAdaptor with a NULL/automatic MAC address). Changing the MAC address on a + * vAdaptor only affects VNIC steering filters; it has no effect on the MAC + * addresses assigned to the vAdaptor's owner. VirtIO clients behave as EVB + * clients. On VirtIO device reset, a vAdaptor is created with an automatic MAC + * address. Querying the VirtIO device's MAC address queries the underlying + * vAdaptor's MAC address. Setting the VirtIO device's MAC address sets the + * underlying vAdaptor's MAC addresses. + */ +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES 0x1c4 +#undef MC_CMD_0x1c4_PRIVILEGE_CTG + +#define MC_CMD_0x1c4_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_CLIENT_MAC_ADDRESSES_IN msgrequest */ +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_IN_LEN 4 +/* A handle for the client for whom MAC address should be obtained. Use + * CLIENT_HANDLE_SELF to obtain the MAC addresses assigned to the calling + * client. + */ +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_IN_CLIENT_HANDLE_OFST 0 +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_IN_CLIENT_HANDLE_LEN 4 + +/* MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT msgresponse */ +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT_LENMIN 0 +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT_LENMAX 252 +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT_LEN(num) (0+6*(num)) +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT_MAC_ADDRS_NUM(len) (((len)-0)/6) +/* An array of MAC addresses assigned to the client. */ +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT_MAC_ADDRS_OFST 0 +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT_MAC_ADDRS_LEN 6 +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT_MAC_ADDRS_MINNUM 0 +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT_MAC_ADDRS_MAXNUM 42 +#define MC_CMD_GET_CLIENT_MAC_ADDRESSES_OUT_MAC_ADDRS_MAXNUM_MCDI2 170 + + +/***********************************/ +/* MC_CMD_SET_CLIENT_MAC_ADDRESSES + * Set the permanent MAC addresses for a client. The caller must by an + * administrator of the target client. See MC_CMD_GET_CLIENT_MAC_ADDRESSES for + * additional detail. + */ +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES 0x1c5 +#undef MC_CMD_0x1c5_PRIVILEGE_CTG + +#define MC_CMD_0x1c5_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN msgrequest */ +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_LENMIN 4 +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_LENMAX 250 +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_LENMAX_MCDI2 1018 +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_LEN(num) (4+6*(num)) +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_MAC_ADDRS_NUM(len) (((len)-4)/6) +/* A handle for the client for whom MAC addresses should be set */ +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_CLIENT_HANDLE_OFST 0 +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_CLIENT_HANDLE_LEN 4 +/* An array of MAC addresses to assign to the client. */ +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_MAC_ADDRS_OFST 4 +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_MAC_ADDRS_LEN 6 +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_MAC_ADDRS_MINNUM 0 +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_MAC_ADDRS_MAXNUM 41 +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_IN_MAC_ADDRS_MAXNUM_MCDI2 169 + +/* MC_CMD_SET_CLIENT_MAC_ADDRESSES_OUT msgresponse */ +#define MC_CMD_SET_CLIENT_MAC_ADDRESSES_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_GET_BOARD_ATTR + * Retrieve physical build-level board attributes as configured at + * manufacturing stage. Fields originate from EEPROM and per-platform constants + * in firmware. Fields are used in development to identify/ differentiate + * boards based on build levels/parameters, and also in manufacturing to cross + * check "what was programmed in manufacturing" is same as "what firmware + * thinks has been programmed" as there are two layers to translation within + * firmware before the attributes reach this MCDI handler. Some parameters are + * retrieved as part of other commands and therefore not replicated here. See + * GET_VERSION_OUT. + */ +#define MC_CMD_GET_BOARD_ATTR 0x1c6 +#undef MC_CMD_0x1c6_PRIVILEGE_CTG + +#define MC_CMD_0x1c6_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_BOARD_ATTR_IN msgrequest */ +#define MC_CMD_GET_BOARD_ATTR_IN_LEN 0 + +/* MC_CMD_GET_BOARD_ATTR_OUT msgresponse */ +#define MC_CMD_GET_BOARD_ATTR_OUT_LEN 16 +/* Defines board capabilities and validity of attributes returned in this + * response-message. + */ +#define MC_CMD_GET_BOARD_ATTR_OUT_FLAGS_OFST 0 +#define MC_CMD_GET_BOARD_ATTR_OUT_FLAGS_LEN 4 +#define MC_CMD_GET_BOARD_ATTR_OUT_HAS_FAN_OFST 0 +#define MC_CMD_GET_BOARD_ATTR_OUT_HAS_FAN_LBN 0 +#define MC_CMD_GET_BOARD_ATTR_OUT_HAS_FAN_WIDTH 1 +#define MC_CMD_GET_BOARD_ATTR_OUT_HAS_SOC_OFST 0 +#define MC_CMD_GET_BOARD_ATTR_OUT_HAS_SOC_LBN 1 +#define MC_CMD_GET_BOARD_ATTR_OUT_HAS_SOC_WIDTH 1 +#define MC_CMD_GET_BOARD_ATTR_OUT_HAS_AUX_POWER_OFST 0 +#define MC_CMD_GET_BOARD_ATTR_OUT_HAS_AUX_POWER_LBN 2 +#define MC_CMD_GET_BOARD_ATTR_OUT_HAS_AUX_POWER_WIDTH 1 +#define MC_CMD_GET_BOARD_ATTR_OUT_ATTRIBUTES_OFST 4 +#define MC_CMD_GET_BOARD_ATTR_OUT_ATTRIBUTES_LEN 4 +#define MC_CMD_GET_BOARD_ATTR_OUT_SOC_EE_OFST 4 +#define MC_CMD_GET_BOARD_ATTR_OUT_SOC_EE_LBN 0 +#define MC_CMD_GET_BOARD_ATTR_OUT_SOC_EE_WIDTH 1 +#define MC_CMD_GET_BOARD_ATTR_OUT_SUC_EE_OFST 4 +#define MC_CMD_GET_BOARD_ATTR_OUT_SUC_EE_LBN 1 +#define MC_CMD_GET_BOARD_ATTR_OUT_SUC_EE_WIDTH 1 +#define MC_CMD_GET_BOARD_ATTR_OUT_FPGA_VOLTAGES_SUPPORTED_OFST 4 +#define MC_CMD_GET_BOARD_ATTR_OUT_FPGA_VOLTAGES_SUPPORTED_LBN 16 +#define MC_CMD_GET_BOARD_ATTR_OUT_FPGA_VOLTAGES_SUPPORTED_WIDTH 8 +/* enum: The FPGA voltage on the adapter can be set to low */ +#define MC_CMD_FPGA_VOLTAGE_LOW 0x0 +/* enum: The FPGA voltage on the adapter can be set to regular */ +#define MC_CMD_FPGA_VOLTAGE_REG 0x1 +/* enum: The FPGA voltage on the adapter can be set to high */ +#define MC_CMD_FPGA_VOLTAGE_HIGH 0x2 +#define MC_CMD_GET_BOARD_ATTR_OUT_CAGE_COUNT_OFST 4 +#define MC_CMD_GET_BOARD_ATTR_OUT_CAGE_COUNT_LBN 24 +#define MC_CMD_GET_BOARD_ATTR_OUT_CAGE_COUNT_WIDTH 8 +/* An array of cage types on the board */ +#define MC_CMD_GET_BOARD_ATTR_OUT_CAGE_TYPE_OFST 8 +#define MC_CMD_GET_BOARD_ATTR_OUT_CAGE_TYPE_LEN 1 +#define MC_CMD_GET_BOARD_ATTR_OUT_CAGE_TYPE_NUM 8 +/* enum: The cages are not known */ +#define MC_CMD_GET_BOARD_ATTR_OUT_CAGE_TYPE_UNKNOWN 0x0 +/* enum: The cages are SFP/SFP+ */ +#define MC_CMD_GET_BOARD_ATTR_OUT_CAGE_TYPE_SFP 0x1 +/* enum: The cages are QSFP/QSFP+ */ +#define MC_CMD_GET_BOARD_ATTR_OUT_CAGE_TYPE_QSFP 0x2 + + +/***********************************/ +/* MC_CMD_GET_SOC_STATE + * Retrieve current state of the System-on-Chip. This command is valid when + * MC_CMD_GET_BOARD_ATTR:HAS_SOC is set. + */ +#define MC_CMD_GET_SOC_STATE 0x1c7 +#undef MC_CMD_0x1c7_PRIVILEGE_CTG + +#define MC_CMD_0x1c7_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_SOC_STATE_IN msgrequest */ +#define MC_CMD_GET_SOC_STATE_IN_LEN 0 + +/* MC_CMD_GET_SOC_STATE_OUT msgresponse */ +#define MC_CMD_GET_SOC_STATE_OUT_LEN 12 +/* Status flags for the SoC */ +#define MC_CMD_GET_SOC_STATE_OUT_FLAGS_OFST 0 +#define MC_CMD_GET_SOC_STATE_OUT_FLAGS_LEN 4 +#define MC_CMD_GET_SOC_STATE_OUT_SHOULD_THROTTLE_OFST 0 +#define MC_CMD_GET_SOC_STATE_OUT_SHOULD_THROTTLE_LBN 0 +#define MC_CMD_GET_SOC_STATE_OUT_SHOULD_THROTTLE_WIDTH 1 +#define MC_CMD_GET_SOC_STATE_OUT_OS_RECOVERY_REQUIRED_OFST 0 +#define MC_CMD_GET_SOC_STATE_OUT_OS_RECOVERY_REQUIRED_LBN 1 +#define MC_CMD_GET_SOC_STATE_OUT_OS_RECOVERY_REQUIRED_WIDTH 1 +#define MC_CMD_GET_SOC_STATE_OUT_WDT_FIRED_OFST 0 +#define MC_CMD_GET_SOC_STATE_OUT_WDT_FIRED_LBN 2 +#define MC_CMD_GET_SOC_STATE_OUT_WDT_FIRED_WIDTH 1 +/* Status fields for the SoC */ +#define MC_CMD_GET_SOC_STATE_OUT_ATTRIBUTES_OFST 4 +#define MC_CMD_GET_SOC_STATE_OUT_ATTRIBUTES_LEN 4 +#define MC_CMD_GET_SOC_STATE_OUT_RUN_STATE_OFST 4 +#define MC_CMD_GET_SOC_STATE_OUT_RUN_STATE_LBN 0 +#define MC_CMD_GET_SOC_STATE_OUT_RUN_STATE_WIDTH 8 +/* enum: Power on (set by SUC on power up) */ +#define MC_CMD_GET_SOC_STATE_OUT_SOC_BOOT 0x0 +/* enum: Running bootloader */ +#define MC_CMD_GET_SOC_STATE_OUT_SOC_BOOTLOADER 0x1 +/* enum: Bootloader has started OS. OS is booting */ +#define MC_CMD_GET_SOC_STATE_OUT_SOC_OS_START 0x2 +/* enum: OS is running */ +#define MC_CMD_GET_SOC_STATE_OUT_SOC_OS_RUNNING 0x3 +/* enum: Maintenance OS is running */ +#define MC_CMD_GET_SOC_STATE_OUT_SOC_OS_MAINTENANCE 0x4 +/* Number of SoC resets since power on */ +#define MC_CMD_GET_SOC_STATE_OUT_RESET_COUNT_OFST 8 +#define MC_CMD_GET_SOC_STATE_OUT_RESET_COUNT_LEN 4 + + +/***********************************/ +/* MC_CMD_CHECK_SCHEDULER_CREDITS + * For debugging purposes. For each source and destination node in the hardware + * schedulers, check whether the number of credits is as it should be. This + * should only be used when the NIC is idle, because collection is not atomic + * and because the expected credit counts are only meaningful when no traffic + * is flowing. + */ +#define MC_CMD_CHECK_SCHEDULER_CREDITS 0x1c8 +#undef MC_CMD_0x1c8_PRIVILEGE_CTG + +#define MC_CMD_0x1c8_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_CHECK_SCHEDULER_CREDITS_IN msgrequest */ +#define MC_CMD_CHECK_SCHEDULER_CREDITS_IN_LEN 8 +/* Flags for the request */ +#define MC_CMD_CHECK_SCHEDULER_CREDITS_IN_FLAGS_OFST 0 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_IN_FLAGS_LEN 4 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_IN_REPORT_ALL_OFST 0 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_IN_REPORT_ALL_LBN 0 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_IN_REPORT_ALL_WIDTH 1 +/* If there are too many results to fit into an MCDI response, they're split + * into pages. This field specifies which (0-indexed) page to request. A + * request with PAGE=0 will snapshot the results, and subsequent requests with + * PAGE>0 will return data from the most recent snapshot. The GENERATION field + * in the response allows callers to verify that all responses correspond to + * the same snapshot. + */ +#define MC_CMD_CHECK_SCHEDULER_CREDITS_IN_PAGE_OFST 4 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_IN_PAGE_LEN 4 + +/* MC_CMD_CHECK_SCHEDULER_CREDITS_OUT msgresponse */ +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_LENMIN 16 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_LENMAX 240 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_LENMAX_MCDI2 1008 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_LEN(num) (16+16*(num)) +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_RESULTS_NUM(len) (((len)-16)/16) +/* The total number of results (across all pages). */ +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_TOTAL_RESULTS_OFST 0 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_TOTAL_RESULTS_LEN 4 +/* The number of pages that the response is split across. */ +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_NUM_PAGES_OFST 4 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_NUM_PAGES_LEN 4 +/* The number of results in this response. */ +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_RESULTS_THIS_PAGE_OFST 8 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_RESULTS_THIS_PAGE_LEN 4 +/* Result generation count. Incremented any time a request is made with PAGE=0. + */ +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_GENERATION_OFST 12 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_GENERATION_LEN 4 +/* The results, as an array of SCHED_CREDIT_CHECK_RESULT structures. */ +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_RESULTS_OFST 16 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_RESULTS_LEN 16 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_RESULTS_MINNUM 0 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_RESULTS_MAXNUM 14 +#define MC_CMD_CHECK_SCHEDULER_CREDITS_OUT_RESULTS_MAXNUM_MCDI2 62 + + +/***********************************/ +/* MC_CMD_TXQ_STATS + * Query per-TXQ statistics. + */ +#define MC_CMD_TXQ_STATS 0x1d5 +#undef MC_CMD_0x1d5_PRIVILEGE_CTG + +#define MC_CMD_0x1d5_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TXQ_STATS_IN msgrequest */ +#define MC_CMD_TXQ_STATS_IN_LEN 8 +/* Instance of TXQ to retrieve statistics for */ +#define MC_CMD_TXQ_STATS_IN_INSTANCE_OFST 0 +#define MC_CMD_TXQ_STATS_IN_INSTANCE_LEN 4 +/* Flags for the request */ +#define MC_CMD_TXQ_STATS_IN_FLAGS_OFST 4 +#define MC_CMD_TXQ_STATS_IN_FLAGS_LEN 4 +#define MC_CMD_TXQ_STATS_IN_CLEAR_OFST 4 +#define MC_CMD_TXQ_STATS_IN_CLEAR_LBN 0 +#define MC_CMD_TXQ_STATS_IN_CLEAR_WIDTH 1 + +/* MC_CMD_TXQ_STATS_OUT msgresponse */ +#define MC_CMD_TXQ_STATS_OUT_LENMIN 0 +#define MC_CMD_TXQ_STATS_OUT_LENMAX 248 +#define MC_CMD_TXQ_STATS_OUT_LENMAX_MCDI2 1016 +#define MC_CMD_TXQ_STATS_OUT_LEN(num) (0+8*(num)) +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_NUM(len) (((len)-0)/8) +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_OFST 0 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_LEN 8 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_LO_OFST 0 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_LO_LEN 4 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_LO_LBN 0 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_LO_WIDTH 32 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_HI_OFST 4 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_HI_LEN 4 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_HI_LBN 32 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_HI_WIDTH 32 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_MINNUM 0 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_MAXNUM 31 +#define MC_CMD_TXQ_STATS_OUT_STATISTICS_MAXNUM_MCDI2 127 +#define MC_CMD_TXQ_STATS_CTPIO_MAX_FILL 0x0 /* enum */ + /* FUNCTION_PERSONALITY structuredef: The meanings of the personalities are * defined in SF-120734-TC with more information in SF-122717-TC. */ @@ -21044,7 +25112,13 @@ #define MC_CMD_VIRTIO_GET_FEATURES_OUT_FEATURES_OFST 0 #define MC_CMD_VIRTIO_GET_FEATURES_OUT_FEATURES_LEN 8 #define MC_CMD_VIRTIO_GET_FEATURES_OUT_FEATURES_LO_OFST 0 +#define MC_CMD_VIRTIO_GET_FEATURES_OUT_FEATURES_LO_LEN 4 +#define MC_CMD_VIRTIO_GET_FEATURES_OUT_FEATURES_LO_LBN 0 +#define MC_CMD_VIRTIO_GET_FEATURES_OUT_FEATURES_LO_WIDTH 32 #define MC_CMD_VIRTIO_GET_FEATURES_OUT_FEATURES_HI_OFST 4 +#define MC_CMD_VIRTIO_GET_FEATURES_OUT_FEATURES_HI_LEN 4 +#define MC_CMD_VIRTIO_GET_FEATURES_OUT_FEATURES_HI_LBN 32 +#define MC_CMD_VIRTIO_GET_FEATURES_OUT_FEATURES_HI_WIDTH 32 /***********************************/ @@ -21075,13 +25149,50 @@ #define MC_CMD_VIRTIO_TEST_FEATURES_IN_FEATURES_OFST 8 #define MC_CMD_VIRTIO_TEST_FEATURES_IN_FEATURES_LEN 8 #define MC_CMD_VIRTIO_TEST_FEATURES_IN_FEATURES_LO_OFST 8 +#define MC_CMD_VIRTIO_TEST_FEATURES_IN_FEATURES_LO_LEN 4 +#define MC_CMD_VIRTIO_TEST_FEATURES_IN_FEATURES_LO_LBN 64 +#define MC_CMD_VIRTIO_TEST_FEATURES_IN_FEATURES_LO_WIDTH 32 #define MC_CMD_VIRTIO_TEST_FEATURES_IN_FEATURES_HI_OFST 12 +#define MC_CMD_VIRTIO_TEST_FEATURES_IN_FEATURES_HI_LEN 4 +#define MC_CMD_VIRTIO_TEST_FEATURES_IN_FEATURES_HI_LBN 96 +#define MC_CMD_VIRTIO_TEST_FEATURES_IN_FEATURES_HI_WIDTH 32 /* MC_CMD_VIRTIO_TEST_FEATURES_OUT msgresponse */ #define MC_CMD_VIRTIO_TEST_FEATURES_OUT_LEN 0 /***********************************/ +/* MC_CMD_VIRTIO_GET_CAPABILITIES + * Get virtio capabilities supported by the device. Returns general virtio + * capabilities and limitations of the hardware / firmware implementation + * (hardware device as a whole), rather than that of individual configured + * virtio devices. At present, only the absolute maximum number of queues + * allowed on multi-queue devices is returned. Response is expected to be + * extended as necessary in the future. + */ +#define MC_CMD_VIRTIO_GET_CAPABILITIES 0x1d3 +#undef MC_CMD_0x1d3_PRIVILEGE_CTG + +#define MC_CMD_0x1d3_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_VIRTIO_GET_CAPABILITIES_IN msgrequest */ +#define MC_CMD_VIRTIO_GET_CAPABILITIES_IN_LEN 4 +/* Type of device to get capabilities for. Matches the device id as defined by + * the virtio spec. + */ +#define MC_CMD_VIRTIO_GET_CAPABILITIES_IN_DEVICE_ID_OFST 0 +#define MC_CMD_VIRTIO_GET_CAPABILITIES_IN_DEVICE_ID_LEN 4 +/* Enum values, see field(s): */ +/* MC_CMD_VIRTIO_GET_FEATURES/MC_CMD_VIRTIO_GET_FEATURES_IN/DEVICE_ID */ + +/* MC_CMD_VIRTIO_GET_CAPABILITIES_OUT msgresponse */ +#define MC_CMD_VIRTIO_GET_CAPABILITIES_OUT_LEN 4 +/* Maximum number of queues supported for a single device instance */ +#define MC_CMD_VIRTIO_GET_CAPABILITIES_OUT_MAX_QUEUES_OFST 0 +#define MC_CMD_VIRTIO_GET_CAPABILITIES_OUT_MAX_QUEUES_LEN 4 + + +/***********************************/ /* MC_CMD_VIRTIO_INIT_QUEUE * Create a virtio virtqueue. Fails with EALREADY if the queue already exists. * Fails with ENOSUP if a feature is requested that isn't supported. Fails with @@ -21133,17 +25244,35 @@ #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_DESC_TBL_ADDR_OFST 16 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_DESC_TBL_ADDR_LEN 8 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_DESC_TBL_ADDR_LO_OFST 16 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_DESC_TBL_ADDR_LO_LEN 4 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_DESC_TBL_ADDR_LO_LBN 128 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_DESC_TBL_ADDR_LO_WIDTH 32 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_DESC_TBL_ADDR_HI_OFST 20 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_DESC_TBL_ADDR_HI_LEN 4 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_DESC_TBL_ADDR_HI_LBN 160 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_DESC_TBL_ADDR_HI_WIDTH 32 /* Address of the available ring in the virtqueue. */ #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_AVAIL_RING_ADDR_OFST 24 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_AVAIL_RING_ADDR_LEN 8 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_AVAIL_RING_ADDR_LO_OFST 24 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_AVAIL_RING_ADDR_LO_LEN 4 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_AVAIL_RING_ADDR_LO_LBN 192 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_AVAIL_RING_ADDR_LO_WIDTH 32 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_AVAIL_RING_ADDR_HI_OFST 28 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_AVAIL_RING_ADDR_HI_LEN 4 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_AVAIL_RING_ADDR_HI_LBN 224 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_AVAIL_RING_ADDR_HI_WIDTH 32 /* Address of the used ring in the virtqueue. */ #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_USED_RING_ADDR_OFST 32 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_USED_RING_ADDR_LEN 8 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_USED_RING_ADDR_LO_OFST 32 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_USED_RING_ADDR_LO_LEN 4 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_USED_RING_ADDR_LO_LBN 256 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_USED_RING_ADDR_LO_WIDTH 32 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_USED_RING_ADDR_HI_OFST 36 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_USED_RING_ADDR_HI_LEN 4 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_USED_RING_ADDR_HI_LBN 288 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_USED_RING_ADDR_HI_WIDTH 32 /* PASID to use on PCIe transactions involving this queue. Ignored if the * USE_PASID flag is not set. */ @@ -21167,21 +25296,35 @@ #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_FEATURES_OFST 48 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_FEATURES_LEN 8 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_FEATURES_LO_OFST 48 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_FEATURES_LO_LEN 4 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_FEATURES_LO_LBN 384 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_FEATURES_LO_WIDTH 32 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_FEATURES_HI_OFST 52 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_FEATURES_HI_LEN 4 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_FEATURES_HI_LBN 416 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_FEATURES_HI_WIDTH 32 /* Enum values, see field(s): */ /* MC_CMD_VIRTIO_GET_FEATURES/MC_CMD_VIRTIO_GET_FEATURES_OUT/FEATURES */ -/* The inital producer index for this queue's used ring. If this queue is being - * created to be migrated into, this should be the FINAL_PIDX value returned by - * MC_CMD_VIRTIO_FINI_QUEUE of the queue being migrated from. Otherwise, it +/* The initial available index for this virtqueue. If this queue is being + * created to be migrated into, this should be the FINAL_AVAIL_IDX value + * returned by MC_CMD_VIRTIO_FINI_QUEUE of the queue being migrated from (or + * equivalent if the original queue was on a thirdparty device). Otherwise, it * should be zero. */ +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_INITIAL_AVAIL_IDX_OFST 56 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_INITIAL_AVAIL_IDX_LEN 4 +/* Alias of INITIAL_AVAIL_IDX, kept for compatibility. */ #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_INITIAL_PIDX_OFST 56 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_INITIAL_PIDX_LEN 4 -/* The inital consumer index for this queue's available ring. If this queue is - * being created to be migrated into, this should be the FINAL_CIDX value - * returned by MC_CMD_VIRTIO_FINI_QUEUE of the queue being migrated from. - * Otherwise, it should be zero. - */ +/* The initial used index for this virtqueue. If this queue is being created to + * be migrated into, this should be the FINAL_USED_IDX value returned by + * MC_CMD_VIRTIO_FINI_QUEUE of the queue being migrated from (or equivalent if + * the original queue was on a thirdparty device). Otherwise, it should be + * zero. + */ +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_INITIAL_USED_IDX_OFST 60 +#define MC_CMD_VIRTIO_INIT_QUEUE_REQ_INITIAL_USED_IDX_LEN 4 +/* Alias of INITIAL_USED_IDX, kept for compatibility. */ #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_INITIAL_CIDX_OFST 60 #define MC_CMD_VIRTIO_INIT_QUEUE_REQ_INITIAL_CIDX_LEN 4 /* A MAE_MPORT_SELECTOR defining which mport this queue should be associated @@ -21226,10 +25369,16 @@ /* MC_CMD_VIRTIO_FINI_QUEUE_RESP msgresponse */ #define MC_CMD_VIRTIO_FINI_QUEUE_RESP_LEN 8 -/* The producer index of the used ring when the queue was stopped. */ +/* The available index of the virtqueue when the queue was stopped. */ +#define MC_CMD_VIRTIO_FINI_QUEUE_RESP_FINAL_AVAIL_IDX_OFST 0 +#define MC_CMD_VIRTIO_FINI_QUEUE_RESP_FINAL_AVAIL_IDX_LEN 4 +/* Alias of FINAL_AVAIL_IDX, kept for compatibility. */ #define MC_CMD_VIRTIO_FINI_QUEUE_RESP_FINAL_PIDX_OFST 0 #define MC_CMD_VIRTIO_FINI_QUEUE_RESP_FINAL_PIDX_LEN 4 -/* The consumer index of the available ring when the queue was stopped. */ +/* The used index of the virtqueue when the queue was stopped. */ +#define MC_CMD_VIRTIO_FINI_QUEUE_RESP_FINAL_USED_IDX_OFST 4 +#define MC_CMD_VIRTIO_FINI_QUEUE_RESP_FINAL_USED_IDX_LEN 4 +/* Alias of FINAL_USED_IDX, kept for compatibility. */ #define MC_CMD_VIRTIO_FINI_QUEUE_RESP_FINAL_CIDX_OFST 4 #define MC_CMD_VIRTIO_FINI_QUEUE_RESP_FINAL_CIDX_LEN 4 @@ -21309,16 +25458,40 @@ #define PCIE_FUNCTION_VF_NULL 0xffff #define PCIE_FUNCTION_VF_LBN 16 #define PCIE_FUNCTION_VF_WIDTH 16 -/* PCIe interface of the function */ +/* PCIe interface of the function. Values should be taken from the + * PCIE_INTERFACE enum + */ #define PCIE_FUNCTION_INTF_OFST 4 #define PCIE_FUNCTION_INTF_LEN 4 -/* enum: Host PCIe interface */ +/* enum: Host PCIe interface. (Alias for HOST_PRIMARY, provided for backwards + * compatibility) + */ #define PCIE_FUNCTION_INTF_HOST 0x0 -/* enum: Application Processor interface */ +/* enum: Application Processor interface (alias for NIC_EMBEDDED, provided for + * backwards compatibility) + */ #define PCIE_FUNCTION_INTF_AP 0x1 #define PCIE_FUNCTION_INTF_LBN 32 #define PCIE_FUNCTION_INTF_WIDTH 32 +/* QUEUE_ID structuredef: Structure representing an absolute queue identifier + * (absolute VI number + VI relative queue number). On Keystone, a VI can + * contain multiple queues (at present, up to 2), each with separate controls + * for direction. This structure is required to uniquely identify the absolute + * source queue for descriptor proxy functions. + */ +#define QUEUE_ID_LEN 4 +/* Absolute VI number */ +#define QUEUE_ID_ABS_VI_OFST 0 +#define QUEUE_ID_ABS_VI_LEN 2 +#define QUEUE_ID_ABS_VI_LBN 0 +#define QUEUE_ID_ABS_VI_WIDTH 16 +/* Relative queue number within the VI */ +#define QUEUE_ID_REL_QUEUE_LBN 16 +#define QUEUE_ID_REL_QUEUE_WIDTH 1 +#define QUEUE_ID_RESERVED_LBN 17 +#define QUEUE_ID_RESERVED_WIDTH 15 + /***********************************/ /* MC_CMD_DESC_PROXY_FUNC_CREATE @@ -21347,7 +25520,19 @@ #define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_OFST 0 #define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_LEN 8 #define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_LO_OFST 0 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_LO_LEN 4 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_LO_LBN 0 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_LO_WIDTH 32 #define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_HI_OFST 4 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_HI_LEN 4 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_HI_LBN 32 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_HI_WIDTH 32 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_PF_OFST 0 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_PF_LEN 2 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_VF_OFST 2 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_VF_LEN 2 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_INTF_OFST 4 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_IN_FUNC_INTF_LEN 4 /* The personality to set. The meanings of the personalities are defined in * SF-120734-TC with more information in SF-122717-TC. At present, we only * support proxying for VIRTIO_BLK @@ -21371,7 +25556,19 @@ #define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_OFST 4 #define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_LEN 8 #define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_LO_OFST 4 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_LO_LEN 4 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_LO_LBN 32 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_LO_WIDTH 32 #define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_HI_OFST 8 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_HI_LEN 4 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_HI_LBN 64 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_HI_WIDTH 32 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_PF_OFST 4 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_PF_LEN 2 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_VF_OFST 6 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_VF_LEN 2 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_INTF_OFST 8 +#define MC_CMD_DESC_PROXY_FUNC_CREATE_OUT_FUNC_INTF_LEN 4 /***********************************/ @@ -21412,7 +25609,13 @@ #define VIRTIO_BLK_CONFIG_FEATURES_OFST 0 #define VIRTIO_BLK_CONFIG_FEATURES_LEN 8 #define VIRTIO_BLK_CONFIG_FEATURES_LO_OFST 0 +#define VIRTIO_BLK_CONFIG_FEATURES_LO_LEN 4 +#define VIRTIO_BLK_CONFIG_FEATURES_LO_LBN 0 +#define VIRTIO_BLK_CONFIG_FEATURES_LO_WIDTH 32 #define VIRTIO_BLK_CONFIG_FEATURES_HI_OFST 4 +#define VIRTIO_BLK_CONFIG_FEATURES_HI_LEN 4 +#define VIRTIO_BLK_CONFIG_FEATURES_HI_LBN 32 +#define VIRTIO_BLK_CONFIG_FEATURES_HI_WIDTH 32 #define VIRTIO_BLK_CONFIG_VIRTIO_BLK_F_BARRIER_OFST 0 #define VIRTIO_BLK_CONFIG_VIRTIO_BLK_F_BARRIER_LBN 0 #define VIRTIO_BLK_CONFIG_VIRTIO_BLK_F_BARRIER_WIDTH 1 @@ -21485,7 +25688,13 @@ #define VIRTIO_BLK_CONFIG_CAPACITY_OFST 8 #define VIRTIO_BLK_CONFIG_CAPACITY_LEN 8 #define VIRTIO_BLK_CONFIG_CAPACITY_LO_OFST 8 +#define VIRTIO_BLK_CONFIG_CAPACITY_LO_LEN 4 +#define VIRTIO_BLK_CONFIG_CAPACITY_LO_LBN 64 +#define VIRTIO_BLK_CONFIG_CAPACITY_LO_WIDTH 32 #define VIRTIO_BLK_CONFIG_CAPACITY_HI_OFST 12 +#define VIRTIO_BLK_CONFIG_CAPACITY_HI_LEN 4 +#define VIRTIO_BLK_CONFIG_CAPACITY_HI_LBN 96 +#define VIRTIO_BLK_CONFIG_CAPACITY_HI_WIDTH 32 #define VIRTIO_BLK_CONFIG_CAPACITY_LBN 64 #define VIRTIO_BLK_CONFIG_CAPACITY_WIDTH 64 /* Maximum size of any single segment. Only valid when VIRTIO_BLK_F_SIZE_MAX is @@ -21720,7 +25929,19 @@ #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_OFST 4 #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_LEN 8 #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_LO_OFST 4 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_LO_LEN 4 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_LO_LBN 32 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_LO_WIDTH 32 #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_HI_OFST 8 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_HI_LEN 4 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_HI_LBN 64 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_HI_WIDTH 32 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_PF_OFST 4 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_PF_LEN 2 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_VF_OFST 6 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_VF_LEN 2 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_INTF_OFST 8 +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_FUNC_INTF_LEN 4 /* Function personality */ #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_PERSONALITY_OFST 12 #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_PERSONALITY_LEN 4 @@ -21733,6 +25954,10 @@ #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_LIVE 0x0 /* enum: Function configuration is pending reset */ #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_PENDING 0x1 +/* enum: Function configuration is missing (created, but no configuration + * committed) + */ +#define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_UNCONFIGURED 0x2 /* Generation count to be delivered in an event once the configuration becomes * live (if status is "pending") */ @@ -21742,7 +25967,7 @@ #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_RESERVED_OFST 24 #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_RESERVED_LEN 16 /* Configuration data corresponding to function personality. Currently, only - * supported format is VIRTIO_BLK_CONFIG + * supported format is VIRTIO_BLK_CONFIG. Not valid if status is UNCONFIGURED. */ #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_CONFIG_OFST 40 #define MC_CMD_DESC_PROXY_FUNC_OPEN_OUT_CONFIG_LEN 1 @@ -21780,9 +26005,27 @@ #define DESC_PROXY_FUNC_MAP_FUNC_OFST 0 #define DESC_PROXY_FUNC_MAP_FUNC_LEN 8 #define DESC_PROXY_FUNC_MAP_FUNC_LO_OFST 0 +#define DESC_PROXY_FUNC_MAP_FUNC_LO_LEN 4 +#define DESC_PROXY_FUNC_MAP_FUNC_LO_LBN 0 +#define DESC_PROXY_FUNC_MAP_FUNC_LO_WIDTH 32 #define DESC_PROXY_FUNC_MAP_FUNC_HI_OFST 4 +#define DESC_PROXY_FUNC_MAP_FUNC_HI_LEN 4 +#define DESC_PROXY_FUNC_MAP_FUNC_HI_LBN 32 +#define DESC_PROXY_FUNC_MAP_FUNC_HI_WIDTH 32 #define DESC_PROXY_FUNC_MAP_FUNC_LBN 0 #define DESC_PROXY_FUNC_MAP_FUNC_WIDTH 64 +#define DESC_PROXY_FUNC_MAP_FUNC_PF_OFST 0 +#define DESC_PROXY_FUNC_MAP_FUNC_PF_LEN 2 +#define DESC_PROXY_FUNC_MAP_FUNC_PF_LBN 0 +#define DESC_PROXY_FUNC_MAP_FUNC_PF_WIDTH 16 +#define DESC_PROXY_FUNC_MAP_FUNC_VF_OFST 2 +#define DESC_PROXY_FUNC_MAP_FUNC_VF_LEN 2 +#define DESC_PROXY_FUNC_MAP_FUNC_VF_LBN 16 +#define DESC_PROXY_FUNC_MAP_FUNC_VF_WIDTH 16 +#define DESC_PROXY_FUNC_MAP_FUNC_INTF_OFST 4 +#define DESC_PROXY_FUNC_MAP_FUNC_INTF_LEN 4 +#define DESC_PROXY_FUNC_MAP_FUNC_INTF_LBN 32 +#define DESC_PROXY_FUNC_MAP_FUNC_INTF_WIDTH 32 /* Function personality */ #define DESC_PROXY_FUNC_MAP_PERSONALITY_OFST 8 #define DESC_PROXY_FUNC_MAP_PERSONALITY_LEN 4 @@ -21840,7 +26083,11 @@ * Enable descriptor proxying for function into target event queue. Returns VI * allocation info for the proxy source function, so that the caller can map * absolute VI IDs from descriptor proxy events back to the originating - * function. + * function. This is a legacy function that only supports single queue proxy + * devices. It is also limited in that it can only be called after host driver + * attach (once VI allocation is known) and will return MC_CMD_ERR_ENOTCONN + * otherwise. For new code, see MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE which + * supports multi-queue devices and has no dependency on host driver attach. */ #define MC_CMD_DESC_PROXY_FUNC_ENABLE 0x178 #undef MC_CMD_0x178_PRIVILEGE_CTG @@ -21871,8 +26118,44 @@ /***********************************/ +/* MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE + * Enable descriptor proxying for a source queue on a host function into target + * event queue. Source queue number is a relative virtqueue number on the + * source function (0 to max_virtqueues-1). For a multi-queue device, the + * caller must enable all source queues individually. To retrieve absolute VI + * information for the source function (so that VI IDs from descriptor proxy + * events can be mapped back to source function / queue) see + * MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO + */ +#define MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE 0x1d0 +#undef MC_CMD_0x1d0_PRIVILEGE_CTG + +#define MC_CMD_0x1d0_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE_IN msgrequest */ +#define MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE_IN_LEN 12 +/* Handle to descriptor proxy function (as returned by + * MC_CMD_DESC_PROXY_FUNC_OPEN) + */ +#define MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE_IN_HANDLE_OFST 0 +#define MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE_IN_HANDLE_LEN 4 +/* Source relative queue number to enable proxying on */ +#define MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE_IN_SOURCE_QUEUE_OFST 4 +#define MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE_IN_SOURCE_QUEUE_LEN 4 +/* Descriptor proxy sink queue (caller function relative). Must be extended + * width event queue + */ +#define MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE_IN_TARGET_EVQ_OFST 8 +#define MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE_IN_TARGET_EVQ_LEN 4 + +/* MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE_OUT msgresponse */ +#define MC_CMD_DESC_PROXY_FUNC_ENABLE_QUEUE_OUT_LEN 0 + + +/***********************************/ /* MC_CMD_DESC_PROXY_FUNC_DISABLE - * Disable descriptor proxying for function + * Disable descriptor proxying for function. For multi-queue functions, + * disables all queues. */ #define MC_CMD_DESC_PROXY_FUNC_DISABLE 0x179 #undef MC_CMD_0x179_PRIVILEGE_CTG @@ -21892,6 +26175,75 @@ /***********************************/ +/* MC_CMD_DESC_PROXY_FUNC_DISABLE_QUEUE + * Disable descriptor proxying for a specific source queue on a function. + */ +#define MC_CMD_DESC_PROXY_FUNC_DISABLE_QUEUE 0x1d1 +#undef MC_CMD_0x1d1_PRIVILEGE_CTG + +#define MC_CMD_0x1d1_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DESC_PROXY_FUNC_DISABLE_QUEUE_IN msgrequest */ +#define MC_CMD_DESC_PROXY_FUNC_DISABLE_QUEUE_IN_LEN 8 +/* Handle to descriptor proxy function (as returned by + * MC_CMD_DESC_PROXY_FUNC_OPEN) + */ +#define MC_CMD_DESC_PROXY_FUNC_DISABLE_QUEUE_IN_HANDLE_OFST 0 +#define MC_CMD_DESC_PROXY_FUNC_DISABLE_QUEUE_IN_HANDLE_LEN 4 +/* Source relative queue number to disable proxying on */ +#define MC_CMD_DESC_PROXY_FUNC_DISABLE_QUEUE_IN_SOURCE_QUEUE_OFST 4 +#define MC_CMD_DESC_PROXY_FUNC_DISABLE_QUEUE_IN_SOURCE_QUEUE_LEN 4 + +/* MC_CMD_DESC_PROXY_FUNC_DISABLE_QUEUE_OUT msgresponse */ +#define MC_CMD_DESC_PROXY_FUNC_DISABLE_QUEUE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_DESC_PROXY_GET_VI_INFO + * Returns absolute VI allocation information for the descriptor proxy source + * function referenced by HANDLE, so that the caller can map absolute VI IDs + * from descriptor proxy events back to the originating function and queue. The + * call is only valid after the host driver for the source function has + * attached (after receiving a driver attach event for the descriptor proxy + * function) and will fail with ENOTCONN otherwise. + */ +#define MC_CMD_DESC_PROXY_GET_VI_INFO 0x1d2 +#undef MC_CMD_0x1d2_PRIVILEGE_CTG + +#define MC_CMD_0x1d2_PRIVILEGE_CTG SRIOV_CTG_ADMIN + +/* MC_CMD_DESC_PROXY_GET_VI_INFO_IN msgrequest */ +#define MC_CMD_DESC_PROXY_GET_VI_INFO_IN_LEN 4 +/* Handle to descriptor proxy function (as returned by + * MC_CMD_DESC_PROXY_FUNC_OPEN) + */ +#define MC_CMD_DESC_PROXY_GET_VI_INFO_IN_HANDLE_OFST 0 +#define MC_CMD_DESC_PROXY_GET_VI_INFO_IN_HANDLE_LEN 4 + +/* MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT msgresponse */ +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_LENMIN 0 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_LENMAX 252 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_LEN(num) (0+4*(num)) +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_NUM(len) (((len)-0)/4) +/* VI information (VI ID + VI relative queue number) for each of the source + * queues (in order from 0 to max_virtqueues-1), as array of QUEUE_ID + * structures. + */ +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_OFST 0 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_LEN 4 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_MINNUM 0 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_MAXNUM 63 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_MAXNUM_MCDI2 255 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_ABS_VI_OFST 0 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_ABS_VI_LEN 2 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_REL_QUEUE_LBN 16 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_REL_QUEUE_WIDTH 1 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_RESERVED_LBN 17 +#define MC_CMD_DESC_PROXY_FUNC_GET_VI_INFO_OUT_VI_MAP_RESERVED_WIDTH 15 + + +/***********************************/ /* MC_CMD_GET_ADDR_SPC_ID * Get Address space identifier for use in mem2mem descriptors for a given * target. See SF-120734-TC for details on ADDR_SPC_IDs and mem2mem @@ -21942,7 +26294,19 @@ #define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_OFST 4 #define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_LEN 8 #define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_LO_OFST 4 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_LO_LEN 4 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_LO_LBN 32 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_LO_WIDTH 32 #define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_HI_OFST 8 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_HI_LEN 4 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_HI_LBN 64 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_HI_WIDTH 32 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_PF_OFST 4 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_PF_LEN 2 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_VF_OFST 6 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_VF_LEN 2 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_INTF_OFST 8 +#define MC_CMD_GET_ADDR_SPC_ID_IN_FUNC_INTF_LEN 4 /* PASID value. Only valid if TYPE is PCI_FUNC_PASID. */ #define MC_CMD_GET_ADDR_SPC_ID_IN_PASID_OFST 12 #define MC_CMD_GET_ADDR_SPC_ID_IN_PASID_LEN 4 @@ -21962,7 +26326,3381 @@ #define MC_CMD_GET_ADDR_SPC_ID_OUT_ADDR_SPC_ID_OFST 0 #define MC_CMD_GET_ADDR_SPC_ID_OUT_ADDR_SPC_ID_LEN 8 #define MC_CMD_GET_ADDR_SPC_ID_OUT_ADDR_SPC_ID_LO_OFST 0 +#define MC_CMD_GET_ADDR_SPC_ID_OUT_ADDR_SPC_ID_LO_LEN 4 +#define MC_CMD_GET_ADDR_SPC_ID_OUT_ADDR_SPC_ID_LO_LBN 0 +#define MC_CMD_GET_ADDR_SPC_ID_OUT_ADDR_SPC_ID_LO_WIDTH 32 #define MC_CMD_GET_ADDR_SPC_ID_OUT_ADDR_SPC_ID_HI_OFST 4 +#define MC_CMD_GET_ADDR_SPC_ID_OUT_ADDR_SPC_ID_HI_LEN 4 +#define MC_CMD_GET_ADDR_SPC_ID_OUT_ADDR_SPC_ID_HI_LBN 32 +#define MC_CMD_GET_ADDR_SPC_ID_OUT_ADDR_SPC_ID_HI_WIDTH 32 + + +/***********************************/ +/* MC_CMD_GET_CLIENT_HANDLE + * Obtain a handle for a client given a description of that client. N.B. this + * command is subject to change given the open discussion about how PCIe + * functions should be referenced on an iEP (integrated endpoint: functions + * span multiple buses) and multihost (multiple PCIe interfaces) system. + */ +#define MC_CMD_GET_CLIENT_HANDLE 0x1c3 +#undef MC_CMD_0x1c3_PRIVILEGE_CTG + +#define MC_CMD_0x1c3_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_GET_CLIENT_HANDLE_IN msgrequest */ +#define MC_CMD_GET_CLIENT_HANDLE_IN_LEN 12 +/* Type of client to get a client handle for */ +#define MC_CMD_GET_CLIENT_HANDLE_IN_TYPE_OFST 0 +#define MC_CMD_GET_CLIENT_HANDLE_IN_TYPE_LEN 4 +/* enum: Obtain a client handle for a PCIe function-type client. */ +#define MC_CMD_GET_CLIENT_HANDLE_IN_TYPE_FUNC 0x0 +/* PCIe Function ID (as struct PCIE_FUNCTION). Valid when TYPE==FUNC. Use: - + * INTF=CALLER, PF=PF_NULL, VF=VF_NULL to refer to the calling function - + * INTF=CALLER, PF=PF_NULL, VF=... to refer to a VF child of the calling PF or + * a sibling VF of the calling VF. - INTF=CALLER, PF=..., VF=VF_NULL to refer + * to a PF on the calling interface - INTF=CALLER, PF=..., VF=... to refer to a + * VF on the calling interface - INTF=..., PF=..., VF=VF_NULL to refer to a PF + * on a named interface - INTF=..., PF=..., VF=... to refer to a VF on a named + * interface where ... refers to a small integer for the VF/PF fields, and to + * values from the PCIE_INTERFACE enum for for the INTF field. It's only + * meaningful to use INTF=CALLER within a structure that's an argument to + * MC_CMD_DEVEL_GET_CLIENT_HANDLE. + */ +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_OFST 4 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_LEN 8 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_LO_OFST 4 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_LO_LEN 4 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_LO_LBN 32 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_LO_WIDTH 32 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_HI_OFST 8 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_HI_LEN 4 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_HI_LBN 64 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_HI_WIDTH 32 +/* enum: NULL value for the INTF field of struct PCIE_FUNCTION. Provided for + * backwards compatibility only, callers should use PCIE_INTERFACE_CALLER. + */ +#define MC_CMD_GET_CLIENT_HANDLE_IN_PCIE_FUNCTION_INTF_NULL 0xffffffff +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_PF_OFST 4 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_PF_LEN 2 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_VF_OFST 6 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_VF_LEN 2 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_INTF_OFST 8 +#define MC_CMD_GET_CLIENT_HANDLE_IN_FUNC_INTF_LEN 4 + +/* MC_CMD_GET_CLIENT_HANDLE_OUT msgresponse */ +#define MC_CMD_GET_CLIENT_HANDLE_OUT_LEN 4 +#define MC_CMD_GET_CLIENT_HANDLE_OUT_HANDLE_OFST 0 +#define MC_CMD_GET_CLIENT_HANDLE_OUT_HANDLE_LEN 4 + +/* MAE_FIELD_FLAGS structuredef */ +#define MAE_FIELD_FLAGS_LEN 4 +#define MAE_FIELD_FLAGS_FLAT_OFST 0 +#define MAE_FIELD_FLAGS_FLAT_LEN 4 +#define MAE_FIELD_FLAGS_SUPPORT_STATUS_OFST 0 +#define MAE_FIELD_FLAGS_SUPPORT_STATUS_LBN 0 +#define MAE_FIELD_FLAGS_SUPPORT_STATUS_WIDTH 6 +#define MAE_FIELD_FLAGS_MASK_AFFECTS_CLASS_OFST 0 +#define MAE_FIELD_FLAGS_MASK_AFFECTS_CLASS_LBN 6 +#define MAE_FIELD_FLAGS_MASK_AFFECTS_CLASS_WIDTH 1 +#define MAE_FIELD_FLAGS_MATCH_AFFECTS_CLASS_OFST 0 +#define MAE_FIELD_FLAGS_MATCH_AFFECTS_CLASS_LBN 7 +#define MAE_FIELD_FLAGS_MATCH_AFFECTS_CLASS_WIDTH 1 +#define MAE_FIELD_FLAGS_FLAT_LBN 0 +#define MAE_FIELD_FLAGS_FLAT_WIDTH 32 + +/* MAE_ENC_FIELD_PAIRS structuredef: Mask and value pairs for all fields that + * it makes sense to use to determine the encapsulation type of a packet. Its + * intended use is to keep a common packing of fields across multiple MCDI + * commands, keeping things inherently sychronised and allowing code shared. To + * use in an MCDI command, the command should end with a variable length byte + * array populated with this structure. Do not extend this structure. Instead, + * create _Vx versions with the necessary fields appended. That way, the + * existing semantics for extending MCDI commands are preserved. + */ +#define MAE_ENC_FIELD_PAIRS_LEN 156 +#define MAE_ENC_FIELD_PAIRS_INGRESS_MPORT_SELECTOR_OFST 0 +#define MAE_ENC_FIELD_PAIRS_INGRESS_MPORT_SELECTOR_LEN 4 +#define MAE_ENC_FIELD_PAIRS_INGRESS_MPORT_SELECTOR_LBN 0 +#define MAE_ENC_FIELD_PAIRS_INGRESS_MPORT_SELECTOR_WIDTH 32 +#define MAE_ENC_FIELD_PAIRS_INGRESS_MPORT_SELECTOR_MASK_OFST 4 +#define MAE_ENC_FIELD_PAIRS_INGRESS_MPORT_SELECTOR_MASK_LEN 4 +#define MAE_ENC_FIELD_PAIRS_INGRESS_MPORT_SELECTOR_MASK_LBN 32 +#define MAE_ENC_FIELD_PAIRS_INGRESS_MPORT_SELECTOR_MASK_WIDTH 32 +#define MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE_OFST 8 +#define MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE_LBN 64 +#define MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE_MASK_OFST 10 +#define MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE_MASK_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE_MASK_LBN 80 +#define MAE_ENC_FIELD_PAIRS_ENC_ETHER_TYPE_BE_MASK_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_TCI_BE_OFST 12 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_TCI_BE_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_TCI_BE_LBN 96 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_TCI_BE_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_TCI_BE_MASK_OFST 14 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_TCI_BE_MASK_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_TCI_BE_MASK_LBN 112 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_TCI_BE_MASK_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_PROTO_BE_OFST 16 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_PROTO_BE_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_PROTO_BE_LBN 128 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_PROTO_BE_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_PROTO_BE_MASK_OFST 18 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_PROTO_BE_MASK_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_PROTO_BE_MASK_LBN 144 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN0_PROTO_BE_MASK_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_TCI_BE_OFST 20 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_TCI_BE_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_TCI_BE_LBN 160 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_TCI_BE_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_TCI_BE_MASK_OFST 22 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_TCI_BE_MASK_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_TCI_BE_MASK_LBN 176 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_TCI_BE_MASK_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_PROTO_BE_OFST 24 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_PROTO_BE_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_PROTO_BE_LBN 192 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_PROTO_BE_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_PROTO_BE_MASK_OFST 26 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_PROTO_BE_MASK_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_PROTO_BE_MASK_LBN 208 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN1_PROTO_BE_MASK_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_SADDR_BE_OFST 28 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_SADDR_BE_LEN 6 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_SADDR_BE_LBN 224 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_SADDR_BE_WIDTH 48 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_SADDR_BE_MASK_OFST 34 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_SADDR_BE_MASK_LEN 6 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_SADDR_BE_MASK_LBN 272 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_SADDR_BE_MASK_WIDTH 48 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_DADDR_BE_OFST 40 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_DADDR_BE_LEN 6 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_DADDR_BE_LBN 320 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_DADDR_BE_WIDTH 48 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_DADDR_BE_MASK_OFST 46 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_DADDR_BE_MASK_LEN 6 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_DADDR_BE_MASK_LBN 368 +#define MAE_ENC_FIELD_PAIRS_ENC_ETH_DADDR_BE_MASK_WIDTH 48 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP4_BE_OFST 52 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP4_BE_LEN 4 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP4_BE_LBN 416 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP4_BE_WIDTH 32 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP4_BE_MASK_OFST 56 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP4_BE_MASK_LEN 4 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP4_BE_MASK_LBN 448 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP4_BE_MASK_WIDTH 32 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP6_BE_OFST 60 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP6_BE_LEN 16 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP6_BE_LBN 480 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP6_BE_WIDTH 128 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP6_BE_MASK_OFST 76 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP6_BE_MASK_LEN 16 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP6_BE_MASK_LBN 608 +#define MAE_ENC_FIELD_PAIRS_ENC_SRC_IP6_BE_MASK_WIDTH 128 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP4_BE_OFST 92 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP4_BE_LEN 4 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP4_BE_LBN 736 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP4_BE_WIDTH 32 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP4_BE_MASK_OFST 96 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP4_BE_MASK_LEN 4 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP4_BE_MASK_LBN 768 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP4_BE_MASK_WIDTH 32 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP6_BE_OFST 100 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP6_BE_LEN 16 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP6_BE_LBN 800 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP6_BE_WIDTH 128 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP6_BE_MASK_OFST 116 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP6_BE_MASK_LEN 16 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP6_BE_MASK_LBN 928 +#define MAE_ENC_FIELD_PAIRS_ENC_DST_IP6_BE_MASK_WIDTH 128 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_PROTO_OFST 132 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_PROTO_LEN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_PROTO_LBN 1056 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_PROTO_WIDTH 8 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_PROTO_MASK_OFST 133 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_PROTO_MASK_LEN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_PROTO_MASK_LBN 1064 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_PROTO_MASK_WIDTH 8 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TOS_OFST 134 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TOS_LEN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TOS_LBN 1072 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TOS_WIDTH 8 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TOS_MASK_OFST 135 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TOS_MASK_LEN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TOS_MASK_LBN 1080 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TOS_MASK_WIDTH 8 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TTL_OFST 136 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TTL_LEN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TTL_LBN 1088 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TTL_WIDTH 8 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TTL_MASK_OFST 137 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TTL_MASK_LEN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TTL_MASK_LBN 1096 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_TTL_MASK_WIDTH 8 +/* Deprecated in favour of ENC_FLAGS alias. */ +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN_FLAGS_OFST 138 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN_FLAGS_LEN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_OVLAN_OFST 138 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_OVLAN_LBN 0 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_OVLAN_WIDTH 1 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_IVLAN_OFST 138 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_IVLAN_LBN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_IVLAN_WIDTH 1 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FRAG_OFST 138 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FRAG_LBN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FRAG_WIDTH 1 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN_FLAGS_LBN 1104 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN_FLAGS_WIDTH 8 +/* More generic alias for ENC_VLAN_FLAGS. */ +#define MAE_ENC_FIELD_PAIRS_ENC_FLAGS_OFST 138 +#define MAE_ENC_FIELD_PAIRS_ENC_FLAGS_LEN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_FLAGS_LBN 1104 +#define MAE_ENC_FIELD_PAIRS_ENC_FLAGS_WIDTH 8 +/* Deprecated in favour of ENC_FLAGS_MASK alias. */ +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN_FLAGS_MASK_OFST 139 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN_FLAGS_MASK_LEN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_OVLAN_MASK_OFST 139 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_OVLAN_MASK_LBN 0 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_OVLAN_MASK_WIDTH 1 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_IVLAN_MASK_OFST 139 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_IVLAN_MASK_LBN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_HAS_IVLAN_MASK_WIDTH 1 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FRAG_MASK_OFST 139 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FRAG_MASK_LBN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FRAG_MASK_WIDTH 1 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN_FLAGS_MASK_LBN 1112 +#define MAE_ENC_FIELD_PAIRS_ENC_VLAN_FLAGS_MASK_WIDTH 8 +/* More generic alias for ENC_FLAGS_MASK. */ +#define MAE_ENC_FIELD_PAIRS_ENC_FLAGS_MASK_OFST 139 +#define MAE_ENC_FIELD_PAIRS_ENC_FLAGS_MASK_LEN 1 +#define MAE_ENC_FIELD_PAIRS_ENC_FLAGS_MASK_LBN 1112 +#define MAE_ENC_FIELD_PAIRS_ENC_FLAGS_MASK_WIDTH 8 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FLAGS_BE_OFST 140 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FLAGS_BE_LEN 4 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FLAGS_BE_LBN 1120 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FLAGS_BE_WIDTH 32 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FLAGS_BE_MASK_OFST 144 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FLAGS_BE_MASK_LEN 4 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FLAGS_BE_MASK_LBN 1152 +#define MAE_ENC_FIELD_PAIRS_ENC_IP_FLAGS_BE_MASK_WIDTH 32 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_SPORT_BE_OFST 148 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_SPORT_BE_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_SPORT_BE_LBN 1184 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_SPORT_BE_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_SPORT_BE_MASK_OFST 150 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_SPORT_BE_MASK_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_SPORT_BE_MASK_LBN 1200 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_SPORT_BE_MASK_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_DPORT_BE_OFST 152 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_DPORT_BE_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_DPORT_BE_LBN 1216 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_DPORT_BE_WIDTH 16 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_DPORT_BE_MASK_OFST 154 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_DPORT_BE_MASK_LEN 2 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_DPORT_BE_MASK_LBN 1232 +#define MAE_ENC_FIELD_PAIRS_ENC_L4_DPORT_BE_MASK_WIDTH 16 + +/* MAE_FIELD_MASK_VALUE_PAIRS structuredef: Mask and value pairs for all fields + * currently defined. Same semantics as MAE_ENC_FIELD_PAIRS. + */ +#define MAE_FIELD_MASK_VALUE_PAIRS_LEN 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_INGRESS_MPORT_SELECTOR_OFST 0 +#define MAE_FIELD_MASK_VALUE_PAIRS_INGRESS_MPORT_SELECTOR_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_INGRESS_MPORT_SELECTOR_LBN 0 +#define MAE_FIELD_MASK_VALUE_PAIRS_INGRESS_MPORT_SELECTOR_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_INGRESS_MPORT_SELECTOR_MASK_OFST 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_INGRESS_MPORT_SELECTOR_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_INGRESS_MPORT_SELECTOR_MASK_LBN 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_INGRESS_MPORT_SELECTOR_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_MARK_OFST 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_MARK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_MARK_LBN 64 +#define MAE_FIELD_MASK_VALUE_PAIRS_MARK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_MARK_MASK_OFST 12 +#define MAE_FIELD_MASK_VALUE_PAIRS_MARK_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_MARK_MASK_LBN 96 +#define MAE_FIELD_MASK_VALUE_PAIRS_MARK_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETHER_TYPE_BE_OFST 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETHER_TYPE_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETHER_TYPE_BE_LBN 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETHER_TYPE_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETHER_TYPE_BE_MASK_OFST 18 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETHER_TYPE_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETHER_TYPE_BE_MASK_LBN 144 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETHER_TYPE_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_TCI_BE_OFST 20 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_TCI_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_TCI_BE_LBN 160 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_TCI_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_TCI_BE_MASK_OFST 22 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_TCI_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_TCI_BE_MASK_LBN 176 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_TCI_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_PROTO_BE_OFST 24 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_PROTO_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_PROTO_BE_LBN 192 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_PROTO_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_PROTO_BE_MASK_OFST 26 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_PROTO_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_PROTO_BE_MASK_LBN 208 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN0_PROTO_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_TCI_BE_OFST 28 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_TCI_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_TCI_BE_LBN 224 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_TCI_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_TCI_BE_MASK_OFST 30 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_TCI_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_TCI_BE_MASK_LBN 240 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_TCI_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_PROTO_BE_OFST 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_PROTO_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_PROTO_BE_LBN 256 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_PROTO_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_PROTO_BE_MASK_OFST 34 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_PROTO_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_PROTO_BE_MASK_LBN 272 +#define MAE_FIELD_MASK_VALUE_PAIRS_VLAN1_PROTO_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_SADDR_BE_OFST 36 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_SADDR_BE_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_SADDR_BE_LBN 288 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_SADDR_BE_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_SADDR_BE_MASK_OFST 42 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_SADDR_BE_MASK_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_SADDR_BE_MASK_LBN 336 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_SADDR_BE_MASK_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_DADDR_BE_OFST 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_DADDR_BE_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_DADDR_BE_LBN 384 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_DADDR_BE_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_DADDR_BE_MASK_OFST 54 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_DADDR_BE_MASK_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_DADDR_BE_MASK_LBN 432 +#define MAE_FIELD_MASK_VALUE_PAIRS_ETH_DADDR_BE_MASK_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP4_BE_OFST 60 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP4_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP4_BE_LBN 480 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP4_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP4_BE_MASK_OFST 64 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP4_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP4_BE_MASK_LBN 512 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP4_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP6_BE_OFST 68 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP6_BE_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP6_BE_LBN 544 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP6_BE_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP6_BE_MASK_OFST 84 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP6_BE_MASK_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP6_BE_MASK_LBN 672 +#define MAE_FIELD_MASK_VALUE_PAIRS_SRC_IP6_BE_MASK_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP4_BE_OFST 100 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP4_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP4_BE_LBN 800 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP4_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP4_BE_MASK_OFST 104 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP4_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP4_BE_MASK_LBN 832 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP4_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP6_BE_OFST 108 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP6_BE_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP6_BE_LBN 864 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP6_BE_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP6_BE_MASK_OFST 124 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP6_BE_MASK_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP6_BE_MASK_LBN 992 +#define MAE_FIELD_MASK_VALUE_PAIRS_DST_IP6_BE_MASK_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_PROTO_OFST 140 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_PROTO_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_PROTO_LBN 1120 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_PROTO_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_PROTO_MASK_OFST 141 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_PROTO_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_PROTO_MASK_LBN 1128 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_PROTO_MASK_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TOS_OFST 142 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TOS_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TOS_LBN 1136 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TOS_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TOS_MASK_OFST 143 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TOS_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TOS_MASK_LBN 1144 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TOS_MASK_WIDTH 8 +/* Due to hardware limitations, firmware may return + * MC_CMD_ERR_EINVAL(BAD_IP_TTL) when attempting to match on an IP_TTL value + * other than 1. + */ +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TTL_OFST 144 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TTL_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TTL_LBN 1152 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TTL_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TTL_MASK_OFST 145 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TTL_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TTL_MASK_LBN 1160 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_TTL_MASK_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_FLAGS_BE_OFST 148 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_FLAGS_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_FLAGS_BE_LBN 1184 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_FLAGS_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_FLAGS_BE_MASK_OFST 152 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_FLAGS_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_FLAGS_BE_MASK_LBN 1216 +#define MAE_FIELD_MASK_VALUE_PAIRS_IP_FLAGS_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_SPORT_BE_OFST 156 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_SPORT_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_SPORT_BE_LBN 1248 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_SPORT_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_SPORT_BE_MASK_OFST 158 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_SPORT_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_SPORT_BE_MASK_LBN 1264 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_SPORT_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_DPORT_BE_OFST 160 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_DPORT_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_DPORT_BE_LBN 1280 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_DPORT_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_DPORT_BE_MASK_OFST 162 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_DPORT_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_DPORT_BE_MASK_LBN 1296 +#define MAE_FIELD_MASK_VALUE_PAIRS_L4_DPORT_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_TCP_FLAGS_BE_OFST 164 +#define MAE_FIELD_MASK_VALUE_PAIRS_TCP_FLAGS_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_TCP_FLAGS_BE_LBN 1312 +#define MAE_FIELD_MASK_VALUE_PAIRS_TCP_FLAGS_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_TCP_FLAGS_BE_MASK_OFST 166 +#define MAE_FIELD_MASK_VALUE_PAIRS_TCP_FLAGS_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_TCP_FLAGS_BE_MASK_LBN 1328 +#define MAE_FIELD_MASK_VALUE_PAIRS_TCP_FLAGS_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENCAP_TYPE_OFST 168 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENCAP_TYPE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENCAP_TYPE_LBN 1344 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENCAP_TYPE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENCAP_TYPE_MASK_OFST 172 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENCAP_TYPE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENCAP_TYPE_MASK_LBN 1376 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENCAP_TYPE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_OUTER_RULE_ID_OFST 176 +#define MAE_FIELD_MASK_VALUE_PAIRS_OUTER_RULE_ID_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_OUTER_RULE_ID_LBN 1408 +#define MAE_FIELD_MASK_VALUE_PAIRS_OUTER_RULE_ID_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_OUTER_RULE_ID_MASK_OFST 180 +#define MAE_FIELD_MASK_VALUE_PAIRS_OUTER_RULE_ID_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_OUTER_RULE_ID_MASK_LBN 1440 +#define MAE_FIELD_MASK_VALUE_PAIRS_OUTER_RULE_ID_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETHER_TYPE_BE_OFST 184 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETHER_TYPE_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETHER_TYPE_BE_LBN 1472 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETHER_TYPE_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETHER_TYPE_BE_MASK_OFST 188 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETHER_TYPE_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETHER_TYPE_BE_MASK_LBN 1504 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETHER_TYPE_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_TCI_BE_OFST 192 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_TCI_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_TCI_BE_LBN 1536 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_TCI_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_TCI_BE_MASK_OFST 194 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_TCI_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_TCI_BE_MASK_LBN 1552 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_TCI_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_PROTO_BE_OFST 196 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_PROTO_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_PROTO_BE_LBN 1568 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_PROTO_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_PROTO_BE_MASK_OFST 198 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_PROTO_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_PROTO_BE_MASK_LBN 1584 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN0_PROTO_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_TCI_BE_OFST 200 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_TCI_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_TCI_BE_LBN 1600 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_TCI_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_TCI_BE_MASK_OFST 202 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_TCI_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_TCI_BE_MASK_LBN 1616 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_TCI_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_PROTO_BE_OFST 204 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_PROTO_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_PROTO_BE_LBN 1632 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_PROTO_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_PROTO_BE_MASK_OFST 206 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_PROTO_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_PROTO_BE_MASK_LBN 1648 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VLAN1_PROTO_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_SADDR_BE_OFST 208 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_SADDR_BE_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_SADDR_BE_LBN 1664 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_SADDR_BE_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_SADDR_BE_MASK_OFST 214 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_SADDR_BE_MASK_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_SADDR_BE_MASK_LBN 1712 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_SADDR_BE_MASK_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_DADDR_BE_OFST 220 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_DADDR_BE_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_DADDR_BE_LBN 1760 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_DADDR_BE_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_DADDR_BE_MASK_OFST 226 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_DADDR_BE_MASK_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_DADDR_BE_MASK_LBN 1808 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_ETH_DADDR_BE_MASK_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP4_BE_OFST 232 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP4_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP4_BE_LBN 1856 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP4_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP4_BE_MASK_OFST 236 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP4_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP4_BE_MASK_LBN 1888 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP4_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP6_BE_OFST 240 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP6_BE_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP6_BE_LBN 1920 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP6_BE_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP6_BE_MASK_OFST 256 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP6_BE_MASK_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP6_BE_MASK_LBN 2048 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_SRC_IP6_BE_MASK_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP4_BE_OFST 272 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP4_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP4_BE_LBN 2176 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP4_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP4_BE_MASK_OFST 276 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP4_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP4_BE_MASK_LBN 2208 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP4_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP6_BE_OFST 280 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP6_BE_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP6_BE_LBN 2240 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP6_BE_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP6_BE_MASK_OFST 296 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP6_BE_MASK_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP6_BE_MASK_LBN 2368 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_DST_IP6_BE_MASK_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_PROTO_OFST 312 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_PROTO_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_PROTO_LBN 2496 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_PROTO_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_PROTO_MASK_OFST 313 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_PROTO_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_PROTO_MASK_LBN 2504 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_PROTO_MASK_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TOS_OFST 314 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TOS_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TOS_LBN 2512 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TOS_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TOS_MASK_OFST 315 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TOS_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TOS_MASK_LBN 2520 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TOS_MASK_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TTL_OFST 316 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TTL_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TTL_LBN 2528 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TTL_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TTL_MASK_OFST 317 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TTL_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TTL_MASK_LBN 2536 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_TTL_MASK_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_FLAGS_BE_OFST 320 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_FLAGS_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_FLAGS_BE_LBN 2560 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_FLAGS_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_FLAGS_BE_MASK_OFST 324 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_FLAGS_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_FLAGS_BE_MASK_LBN 2592 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_IP_FLAGS_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_SPORT_BE_OFST 328 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_SPORT_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_SPORT_BE_LBN 2624 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_SPORT_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_SPORT_BE_MASK_OFST 330 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_SPORT_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_SPORT_BE_MASK_LBN 2640 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_SPORT_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_DPORT_BE_OFST 332 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_DPORT_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_DPORT_BE_LBN 2656 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_DPORT_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_DPORT_BE_MASK_OFST 334 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_DPORT_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_DPORT_BE_MASK_LBN 2672 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_L4_DPORT_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VNET_ID_BE_OFST 336 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VNET_ID_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VNET_ID_BE_LBN 2688 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VNET_ID_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VNET_ID_BE_MASK_OFST 340 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VNET_ID_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VNET_ID_BE_MASK_LBN 2720 +#define MAE_FIELD_MASK_VALUE_PAIRS_ENC_VNET_ID_BE_MASK_WIDTH 32 + +/* MAE_FIELD_MASK_VALUE_PAIRS_V2 structuredef */ +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_LEN 372 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_OFST 0 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_LBN 0 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_MASK_OFST 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_MASK_LBN 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_MARK_OFST 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_MARK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_MARK_LBN 64 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_MARK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_MARK_MASK_OFST 12 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_MARK_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_MARK_MASK_LBN 96 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_MARK_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE_OFST 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE_LBN 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE_MASK_OFST 18 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE_MASK_LBN 144 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE_OFST 20 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE_LBN 160 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE_MASK_OFST 22 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE_MASK_LBN 176 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE_OFST 24 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE_LBN 192 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE_MASK_OFST 26 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE_MASK_LBN 208 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE_OFST 28 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE_LBN 224 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE_MASK_OFST 30 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE_MASK_LBN 240 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE_OFST 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE_LBN 256 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE_MASK_OFST 34 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE_MASK_LBN 272 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE_OFST 36 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE_LBN 288 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE_MASK_OFST 42 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE_MASK_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE_MASK_LBN 336 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE_MASK_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE_OFST 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE_LBN 384 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE_MASK_OFST 54 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE_MASK_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE_MASK_LBN 432 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE_MASK_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE_OFST 60 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE_LBN 480 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE_MASK_OFST 64 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE_MASK_LBN 512 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE_OFST 68 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE_LBN 544 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE_MASK_OFST 84 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE_MASK_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE_MASK_LBN 672 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE_MASK_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE_OFST 100 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE_LBN 800 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE_MASK_OFST 104 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE_MASK_LBN 832 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE_OFST 108 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE_LBN 864 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE_MASK_OFST 124 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE_MASK_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE_MASK_LBN 992 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE_MASK_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO_OFST 140 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO_LBN 1120 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO_MASK_OFST 141 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO_MASK_LBN 1128 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO_MASK_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS_OFST 142 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS_LBN 1136 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS_MASK_OFST 143 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS_MASK_LBN 1144 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS_MASK_WIDTH 8 +/* Due to hardware limitations, firmware may return + * MC_CMD_ERR_EINVAL(BAD_IP_TTL) when attempting to match on an IP_TTL value + * other than 1. + */ +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL_OFST 144 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL_LBN 1152 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL_MASK_OFST 145 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL_MASK_LBN 1160 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL_MASK_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FLAGS_BE_OFST 148 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FLAGS_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FLAGS_BE_LBN 1184 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FLAGS_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FLAGS_BE_MASK_OFST 152 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FLAGS_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FLAGS_BE_MASK_LBN 1216 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FLAGS_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE_OFST 156 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE_LBN 1248 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE_MASK_OFST 158 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE_MASK_LBN 1264 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE_OFST 160 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE_LBN 1280 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE_MASK_OFST 162 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE_MASK_LBN 1296 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_OFST 164 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_LBN 1312 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_MASK_OFST 166 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_MASK_LBN 1328 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENCAP_TYPE_OFST 168 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENCAP_TYPE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENCAP_TYPE_LBN 1344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENCAP_TYPE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENCAP_TYPE_MASK_OFST 172 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENCAP_TYPE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENCAP_TYPE_MASK_LBN 1376 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENCAP_TYPE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_OUTER_RULE_ID_OFST 176 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_OUTER_RULE_ID_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_OUTER_RULE_ID_LBN 1408 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_OUTER_RULE_ID_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_OUTER_RULE_ID_MASK_OFST 180 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_OUTER_RULE_ID_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_OUTER_RULE_ID_MASK_LBN 1440 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_OUTER_RULE_ID_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETHER_TYPE_BE_OFST 184 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETHER_TYPE_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETHER_TYPE_BE_LBN 1472 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETHER_TYPE_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETHER_TYPE_BE_MASK_OFST 188 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETHER_TYPE_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETHER_TYPE_BE_MASK_LBN 1504 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETHER_TYPE_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_TCI_BE_OFST 192 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_TCI_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_TCI_BE_LBN 1536 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_TCI_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_TCI_BE_MASK_OFST 194 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_TCI_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_TCI_BE_MASK_LBN 1552 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_TCI_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_PROTO_BE_OFST 196 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_PROTO_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_PROTO_BE_LBN 1568 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_PROTO_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_PROTO_BE_MASK_OFST 198 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_PROTO_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_PROTO_BE_MASK_LBN 1584 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN0_PROTO_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_TCI_BE_OFST 200 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_TCI_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_TCI_BE_LBN 1600 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_TCI_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_TCI_BE_MASK_OFST 202 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_TCI_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_TCI_BE_MASK_LBN 1616 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_TCI_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_PROTO_BE_OFST 204 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_PROTO_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_PROTO_BE_LBN 1632 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_PROTO_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_PROTO_BE_MASK_OFST 206 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_PROTO_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_PROTO_BE_MASK_LBN 1648 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VLAN1_PROTO_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_SADDR_BE_OFST 208 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_SADDR_BE_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_SADDR_BE_LBN 1664 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_SADDR_BE_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_SADDR_BE_MASK_OFST 214 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_SADDR_BE_MASK_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_SADDR_BE_MASK_LBN 1712 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_SADDR_BE_MASK_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_DADDR_BE_OFST 220 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_DADDR_BE_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_DADDR_BE_LBN 1760 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_DADDR_BE_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_DADDR_BE_MASK_OFST 226 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_DADDR_BE_MASK_LEN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_DADDR_BE_MASK_LBN 1808 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_ETH_DADDR_BE_MASK_WIDTH 48 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP4_BE_OFST 232 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP4_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP4_BE_LBN 1856 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP4_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP4_BE_MASK_OFST 236 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP4_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP4_BE_MASK_LBN 1888 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP4_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP6_BE_OFST 240 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP6_BE_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP6_BE_LBN 1920 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP6_BE_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP6_BE_MASK_OFST 256 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP6_BE_MASK_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP6_BE_MASK_LBN 2048 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_SRC_IP6_BE_MASK_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP4_BE_OFST 272 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP4_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP4_BE_LBN 2176 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP4_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP4_BE_MASK_OFST 276 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP4_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP4_BE_MASK_LBN 2208 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP4_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP6_BE_OFST 280 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP6_BE_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP6_BE_LBN 2240 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP6_BE_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP6_BE_MASK_OFST 296 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP6_BE_MASK_LEN 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP6_BE_MASK_LBN 2368 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_DST_IP6_BE_MASK_WIDTH 128 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_PROTO_OFST 312 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_PROTO_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_PROTO_LBN 2496 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_PROTO_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_PROTO_MASK_OFST 313 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_PROTO_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_PROTO_MASK_LBN 2504 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_PROTO_MASK_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TOS_OFST 314 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TOS_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TOS_LBN 2512 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TOS_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TOS_MASK_OFST 315 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TOS_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TOS_MASK_LBN 2520 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TOS_MASK_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TTL_OFST 316 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TTL_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TTL_LBN 2528 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TTL_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TTL_MASK_OFST 317 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TTL_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TTL_MASK_LBN 2536 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_TTL_MASK_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_FLAGS_BE_OFST 320 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_FLAGS_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_FLAGS_BE_LBN 2560 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_FLAGS_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_FLAGS_BE_MASK_OFST 324 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_FLAGS_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_FLAGS_BE_MASK_LBN 2592 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_IP_FLAGS_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_SPORT_BE_OFST 328 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_SPORT_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_SPORT_BE_LBN 2624 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_SPORT_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_SPORT_BE_MASK_OFST 330 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_SPORT_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_SPORT_BE_MASK_LBN 2640 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_SPORT_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_DPORT_BE_OFST 332 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_DPORT_BE_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_DPORT_BE_LBN 2656 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_DPORT_BE_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_DPORT_BE_MASK_OFST 334 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_DPORT_BE_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_DPORT_BE_MASK_LBN 2672 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_L4_DPORT_BE_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VNET_ID_BE_OFST 336 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VNET_ID_BE_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VNET_ID_BE_LBN 2688 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VNET_ID_BE_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VNET_ID_BE_MASK_OFST 340 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VNET_ID_BE_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VNET_ID_BE_MASK_LBN 2720 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_VNET_ID_BE_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IS_IP_FRAG_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IS_IP_FRAG_LBN 0 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IS_IP_FRAG_WIDTH 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DO_CT_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DO_CT_LBN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_DO_CT_WIDTH 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_HIT_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_HIT_LBN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_HIT_WIDTH 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IS_FROM_NETWORK_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IS_FROM_NETWORK_LBN 3 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IS_FROM_NETWORK_WIDTH 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_HAS_OVLAN_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_HAS_OVLAN_LBN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_HAS_OVLAN_WIDTH 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_HAS_IVLAN_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_HAS_IVLAN_LBN 5 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_HAS_IVLAN_WIDTH 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_HAS_OVLAN_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_HAS_OVLAN_LBN 6 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_HAS_OVLAN_WIDTH 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_HAS_IVLAN_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_HAS_IVLAN_LBN 7 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_ENC_HAS_IVLAN_WIDTH 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_SYN_FIN_RST_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_SYN_FIN_RST_LBN 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_SYN_FIN_RST_WIDTH 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FIRST_FRAG_OFST 344 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FIRST_FRAG_LBN 9 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FIRST_FRAG_WIDTH 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_LBN 2752 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_MASK_OFST 348 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_MASK_LBN 2784 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_DOMAIN_OFST 352 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_DOMAIN_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_DOMAIN_LBN 2816 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_DOMAIN_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_DOMAIN_MASK_OFST 354 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_DOMAIN_MASK_LEN 2 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_DOMAIN_MASK_LBN 2832 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_DOMAIN_MASK_WIDTH 16 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_MARK_OFST 356 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_MARK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_MARK_LBN 2848 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_MARK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_MARK_MASK_OFST 360 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_MARK_MASK_LEN 4 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_MARK_MASK_LBN 2880 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_MARK_MASK_WIDTH 32 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_PRIVATE_FLAGS_OFST 364 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_PRIVATE_FLAGS_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_PRIVATE_FLAGS_LBN 2912 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_PRIVATE_FLAGS_WIDTH 8 +/* Set to zero. */ +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD2_OFST 365 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD2_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD2_LBN 2920 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD2_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_PRIVATE_FLAGS_MASK_OFST 366 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_PRIVATE_FLAGS_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_PRIVATE_FLAGS_MASK_LBN 2928 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_CT_PRIVATE_FLAGS_MASK_WIDTH 8 +/* Set to zero. */ +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD3_OFST 367 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD3_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD3_LBN 2936 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD3_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_OFST 368 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_LBN 2944 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_WIDTH 8 +/* Set to zero */ +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD4_OFST 369 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD4_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD4_LBN 2952 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD4_WIDTH 8 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_MASK_OFST 370 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_MASK_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_MASK_LBN 2960 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_MASK_WIDTH 8 +/* Set to zero */ +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD5_OFST 371 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD5_LEN 1 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD5_LBN 2968 +#define MAE_FIELD_MASK_VALUE_PAIRS_V2_RSVD5_WIDTH 8 + +/* MAE_MPORT_SELECTOR structuredef: MPORTS are identified by an opaque unsigned + * integer value (mport_id) that is guaranteed to be representable within + * 32-bits or within any NIC interface field that needs store the value + * (whichever is narrowers). This selector structure provides a stable way to + * refer to m-ports. + */ +#define MAE_MPORT_SELECTOR_LEN 4 +/* Used to force the tools to output bitfield-style defines for this structure. + */ +#define MAE_MPORT_SELECTOR_FLAT_OFST 0 +#define MAE_MPORT_SELECTOR_FLAT_LEN 4 +/* enum: An m-port selector value that is guaranteed never to represent a real + * mport + */ +#define MAE_MPORT_SELECTOR_NULL 0x0 +/* enum: The m-port assigned to the calling client. */ +#define MAE_MPORT_SELECTOR_ASSIGNED 0x1000000 +#define MAE_MPORT_SELECTOR_TYPE_OFST 0 +#define MAE_MPORT_SELECTOR_TYPE_LBN 24 +#define MAE_MPORT_SELECTOR_TYPE_WIDTH 8 +/* enum: The MPORT connected to a given physical port */ +#define MAE_MPORT_SELECTOR_TYPE_PPORT 0x2 +/* enum: The MPORT assigned to a given PCIe function. Deprecated in favour of + * MH_FUNC. + */ +#define MAE_MPORT_SELECTOR_TYPE_FUNC 0x3 +/* enum: An mport_id */ +#define MAE_MPORT_SELECTOR_TYPE_MPORT_ID 0x4 +/* enum: The MPORT assigned to a given PCIe function (see also FWRIVERHD-1108) + */ +#define MAE_MPORT_SELECTOR_TYPE_MH_FUNC 0x5 +/* enum: This is guaranteed never to be a valid selector type */ +#define MAE_MPORT_SELECTOR_TYPE_INVALID 0xff +#define MAE_MPORT_SELECTOR_MPORT_ID_OFST 0 +#define MAE_MPORT_SELECTOR_MPORT_ID_LBN 0 +#define MAE_MPORT_SELECTOR_MPORT_ID_WIDTH 24 +#define MAE_MPORT_SELECTOR_PPORT_ID_OFST 0 +#define MAE_MPORT_SELECTOR_PPORT_ID_LBN 0 +#define MAE_MPORT_SELECTOR_PPORT_ID_WIDTH 4 +#define MAE_MPORT_SELECTOR_FUNC_INTF_ID_OFST 0 +#define MAE_MPORT_SELECTOR_FUNC_INTF_ID_LBN 20 +#define MAE_MPORT_SELECTOR_FUNC_INTF_ID_WIDTH 4 +#define MAE_MPORT_SELECTOR_HOST_PRIMARY 0x1 /* enum */ +#define MAE_MPORT_SELECTOR_NIC_EMBEDDED 0x2 /* enum */ +/* enum: Deprecated, use CALLER_INTF instead. */ +#define MAE_MPORT_SELECTOR_CALLER 0xf +#define MAE_MPORT_SELECTOR_CALLER_INTF 0xf /* enum */ +#define MAE_MPORT_SELECTOR_FUNC_MH_PF_ID_OFST 0 +#define MAE_MPORT_SELECTOR_FUNC_MH_PF_ID_LBN 16 +#define MAE_MPORT_SELECTOR_FUNC_MH_PF_ID_WIDTH 4 +#define MAE_MPORT_SELECTOR_FUNC_PF_ID_OFST 0 +#define MAE_MPORT_SELECTOR_FUNC_PF_ID_LBN 16 +#define MAE_MPORT_SELECTOR_FUNC_PF_ID_WIDTH 8 +#define MAE_MPORT_SELECTOR_FUNC_VF_ID_OFST 0 +#define MAE_MPORT_SELECTOR_FUNC_VF_ID_LBN 0 +#define MAE_MPORT_SELECTOR_FUNC_VF_ID_WIDTH 16 +/* enum: Used for VF_ID to indicate a physical function. */ +#define MAE_MPORT_SELECTOR_FUNC_VF_ID_NULL 0xffff +/* enum: Used for PF_ID to indicate the physical function of the calling + * client. - When used by a PF with VF_ID == VF_ID_NULL, the mport selector + * relates to the calling function. (For clarity, it is recommended that + * clients use ASSIGNED to achieve this behaviour). - When used by a PF with + * VF_ID != VF_ID_NULL, the mport selector relates to a VF child of the calling + * function. - When used by a VF with VF_ID == VF_ID_NULL, the mport selector + * relates to the PF owning the calling function. - When used by a VF with + * VF_ID != VF_ID_NULL, the mport selector relates to a sibling VF of the + * calling function. - Not meaningful used by a client that is not a PCIe + * function. + */ +#define MAE_MPORT_SELECTOR_FUNC_PF_ID_CALLER 0xff +/* enum: Same as PF_ID_CALLER, but for use in the smaller MH_PF_ID field. Only + * valid if FUNC_INTF_ID is CALLER. + */ +#define MAE_MPORT_SELECTOR_FUNC_MH_PF_ID_CALLER 0xf +#define MAE_MPORT_SELECTOR_FLAT_LBN 0 +#define MAE_MPORT_SELECTOR_FLAT_WIDTH 32 + +/* MAE_LINK_ENDPOINT_SELECTOR structuredef: Structure that identifies a real or + * virtual network port by MAE port and link end + */ +#define MAE_LINK_ENDPOINT_SELECTOR_LEN 8 +/* The MAE MPORT of interest */ +#define MAE_LINK_ENDPOINT_SELECTOR_MPORT_SELECTOR_OFST 0 +#define MAE_LINK_ENDPOINT_SELECTOR_MPORT_SELECTOR_LEN 4 +#define MAE_LINK_ENDPOINT_SELECTOR_MPORT_SELECTOR_LBN 0 +#define MAE_LINK_ENDPOINT_SELECTOR_MPORT_SELECTOR_WIDTH 32 +/* Which end of the link identified by MPORT to consider */ +#define MAE_LINK_ENDPOINT_SELECTOR_LINK_END_OFST 4 +#define MAE_LINK_ENDPOINT_SELECTOR_LINK_END_LEN 4 +/* Enum values, see field(s): */ +/* MAE_MPORT_END */ +#define MAE_LINK_ENDPOINT_SELECTOR_LINK_END_LBN 32 +#define MAE_LINK_ENDPOINT_SELECTOR_LINK_END_WIDTH 32 +/* A field for accessing the endpoint selector as a collection of bits */ +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_OFST 0 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_LEN 8 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_LO_OFST 0 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_LO_LEN 4 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_LO_LBN 0 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_LO_WIDTH 32 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_HI_OFST 4 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_HI_LEN 4 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_HI_LBN 32 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_HI_WIDTH 32 +/* enum: Set FLAT to this value to obtain backward-compatible behaviour in + * commands that have been extended to take a MAE_LINK_ENDPOINT_SELECTOR + * argument. New commands that are designed to take such an argument from the + * start will not support this. + */ +#define MAE_LINK_ENDPOINT_SELECTOR_MAE_LINK_ENDPOINT_COMPAT 0x0 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_LBN 0 +#define MAE_LINK_ENDPOINT_SELECTOR_FLAT_WIDTH 64 + + +/***********************************/ +/* MC_CMD_MAE_GET_CAPS + * Describes capabilities of the MAE (Match-Action Engine) + */ +#define MC_CMD_MAE_GET_CAPS 0x140 +#undef MC_CMD_0x140_PRIVILEGE_CTG + +#define MC_CMD_0x140_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_MAE_GET_CAPS_IN msgrequest */ +#define MC_CMD_MAE_GET_CAPS_IN_LEN 0 + +/* MC_CMD_MAE_GET_CAPS_OUT msgresponse */ +#define MC_CMD_MAE_GET_CAPS_OUT_LEN 52 +/* The number of field IDs that the NIC supports. Any field with a ID greater + * than or equal to the value returned in this field must be treated as having + * a support level of MAE_FIELD_UNSUPPORTED in all requests. + */ +#define MC_CMD_MAE_GET_CAPS_OUT_MATCH_FIELD_COUNT_OFST 0 +#define MC_CMD_MAE_GET_CAPS_OUT_MATCH_FIELD_COUNT_LEN 4 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPES_SUPPORTED_OFST 4 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPES_SUPPORTED_LEN 4 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_VXLAN_OFST 4 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_VXLAN_LBN 0 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_VXLAN_WIDTH 1 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_NVGRE_OFST 4 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_NVGRE_LBN 1 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_NVGRE_WIDTH 1 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_GENEVE_OFST 4 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_GENEVE_LBN 2 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_GENEVE_WIDTH 1 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_L2GRE_OFST 4 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_L2GRE_LBN 3 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_TYPE_L2GRE_WIDTH 1 +/* Deprecated alias for AR_COUNTERS. */ +#define MC_CMD_MAE_GET_CAPS_OUT_COUNTERS_OFST 8 +#define MC_CMD_MAE_GET_CAPS_OUT_COUNTERS_LEN 4 +/* The total number of AR counters available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_OUT_AR_COUNTERS_OFST 8 +#define MC_CMD_MAE_GET_CAPS_OUT_AR_COUNTERS_LEN 4 +/* The total number of counters lists available to allocate. A value of zero + * indicates that counter lists are not supported by the NIC. (But single + * counters may still be.) + */ +#define MC_CMD_MAE_GET_CAPS_OUT_COUNTER_LISTS_OFST 12 +#define MC_CMD_MAE_GET_CAPS_OUT_COUNTER_LISTS_LEN 4 +/* The total number of encap header structures available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_HEADER_LIMIT_OFST 16 +#define MC_CMD_MAE_GET_CAPS_OUT_ENCAP_HEADER_LIMIT_LEN 4 +/* Reserved. Should be zero. */ +#define MC_CMD_MAE_GET_CAPS_OUT_RSVD_OFST 20 +#define MC_CMD_MAE_GET_CAPS_OUT_RSVD_LEN 4 +/* The total number of action sets available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_OUT_ACTION_SETS_OFST 24 +#define MC_CMD_MAE_GET_CAPS_OUT_ACTION_SETS_LEN 4 +/* The total number of action set lists available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_OUT_ACTION_SET_LISTS_OFST 28 +#define MC_CMD_MAE_GET_CAPS_OUT_ACTION_SET_LISTS_LEN 4 +/* The total number of outer rules available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_OUT_OUTER_RULES_OFST 32 +#define MC_CMD_MAE_GET_CAPS_OUT_OUTER_RULES_LEN 4 +/* The total number of action rules available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_OUT_ACTION_RULES_OFST 36 +#define MC_CMD_MAE_GET_CAPS_OUT_ACTION_RULES_LEN 4 +/* The number of priorities available for ACTION_RULE filters. It is invalid to + * install a MATCH_ACTION filter with a priority number >= ACTION_PRIOS. + */ +#define MC_CMD_MAE_GET_CAPS_OUT_ACTION_PRIOS_OFST 40 +#define MC_CMD_MAE_GET_CAPS_OUT_ACTION_PRIOS_LEN 4 +/* The number of priorities available for OUTER_RULE filters. It is invalid to + * install an OUTER_RULE filter with a priority number >= OUTER_PRIOS. + */ +#define MC_CMD_MAE_GET_CAPS_OUT_OUTER_PRIOS_OFST 44 +#define MC_CMD_MAE_GET_CAPS_OUT_OUTER_PRIOS_LEN 4 +/* MAE API major version. Currently 1. If this field is not present in the + * response (i.e. response shorter than 384 bits), then its value is zero. If + * the value does not match the client's expectations, the client should raise + * a fatal error. + */ +#define MC_CMD_MAE_GET_CAPS_OUT_API_VER_OFST 48 +#define MC_CMD_MAE_GET_CAPS_OUT_API_VER_LEN 4 + +/* MC_CMD_MAE_GET_CAPS_V2_OUT msgresponse */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_LEN 60 +/* The number of field IDs that the NIC supports. Any field with a ID greater + * than or equal to the value returned in this field must be treated as having + * a support level of MAE_FIELD_UNSUPPORTED in all requests. + */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_MATCH_FIELD_COUNT_OFST 0 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_MATCH_FIELD_COUNT_LEN 4 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPES_SUPPORTED_OFST 4 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPES_SUPPORTED_LEN 4 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_VXLAN_OFST 4 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_VXLAN_LBN 0 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_VXLAN_WIDTH 1 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_NVGRE_OFST 4 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_NVGRE_LBN 1 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_NVGRE_WIDTH 1 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_GENEVE_OFST 4 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_GENEVE_LBN 2 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_GENEVE_WIDTH 1 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_L2GRE_OFST 4 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_L2GRE_LBN 3 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_TYPE_L2GRE_WIDTH 1 +/* Deprecated alias for AR_COUNTERS. */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_COUNTERS_OFST 8 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_COUNTERS_LEN 4 +/* The total number of AR counters available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_AR_COUNTERS_OFST 8 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_AR_COUNTERS_LEN 4 +/* The total number of counters lists available to allocate. A value of zero + * indicates that counter lists are not supported by the NIC. (But single + * counters may still be.) + */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_COUNTER_LISTS_OFST 12 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_COUNTER_LISTS_LEN 4 +/* The total number of encap header structures available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_HEADER_LIMIT_OFST 16 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ENCAP_HEADER_LIMIT_LEN 4 +/* Reserved. Should be zero. */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_RSVD_OFST 20 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_RSVD_LEN 4 +/* The total number of action sets available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ACTION_SETS_OFST 24 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ACTION_SETS_LEN 4 +/* The total number of action set lists available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ACTION_SET_LISTS_OFST 28 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ACTION_SET_LISTS_LEN 4 +/* The total number of outer rules available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_OUTER_RULES_OFST 32 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_OUTER_RULES_LEN 4 +/* The total number of action rules available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ACTION_RULES_OFST 36 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ACTION_RULES_LEN 4 +/* The number of priorities available for ACTION_RULE filters. It is invalid to + * install a MATCH_ACTION filter with a priority number >= ACTION_PRIOS. + */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ACTION_PRIOS_OFST 40 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_ACTION_PRIOS_LEN 4 +/* The number of priorities available for OUTER_RULE filters. It is invalid to + * install an OUTER_RULE filter with a priority number >= OUTER_PRIOS. + */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_OUTER_PRIOS_OFST 44 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_OUTER_PRIOS_LEN 4 +/* MAE API major version. Currently 1. If this field is not present in the + * response (i.e. response shorter than 384 bits), then its value is zero. If + * the value does not match the client's expectations, the client should raise + * a fatal error. + */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_API_VER_OFST 48 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_API_VER_LEN 4 +/* Mask of supported counter types. Each bit position corresponds to a value of + * the MAE_COUNTER_TYPE enum. If this field is missing (i.e. V1 response), + * clients must assume that only AR counters are supported (i.e. + * COUNTER_TYPES_SUPPORTED==0x1). See also + * MC_CMD_MAE_COUNTERS_STREAM_START/COUNTER_TYPES_MASK. + */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_COUNTER_TYPES_SUPPORTED_OFST 52 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_COUNTER_TYPES_SUPPORTED_LEN 4 +/* The total number of conntrack counters available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V2_OUT_CT_COUNTERS_OFST 56 +#define MC_CMD_MAE_GET_CAPS_V2_OUT_CT_COUNTERS_LEN 4 + +/* MC_CMD_MAE_GET_CAPS_V3_OUT msgresponse */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_LEN 64 +/* The number of field IDs that the NIC supports. Any field with a ID greater + * than or equal to the value returned in this field must be treated as having + * a support level of MAE_FIELD_UNSUPPORTED in all requests. + */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_MATCH_FIELD_COUNT_OFST 0 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_MATCH_FIELD_COUNT_LEN 4 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPES_SUPPORTED_OFST 4 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPES_SUPPORTED_LEN 4 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_VXLAN_OFST 4 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_VXLAN_LBN 0 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_VXLAN_WIDTH 1 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_NVGRE_OFST 4 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_NVGRE_LBN 1 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_NVGRE_WIDTH 1 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_GENEVE_OFST 4 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_GENEVE_LBN 2 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_GENEVE_WIDTH 1 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_L2GRE_OFST 4 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_L2GRE_LBN 3 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_TYPE_L2GRE_WIDTH 1 +/* Deprecated alias for AR_COUNTERS. */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_COUNTERS_OFST 8 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_COUNTERS_LEN 4 +/* The total number of AR counters available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_AR_COUNTERS_OFST 8 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_AR_COUNTERS_LEN 4 +/* The total number of counters lists available to allocate. A value of zero + * indicates that counter lists are not supported by the NIC. (But single + * counters may still be.) + */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_COUNTER_LISTS_OFST 12 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_COUNTER_LISTS_LEN 4 +/* The total number of encap header structures available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_HEADER_LIMIT_OFST 16 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ENCAP_HEADER_LIMIT_LEN 4 +/* Reserved. Should be zero. */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_RSVD_OFST 20 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_RSVD_LEN 4 +/* The total number of action sets available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ACTION_SETS_OFST 24 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ACTION_SETS_LEN 4 +/* The total number of action set lists available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ACTION_SET_LISTS_OFST 28 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ACTION_SET_LISTS_LEN 4 +/* The total number of outer rules available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_OUTER_RULES_OFST 32 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_OUTER_RULES_LEN 4 +/* The total number of action rules available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ACTION_RULES_OFST 36 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ACTION_RULES_LEN 4 +/* The number of priorities available for ACTION_RULE filters. It is invalid to + * install a MATCH_ACTION filter with a priority number >= ACTION_PRIOS. + */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ACTION_PRIOS_OFST 40 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_ACTION_PRIOS_LEN 4 +/* The number of priorities available for OUTER_RULE filters. It is invalid to + * install an OUTER_RULE filter with a priority number >= OUTER_PRIOS. + */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_OUTER_PRIOS_OFST 44 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_OUTER_PRIOS_LEN 4 +/* MAE API major version. Currently 1. If this field is not present in the + * response (i.e. response shorter than 384 bits), then its value is zero. If + * the value does not match the client's expectations, the client should raise + * a fatal error. + */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_API_VER_OFST 48 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_API_VER_LEN 4 +/* Mask of supported counter types. Each bit position corresponds to a value of + * the MAE_COUNTER_TYPE enum. If this field is missing (i.e. V1 response), + * clients must assume that only AR counters are supported (i.e. + * COUNTER_TYPES_SUPPORTED==0x1). See also + * MC_CMD_MAE_COUNTERS_STREAM_START/COUNTER_TYPES_MASK. + */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_COUNTER_TYPES_SUPPORTED_OFST 52 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_COUNTER_TYPES_SUPPORTED_LEN 4 +/* The total number of conntrack counters available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_CT_COUNTERS_OFST 56 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_CT_COUNTERS_LEN 4 +/* The total number of Outer Rule counters available to allocate. */ +#define MC_CMD_MAE_GET_CAPS_V3_OUT_OR_COUNTERS_OFST 60 +#define MC_CMD_MAE_GET_CAPS_V3_OUT_OR_COUNTERS_LEN 4 + + +/***********************************/ +/* MC_CMD_MAE_GET_AR_CAPS + * Get a level of support for match fields when used in match-action rules + */ +#define MC_CMD_MAE_GET_AR_CAPS 0x141 +#undef MC_CMD_0x141_PRIVILEGE_CTG + +#define MC_CMD_0x141_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_GET_AR_CAPS_IN msgrequest */ +#define MC_CMD_MAE_GET_AR_CAPS_IN_LEN 0 + +/* MC_CMD_MAE_GET_AR_CAPS_OUT msgresponse */ +#define MC_CMD_MAE_GET_AR_CAPS_OUT_LENMIN 4 +#define MC_CMD_MAE_GET_AR_CAPS_OUT_LENMAX 252 +#define MC_CMD_MAE_GET_AR_CAPS_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_MAE_GET_AR_CAPS_OUT_LEN(num) (4+4*(num)) +#define MC_CMD_MAE_GET_AR_CAPS_OUT_FIELD_FLAGS_NUM(len) (((len)-4)/4) +/* Number of fields actually returned in FIELD_FLAGS. */ +#define MC_CMD_MAE_GET_AR_CAPS_OUT_COUNT_OFST 0 +#define MC_CMD_MAE_GET_AR_CAPS_OUT_COUNT_LEN 4 +/* Array of values indicating the NIC's support for a given field, indexed by + * field id. The driver must ensure space for + * MC_CMD_MAE_GET_CAPS.MATCH_FIELD_COUNT entries in the array.. + */ +#define MC_CMD_MAE_GET_AR_CAPS_OUT_FIELD_FLAGS_OFST 4 +#define MC_CMD_MAE_GET_AR_CAPS_OUT_FIELD_FLAGS_LEN 4 +#define MC_CMD_MAE_GET_AR_CAPS_OUT_FIELD_FLAGS_MINNUM 0 +#define MC_CMD_MAE_GET_AR_CAPS_OUT_FIELD_FLAGS_MAXNUM 62 +#define MC_CMD_MAE_GET_AR_CAPS_OUT_FIELD_FLAGS_MAXNUM_MCDI2 254 + + +/***********************************/ +/* MC_CMD_MAE_GET_OR_CAPS + * Get a level of support for fields used in outer rule keys. + */ +#define MC_CMD_MAE_GET_OR_CAPS 0x142 +#undef MC_CMD_0x142_PRIVILEGE_CTG + +#define MC_CMD_0x142_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_GET_OR_CAPS_IN msgrequest */ +#define MC_CMD_MAE_GET_OR_CAPS_IN_LEN 0 + +/* MC_CMD_MAE_GET_OR_CAPS_OUT msgresponse */ +#define MC_CMD_MAE_GET_OR_CAPS_OUT_LENMIN 4 +#define MC_CMD_MAE_GET_OR_CAPS_OUT_LENMAX 252 +#define MC_CMD_MAE_GET_OR_CAPS_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_MAE_GET_OR_CAPS_OUT_LEN(num) (4+4*(num)) +#define MC_CMD_MAE_GET_OR_CAPS_OUT_FIELD_FLAGS_NUM(len) (((len)-4)/4) +/* Number of fields actually returned in FIELD_FLAGS. */ +#define MC_CMD_MAE_GET_OR_CAPS_OUT_COUNT_OFST 0 +#define MC_CMD_MAE_GET_OR_CAPS_OUT_COUNT_LEN 4 +/* Same semantics as MC_CMD_MAE_GET_AR_CAPS.MAE_FIELD_FLAGS */ +#define MC_CMD_MAE_GET_OR_CAPS_OUT_FIELD_FLAGS_OFST 4 +#define MC_CMD_MAE_GET_OR_CAPS_OUT_FIELD_FLAGS_LEN 4 +#define MC_CMD_MAE_GET_OR_CAPS_OUT_FIELD_FLAGS_MINNUM 0 +#define MC_CMD_MAE_GET_OR_CAPS_OUT_FIELD_FLAGS_MAXNUM 62 +#define MC_CMD_MAE_GET_OR_CAPS_OUT_FIELD_FLAGS_MAXNUM_MCDI2 254 + + +/***********************************/ +/* MC_CMD_MAE_COUNTER_ALLOC + * Allocate match-action-engine counters, which can be referenced in various + * tables. + */ +#define MC_CMD_MAE_COUNTER_ALLOC 0x143 +#undef MC_CMD_0x143_PRIVILEGE_CTG + +#define MC_CMD_0x143_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_COUNTER_ALLOC_IN msgrequest: Using this is equivalent to using V2 + * with COUNTER_TYPE=AR. + */ +#define MC_CMD_MAE_COUNTER_ALLOC_IN_LEN 4 +/* The number of counters that the driver would like allocated */ +#define MC_CMD_MAE_COUNTER_ALLOC_IN_REQUESTED_COUNT_OFST 0 +#define MC_CMD_MAE_COUNTER_ALLOC_IN_REQUESTED_COUNT_LEN 4 + +/* MC_CMD_MAE_COUNTER_ALLOC_V2_IN msgrequest */ +#define MC_CMD_MAE_COUNTER_ALLOC_V2_IN_LEN 8 +/* The number of counters that the driver would like allocated */ +#define MC_CMD_MAE_COUNTER_ALLOC_V2_IN_REQUESTED_COUNT_OFST 0 +#define MC_CMD_MAE_COUNTER_ALLOC_V2_IN_REQUESTED_COUNT_LEN 4 +/* Which type of counter to allocate. */ +#define MC_CMD_MAE_COUNTER_ALLOC_V2_IN_COUNTER_TYPE_OFST 4 +#define MC_CMD_MAE_COUNTER_ALLOC_V2_IN_COUNTER_TYPE_LEN 4 +/* Enum values, see field(s): */ +/* MAE_COUNTER_TYPE */ + +/* MC_CMD_MAE_COUNTER_ALLOC_OUT msgresponse */ +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_LENMIN 12 +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_LENMAX 252 +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_LEN(num) (8+4*(num)) +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NUM(len) (((len)-8)/4) +/* Generation count. Packets with generation count >= GENERATION_COUNT will + * contain valid counter values for counter IDs allocated in this call, unless + * the counter values are zero and zero squash is enabled. Note that there is + * an independent GENERATION_COUNT object per counter type, and that generation + * counts wrap from 0xffffffff to 1. + */ +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_GENERATION_COUNT_OFST 0 +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_GENERATION_COUNT_LEN 4 +/* enum: Generation counter 0 is reserved and unused. */ +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_GENERATION_COUNT_INVALID 0x0 +/* The number of counter IDs that the NIC allocated. It is never less than 1; + * failure to allocate a single counter will cause an error to be returned. It + * is never greater than REQUESTED_COUNT, but may be less. + */ +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_COUNT_OFST 4 +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_COUNT_LEN 4 +/* An array containing the IDs for the counters allocated. */ +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_OFST 8 +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_LEN 4 +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_MINNUM 1 +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_MAXNUM 61 +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_MAXNUM_MCDI2 253 +/* enum: A counter ID that is guaranteed never to represent a real counter */ +#define MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL 0xffffffff + + +/***********************************/ +/* MC_CMD_MAE_COUNTER_FREE + * Free match-action-engine counters + */ +#define MC_CMD_MAE_COUNTER_FREE 0x144 +#undef MC_CMD_0x144_PRIVILEGE_CTG + +#define MC_CMD_0x144_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_COUNTER_FREE_IN msgrequest: Using this is equivalent to using V2 + * with COUNTER_TYPE=AR. + */ +#define MC_CMD_MAE_COUNTER_FREE_IN_LENMIN 8 +#define MC_CMD_MAE_COUNTER_FREE_IN_LENMAX 132 +#define MC_CMD_MAE_COUNTER_FREE_IN_LENMAX_MCDI2 132 +#define MC_CMD_MAE_COUNTER_FREE_IN_LEN(num) (4+4*(num)) +#define MC_CMD_MAE_COUNTER_FREE_IN_FREE_COUNTER_ID_NUM(len) (((len)-4)/4) +/* The number of counter IDs to be freed. */ +#define MC_CMD_MAE_COUNTER_FREE_IN_COUNTER_ID_COUNT_OFST 0 +#define MC_CMD_MAE_COUNTER_FREE_IN_COUNTER_ID_COUNT_LEN 4 +/* An array containing the counter IDs to be freed. */ +#define MC_CMD_MAE_COUNTER_FREE_IN_FREE_COUNTER_ID_OFST 4 +#define MC_CMD_MAE_COUNTER_FREE_IN_FREE_COUNTER_ID_LEN 4 +#define MC_CMD_MAE_COUNTER_FREE_IN_FREE_COUNTER_ID_MINNUM 1 +#define MC_CMD_MAE_COUNTER_FREE_IN_FREE_COUNTER_ID_MAXNUM 32 +#define MC_CMD_MAE_COUNTER_FREE_IN_FREE_COUNTER_ID_MAXNUM_MCDI2 32 + +/* MC_CMD_MAE_COUNTER_FREE_V2_IN msgrequest */ +#define MC_CMD_MAE_COUNTER_FREE_V2_IN_LEN 136 +/* The number of counter IDs to be freed. */ +#define MC_CMD_MAE_COUNTER_FREE_V2_IN_COUNTER_ID_COUNT_OFST 0 +#define MC_CMD_MAE_COUNTER_FREE_V2_IN_COUNTER_ID_COUNT_LEN 4 +/* An array containing the counter IDs to be freed. */ +#define MC_CMD_MAE_COUNTER_FREE_V2_IN_FREE_COUNTER_ID_OFST 4 +#define MC_CMD_MAE_COUNTER_FREE_V2_IN_FREE_COUNTER_ID_LEN 4 +#define MC_CMD_MAE_COUNTER_FREE_V2_IN_FREE_COUNTER_ID_MINNUM 1 +#define MC_CMD_MAE_COUNTER_FREE_V2_IN_FREE_COUNTER_ID_MAXNUM 32 +#define MC_CMD_MAE_COUNTER_FREE_V2_IN_FREE_COUNTER_ID_MAXNUM_MCDI2 32 +/* Which type of counter to free. */ +#define MC_CMD_MAE_COUNTER_FREE_V2_IN_COUNTER_TYPE_OFST 132 +#define MC_CMD_MAE_COUNTER_FREE_V2_IN_COUNTER_TYPE_LEN 4 +/* Enum values, see field(s): */ +/* MAE_COUNTER_TYPE */ + +/* MC_CMD_MAE_COUNTER_FREE_OUT msgresponse */ +#define MC_CMD_MAE_COUNTER_FREE_OUT_LENMIN 12 +#define MC_CMD_MAE_COUNTER_FREE_OUT_LENMAX 136 +#define MC_CMD_MAE_COUNTER_FREE_OUT_LENMAX_MCDI2 136 +#define MC_CMD_MAE_COUNTER_FREE_OUT_LEN(num) (8+4*(num)) +#define MC_CMD_MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID_NUM(len) (((len)-8)/4) +/* Generation count. A packet with generation count == GENERATION_COUNT will + * contain the final values for these counter IDs, unless the counter values + * are zero and zero squash is enabled. Note that the GENERATION_COUNT value is + * specific to the COUNTER_TYPE (IDENTIFIER field in packet header). Receiving + * a packet with generation count > GENERATION_COUNT guarantees that no more + * values will be written for these counters. If values for these counter IDs + * are present, the counter ID has been reallocated. A counter ID will not be + * reallocated within a single read cycle as this would merge increments from + * the 'old' and 'new' counters. GENERATION_COUNT_INVALID is reserved and + * unused. + */ +#define MC_CMD_MAE_COUNTER_FREE_OUT_GENERATION_COUNT_OFST 0 +#define MC_CMD_MAE_COUNTER_FREE_OUT_GENERATION_COUNT_LEN 4 +/* The number of counter IDs actually freed. It is never less than 1; failure + * to free a single counter will cause an error to be returned. It is never + * greater than the number that were requested to be freed, but may be less if + * counters could not be freed. + */ +#define MC_CMD_MAE_COUNTER_FREE_OUT_COUNTER_ID_COUNT_OFST 4 +#define MC_CMD_MAE_COUNTER_FREE_OUT_COUNTER_ID_COUNT_LEN 4 +/* An array containing the IDs for the counters to that were freed. Note, + * failure to free a counter can only occur on incorrect driver behaviour, so + * asserting that the expected counters were freed is reasonable. When + * debugging, attempting to free a single counter at a time will provide a + * reason for the failure to free said counter. + */ +#define MC_CMD_MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID_OFST 8 +#define MC_CMD_MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID_LEN 4 +#define MC_CMD_MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID_MINNUM 1 +#define MC_CMD_MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID_MAXNUM 32 +#define MC_CMD_MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID_MAXNUM_MCDI2 32 + + +/***********************************/ +/* MC_CMD_MAE_COUNTERS_STREAM_START + * Start streaming counter values, specifying an RxQ to deliver packets to. + * Counters allocated to the calling function will be written in a round robin + * at a fixed cycle rate, assuming sufficient credits are available. The driver + * may cause the counter values to be written at a slower rate by constraining + * the availability of credits. Note that if the driver wishes to deliver + * packets to a different queue, it must call MAE_COUNTERS_STREAM_STOP to stop + * delivering packets to the current queue first. + */ +#define MC_CMD_MAE_COUNTERS_STREAM_START 0x151 +#undef MC_CMD_0x151_PRIVILEGE_CTG + +#define MC_CMD_0x151_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_COUNTERS_STREAM_START_IN msgrequest: Using V1 is equivalent to V2 + * with COUNTER_TYPES_MASK=0x1 (i.e. AR counters only). + */ +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_LEN 8 +/* The RxQ to write packets to. */ +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_QID_OFST 0 +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_QID_LEN 2 +/* Maximum size in bytes of packets that may be written to the RxQ. */ +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_PACKET_SIZE_OFST 2 +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_PACKET_SIZE_LEN 2 +/* Optional flags. */ +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_FLAGS_OFST 4 +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_FLAGS_LEN 4 +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_ZERO_SQUASH_DISABLE_OFST 4 +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_ZERO_SQUASH_DISABLE_LBN 0 +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_ZERO_SQUASH_DISABLE_WIDTH 1 +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_COUNTER_STALL_EN_OFST 4 +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_COUNTER_STALL_EN_LBN 1 +#define MC_CMD_MAE_COUNTERS_STREAM_START_IN_COUNTER_STALL_EN_WIDTH 1 + +/* MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN msgrequest */ +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_LEN 12 +/* The RxQ to write packets to. */ +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_QID_OFST 0 +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_QID_LEN 2 +/* Maximum size in bytes of packets that may be written to the RxQ. */ +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_PACKET_SIZE_OFST 2 +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_PACKET_SIZE_LEN 2 +/* Optional flags. */ +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_FLAGS_OFST 4 +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_FLAGS_LEN 4 +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_ZERO_SQUASH_DISABLE_OFST 4 +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_ZERO_SQUASH_DISABLE_LBN 0 +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_ZERO_SQUASH_DISABLE_WIDTH 1 +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_COUNTER_STALL_EN_OFST 4 +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_COUNTER_STALL_EN_LBN 1 +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_COUNTER_STALL_EN_WIDTH 1 +/* Mask of which counter types should be reported. Each bit position + * corresponds to a value of the MAE_COUNTER_TYPE enum. For example a value of + * 0x3 requests both AR and CT counters. A value of zero is invalid. Counter + * types not selected by the mask value won't be included in the stream. If a + * client wishes to change which counter types are reported, it must first call + * MAE_COUNTERS_STREAM_STOP, then restart it with the new mask value. + * Requesting a counter type which isn't supported by firmware (reported in + * MC_CMD_MAE_GET_CAPS/COUNTER_TYPES_SUPPORTED) will result in ENOTSUP. + */ +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_COUNTER_TYPES_MASK_OFST 8 +#define MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_COUNTER_TYPES_MASK_LEN 4 + +/* MC_CMD_MAE_COUNTERS_STREAM_START_OUT msgresponse */ +#define MC_CMD_MAE_COUNTERS_STREAM_START_OUT_LEN 4 +#define MC_CMD_MAE_COUNTERS_STREAM_START_OUT_FLAGS_OFST 0 +#define MC_CMD_MAE_COUNTERS_STREAM_START_OUT_FLAGS_LEN 4 +#define MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_OFST 0 +#define MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_LBN 0 +#define MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_WIDTH 1 + + +/***********************************/ +/* MC_CMD_MAE_COUNTERS_STREAM_STOP + * Stop streaming counter values to the specified RxQ. + */ +#define MC_CMD_MAE_COUNTERS_STREAM_STOP 0x152 +#undef MC_CMD_0x152_PRIVILEGE_CTG + +#define MC_CMD_0x152_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_COUNTERS_STREAM_STOP_IN msgrequest */ +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_IN_LEN 2 +/* The RxQ to stop writing packets to. */ +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_IN_QID_OFST 0 +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_IN_QID_LEN 2 + +/* MC_CMD_MAE_COUNTERS_STREAM_STOP_OUT msgresponse */ +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_OUT_LEN 4 +/* Generation count for AR counters. The final set of AR counter values will be + * written out in packets with count == GENERATION_COUNT. An empty packet with + * count > GENERATION_COUNT indicates that no more counter values of this type + * will be written to this stream. GENERATION_COUNT_INVALID is reserved and + * unused. + */ +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_OUT_GENERATION_COUNT_OFST 0 +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_OUT_GENERATION_COUNT_LEN 4 + +/* MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT msgresponse */ +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_LENMIN 4 +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_LENMAX 32 +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_LENMAX_MCDI2 32 +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_GENERATION_COUNT_NUM(len) (((len)-0)/4) +/* Array of generation counts, indexed by MAE_COUNTER_TYPE. Note that since + * MAE_COUNTER_TYPE_AR==0, this response is backwards-compatible with V1. The + * final set of counter values will be written out in packets with count == + * GENERATION_COUNT. An empty packet with count > GENERATION_COUNT indicates + * that no more counter values of this type will be written to this stream. + * GENERATION_COUNT_INVALID is reserved and unused. + */ +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_GENERATION_COUNT_OFST 0 +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_GENERATION_COUNT_LEN 4 +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_GENERATION_COUNT_MINNUM 1 +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_GENERATION_COUNT_MAXNUM 8 +#define MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_GENERATION_COUNT_MAXNUM_MCDI2 8 + + +/***********************************/ +/* MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS + * Give a number of credits to the packetiser. Each credit received allows the + * MC to write one packet to the RxQ, therefore for each credit the driver must + * have written sufficient descriptors for a packet of length + * MAE_COUNTERS_PACKETISER_STREAM_START/PACKET_SIZE and rung the doorbell. + */ +#define MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS 0x153 +#undef MC_CMD_0x153_PRIVILEGE_CTG + +#define MC_CMD_0x153_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_IN msgrequest */ +#define MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_LEN 4 +/* Number of credits to give to the packetiser. */ +#define MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_NUM_CREDITS_OFST 0 +#define MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_NUM_CREDITS_LEN 4 + +/* MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_OUT msgresponse */ +#define MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_MAE_ENCAP_HEADER_ALLOC + * Allocate an encapsulation header to be used in an Action Rule response. The + * header must be constructed as a valid packet with 0-length payload. + * Specifically, the L3/L4 lengths & checksums will only be incrementally fixed + * by the NIC, rather than recomputed entirely. Currently only IPv4, IPv6 and + * UDP are supported. If the maximum number of headers have already been + * allocated then the command will fail with MC_CMD_ERR_ENOSPC. + */ +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC 0x148 +#undef MC_CMD_0x148_PRIVILEGE_CTG + +#define MC_CMD_0x148_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN msgrequest */ +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_LENMIN 4 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_LENMAX 252 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_LENMAX_MCDI2 1020 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_LEN(num) (4+1*(num)) +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_HDR_DATA_NUM(len) (((len)-4)/1) +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_ENCAP_TYPE_OFST 0 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_ENCAP_TYPE_LEN 4 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_HDR_DATA_OFST 4 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_HDR_DATA_LEN 1 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_HDR_DATA_MINNUM 0 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_HDR_DATA_MAXNUM 248 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_HDR_DATA_MAXNUM_MCDI2 1016 + +/* MC_CMD_MAE_ENCAP_HEADER_ALLOC_OUT msgresponse */ +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_OUT_LEN 4 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_OUT_ENCAP_HEADER_ID_OFST 0 +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_OUT_ENCAP_HEADER_ID_LEN 4 +/* enum: An encap metadata ID that is guaranteed never to represent real encap + * metadata + */ +#define MC_CMD_MAE_ENCAP_HEADER_ALLOC_OUT_ENCAP_HEADER_ID_NULL 0xffffffff + + +/***********************************/ +/* MC_CMD_MAE_ENCAP_HEADER_UPDATE + * Update encap action metadata. See comments for MAE_ENCAP_HEADER_ALLOC. + */ +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE 0x149 +#undef MC_CMD_0x149_PRIVILEGE_CTG + +#define MC_CMD_0x149_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN msgrequest */ +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_LENMIN 8 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_LENMAX 252 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_LENMAX_MCDI2 1020 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_LEN(num) (8+1*(num)) +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_HDR_DATA_NUM(len) (((len)-8)/1) +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_EH_ID_OFST 0 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_EH_ID_LEN 4 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_ENCAP_TYPE_OFST 4 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_ENCAP_TYPE_LEN 4 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_HDR_DATA_OFST 8 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_HDR_DATA_LEN 1 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_HDR_DATA_MINNUM 0 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_HDR_DATA_MAXNUM 244 +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_IN_HDR_DATA_MAXNUM_MCDI2 1012 + +/* MC_CMD_MAE_ENCAP_HEADER_UPDATE_OUT msgresponse */ +#define MC_CMD_MAE_ENCAP_HEADER_UPDATE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_MAE_ENCAP_HEADER_FREE + * Free encap action metadata + */ +#define MC_CMD_MAE_ENCAP_HEADER_FREE 0x14a +#undef MC_CMD_0x14a_PRIVILEGE_CTG + +#define MC_CMD_0x14a_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_ENCAP_HEADER_FREE_IN msgrequest */ +#define MC_CMD_MAE_ENCAP_HEADER_FREE_IN_LENMIN 4 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_IN_LENMAX 128 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_IN_LENMAX_MCDI2 128 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_IN_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_ENCAP_HEADER_FREE_IN_EH_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_ENCAP_HEADER_FREE_IN_EH_ID_OFST 0 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_IN_EH_ID_LEN 4 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_IN_EH_ID_MINNUM 1 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_IN_EH_ID_MAXNUM 32 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_IN_EH_ID_MAXNUM_MCDI2 32 + +/* MC_CMD_MAE_ENCAP_HEADER_FREE_OUT msgresponse */ +#define MC_CMD_MAE_ENCAP_HEADER_FREE_OUT_LENMIN 4 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_OUT_LENMAX 128 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_OUT_LENMAX_MCDI2 128 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_OUT_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_ENCAP_HEADER_FREE_OUT_FREED_EH_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_ENCAP_HEADER_FREE_OUT_FREED_EH_ID_OFST 0 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_OUT_FREED_EH_ID_LEN 4 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_OUT_FREED_EH_ID_MINNUM 1 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_OUT_FREED_EH_ID_MAXNUM 32 +#define MC_CMD_MAE_ENCAP_HEADER_FREE_OUT_FREED_EH_ID_MAXNUM_MCDI2 32 + + +/***********************************/ +/* MC_CMD_MAE_MAC_ADDR_ALLOC + * Allocate MAC address. Hardware implementations have MAC addresses programmed + * into an indirection table, and clients should take care not to allocate the + * same MAC address twice (but instead reuse its ID). If the maximum number of + * MAC addresses have already been allocated then the command will fail with + * MC_CMD_ERR_ENOSPC. + */ +#define MC_CMD_MAE_MAC_ADDR_ALLOC 0x15e +#undef MC_CMD_0x15e_PRIVILEGE_CTG + +#define MC_CMD_0x15e_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_MAC_ADDR_ALLOC_IN msgrequest */ +#define MC_CMD_MAE_MAC_ADDR_ALLOC_IN_LEN 6 +/* MAC address as bytes in network order. */ +#define MC_CMD_MAE_MAC_ADDR_ALLOC_IN_MAC_ADDR_OFST 0 +#define MC_CMD_MAE_MAC_ADDR_ALLOC_IN_MAC_ADDR_LEN 6 + +/* MC_CMD_MAE_MAC_ADDR_ALLOC_OUT msgresponse */ +#define MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_LEN 4 +#define MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_OFST 0 +#define MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_LEN 4 +/* enum: An MAC address ID that is guaranteed never to represent a real MAC + * address. + */ +#define MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL 0xffffffff + + +/***********************************/ +/* MC_CMD_MAE_MAC_ADDR_FREE + * Free MAC address. + */ +#define MC_CMD_MAE_MAC_ADDR_FREE 0x15f +#undef MC_CMD_0x15f_PRIVILEGE_CTG + +#define MC_CMD_0x15f_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_MAC_ADDR_FREE_IN msgrequest */ +#define MC_CMD_MAE_MAC_ADDR_FREE_IN_LENMIN 4 +#define MC_CMD_MAE_MAC_ADDR_FREE_IN_LENMAX 128 +#define MC_CMD_MAE_MAC_ADDR_FREE_IN_LENMAX_MCDI2 128 +#define MC_CMD_MAE_MAC_ADDR_FREE_IN_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_MAC_ADDR_FREE_IN_MAC_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_MAC_ADDR_FREE_IN_MAC_ID_OFST 0 +#define MC_CMD_MAE_MAC_ADDR_FREE_IN_MAC_ID_LEN 4 +#define MC_CMD_MAE_MAC_ADDR_FREE_IN_MAC_ID_MINNUM 1 +#define MC_CMD_MAE_MAC_ADDR_FREE_IN_MAC_ID_MAXNUM 32 +#define MC_CMD_MAE_MAC_ADDR_FREE_IN_MAC_ID_MAXNUM_MCDI2 32 + +/* MC_CMD_MAE_MAC_ADDR_FREE_OUT msgresponse */ +#define MC_CMD_MAE_MAC_ADDR_FREE_OUT_LENMIN 4 +#define MC_CMD_MAE_MAC_ADDR_FREE_OUT_LENMAX 128 +#define MC_CMD_MAE_MAC_ADDR_FREE_OUT_LENMAX_MCDI2 128 +#define MC_CMD_MAE_MAC_ADDR_FREE_OUT_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_MAC_ADDR_FREE_OUT_FREED_MAC_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_MAC_ADDR_FREE_OUT_FREED_MAC_ID_OFST 0 +#define MC_CMD_MAE_MAC_ADDR_FREE_OUT_FREED_MAC_ID_LEN 4 +#define MC_CMD_MAE_MAC_ADDR_FREE_OUT_FREED_MAC_ID_MINNUM 1 +#define MC_CMD_MAE_MAC_ADDR_FREE_OUT_FREED_MAC_ID_MAXNUM 32 +#define MC_CMD_MAE_MAC_ADDR_FREE_OUT_FREED_MAC_ID_MAXNUM_MCDI2 32 + + +/***********************************/ +/* MC_CMD_MAE_ACTION_SET_ALLOC + * Allocate an action set, which can be referenced either in response to an + * Action Rule, or as part of an Action Set List. If the maxmimum number of + * action sets have already been allocated then the command will fail with + * MC_CMD_ERR_ENOSPC. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC 0x14d +#undef MC_CMD_0x14d_PRIVILEGE_CTG + +#define MC_CMD_0x14d_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_ACTION_SET_ALLOC_IN msgrequest */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_LEN 44 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_FLAGS_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_FLAGS_LEN 4 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN_PUSH_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN_PUSH_LBN 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN_PUSH_WIDTH 2 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN_POP_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN_POP_LBN 4 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN_POP_WIDTH 2 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DECAP_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DECAP_LBN 8 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DECAP_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_MARK_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_MARK_LBN 9 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_MARK_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_FLAG_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_FLAG_LBN 10 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_FLAG_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DO_NAT_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DO_NAT_LBN 11 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DO_NAT_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DO_DECR_IP_TTL_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DO_DECR_IP_TTL_LBN 12 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DO_DECR_IP_TTL_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DO_SET_SRC_MPORT_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DO_SET_SRC_MPORT_LBN 13 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DO_SET_SRC_MPORT_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_SUPPRESS_SELF_DELIVERY_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_SUPPRESS_SELF_DELIVERY_LBN 14 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_SUPPRESS_SELF_DELIVERY_WIDTH 1 +/* If VLAN_PUSH >= 1, TCI value to be inserted as outermost VLAN. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN0_TCI_BE_OFST 4 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN0_TCI_BE_LEN 2 +/* If VLAN_PUSH >= 1, TPID value to be inserted as outermost VLAN. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN0_PROTO_BE_OFST 6 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN0_PROTO_BE_LEN 2 +/* If VLAN_PUSH == 2, inner TCI value to be inserted. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN1_TCI_BE_OFST 8 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN1_TCI_BE_LEN 2 +/* If VLAN_PUSH == 2, inner TPID value to be inserted. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN1_PROTO_BE_OFST 10 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_VLAN1_PROTO_BE_LEN 2 +/* Reserved. Ignored by firmware. Should be set to zero or 0xffffffff. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_RSVD_OFST 12 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_RSVD_LEN 4 +/* Set to ENCAP_HEADER_ID_NULL to request no encap action */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_ENCAP_HEADER_ID_OFST 16 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_ENCAP_HEADER_ID_LEN 4 +/* An m-port selector identifying the m-port that the modified packet should be + * delivered to. Set to MPORT_SELECTOR_NULL to request no delivery of the + * packet. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DELIVER_OFST 20 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DELIVER_LEN 4 +/* Allows an action set to trigger several counter updates. Set to + * COUNTER_LIST_ID_NULL to request no counter action. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_COUNTER_LIST_ID_OFST 24 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_COUNTER_LIST_ID_LEN 4 +/* If a driver only wished to update one counter within this action set, then + * it can supply a COUNTER_ID instead of allocating a single-element counter + * list. The ID must have been allocated with COUNTER_TYPE=AR. This field + * should be set to COUNTER_ID_NULL if this behaviour is not required. It is + * not valid to supply a non-NULL value for both COUNTER_LIST_ID and + * COUNTER_ID. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_COUNTER_ID_OFST 28 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_COUNTER_ID_LEN 4 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_MARK_VALUE_OFST 32 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_MARK_VALUE_LEN 4 +/* Set to MAC_ID_NULL to request no source MAC replacement. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_SRC_MAC_ID_OFST 36 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_SRC_MAC_ID_LEN 4 +/* Set to MAC_ID_NULL to request no destination MAC replacement. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DST_MAC_ID_OFST 40 +#define MC_CMD_MAE_ACTION_SET_ALLOC_IN_DST_MAC_ID_LEN 4 + +/* MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN msgrequest: Only supported if + * MAE_ACTION_SET_ALLOC_V2_SUPPORTED is advertised in + * MC_CMD_GET_CAPABILITIES_V7_OUT. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_LEN 51 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_FLAGS_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_FLAGS_LEN 4 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN_PUSH_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN_PUSH_LBN 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN_PUSH_WIDTH 2 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN_POP_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN_POP_LBN 4 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN_POP_WIDTH 2 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DECAP_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DECAP_LBN 8 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DECAP_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_MARK_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_MARK_LBN 9 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_MARK_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_FLAG_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_FLAG_LBN 10 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_FLAG_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_NAT_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_NAT_LBN 11 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_NAT_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_DECR_IP_TTL_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_DECR_IP_TTL_LBN 12 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_DECR_IP_TTL_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_SET_SRC_MPORT_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_SET_SRC_MPORT_LBN 13 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_SET_SRC_MPORT_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_SUPPRESS_SELF_DELIVERY_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_SUPPRESS_SELF_DELIVERY_LBN 14 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_SUPPRESS_SELF_DELIVERY_WIDTH 1 +/* If VLAN_PUSH >= 1, TCI value to be inserted as outermost VLAN. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN0_TCI_BE_OFST 4 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN0_TCI_BE_LEN 2 +/* If VLAN_PUSH >= 1, TPID value to be inserted as outermost VLAN. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN0_PROTO_BE_OFST 6 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN0_PROTO_BE_LEN 2 +/* If VLAN_PUSH == 2, inner TCI value to be inserted. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN1_TCI_BE_OFST 8 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN1_TCI_BE_LEN 2 +/* If VLAN_PUSH == 2, inner TPID value to be inserted. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN1_PROTO_BE_OFST 10 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_VLAN1_PROTO_BE_LEN 2 +/* Reserved. Ignored by firmware. Should be set to zero or 0xffffffff. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_RSVD_OFST 12 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_RSVD_LEN 4 +/* Set to ENCAP_HEADER_ID_NULL to request no encap action */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ENCAP_HEADER_ID_OFST 16 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ENCAP_HEADER_ID_LEN 4 +/* An m-port selector identifying the m-port that the modified packet should be + * delivered to. Set to MPORT_SELECTOR_NULL to request no delivery of the + * packet. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DELIVER_OFST 20 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DELIVER_LEN 4 +/* Allows an action set to trigger several counter updates. Set to + * COUNTER_LIST_ID_NULL to request no counter action. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_COUNTER_LIST_ID_OFST 24 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_COUNTER_LIST_ID_LEN 4 +/* If a driver only wished to update one counter within this action set, then + * it can supply a COUNTER_ID instead of allocating a single-element counter + * list. The ID must have been allocated with COUNTER_TYPE=AR. This field + * should be set to COUNTER_ID_NULL if this behaviour is not required. It is + * not valid to supply a non-NULL value for both COUNTER_LIST_ID and + * COUNTER_ID. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_COUNTER_ID_OFST 28 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_COUNTER_ID_LEN 4 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_MARK_VALUE_OFST 32 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_MARK_VALUE_LEN 4 +/* Set to MAC_ID_NULL to request no source MAC replacement. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_SRC_MAC_ID_OFST 36 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_SRC_MAC_ID_LEN 4 +/* Set to MAC_ID_NULL to request no destination MAC replacement. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DST_MAC_ID_OFST 40 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DST_MAC_ID_LEN 4 +/* Source m-port ID to be reported for DO_SET_SRC_MPORT action. */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_REPORTED_SRC_MPORT_OFST 44 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_REPORTED_SRC_MPORT_LEN 4 +/* Actions for modifying the Differentiated Services Code-Point (DSCP) bits + * within IPv4 and IPv6 headers. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DSCP_CONTROL_OFST 48 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DSCP_CONTROL_LEN 2 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_DSCP_ENCAP_COPY_OFST 48 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_DSCP_ENCAP_COPY_LBN 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_DSCP_ENCAP_COPY_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_DSCP_DECAP_COPY_OFST 48 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_DSCP_DECAP_COPY_LBN 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_DSCP_DECAP_COPY_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_REPLACE_DSCP_OFST 48 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_REPLACE_DSCP_LBN 2 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_REPLACE_DSCP_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DSCP_VALUE_OFST 48 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DSCP_VALUE_LBN 3 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DSCP_VALUE_WIDTH 6 +/* Actions for modifying the Explicit Congestion Notification (ECN) bits within + * IPv4 and IPv6 headers. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_CONTROL_OFST 50 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_CONTROL_LEN 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_ECN_ENCAP_COPY_OFST 50 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_ECN_ENCAP_COPY_LBN 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_ECN_ENCAP_COPY_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_ECN_DECAP_COPY_OFST 50 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_ECN_DECAP_COPY_LBN 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_ECN_DECAP_COPY_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_REPLACE_ECN_OFST 50 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_REPLACE_ECN_LBN 2 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_DO_REPLACE_ECN_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_VALUE_OFST 50 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_VALUE_LBN 3 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_VALUE_WIDTH 2 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_ECT_0_TO_CE_OFST 50 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_ECT_0_TO_CE_LBN 5 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_ECT_0_TO_CE_WIDTH 1 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_ECT_1_TO_CE_OFST 50 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_ECT_1_TO_CE_LBN 6 +#define MC_CMD_MAE_ACTION_SET_ALLOC_V2_IN_ECN_ECT_1_TO_CE_WIDTH 1 + +/* MC_CMD_MAE_ACTION_SET_ALLOC_OUT msgresponse */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_OUT_LEN 4 +/* The MSB of the AS_ID is guaranteed to be clear if the ID is not + * ACTION_SET_ID_NULL. This allows an AS_ID to be distinguished from an ASL_ID + * returned from MC_CMD_MAE_ACTION_SET_LIST_ALLOC. + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_OUT_AS_ID_OFST 0 +#define MC_CMD_MAE_ACTION_SET_ALLOC_OUT_AS_ID_LEN 4 +/* enum: An action set ID that is guaranteed never to represent an action set + */ +#define MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL 0xffffffff + + +/***********************************/ +/* MC_CMD_MAE_ACTION_SET_FREE + */ +#define MC_CMD_MAE_ACTION_SET_FREE 0x14e +#undef MC_CMD_0x14e_PRIVILEGE_CTG + +#define MC_CMD_0x14e_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_ACTION_SET_FREE_IN msgrequest */ +#define MC_CMD_MAE_ACTION_SET_FREE_IN_LENMIN 4 +#define MC_CMD_MAE_ACTION_SET_FREE_IN_LENMAX 128 +#define MC_CMD_MAE_ACTION_SET_FREE_IN_LENMAX_MCDI2 128 +#define MC_CMD_MAE_ACTION_SET_FREE_IN_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_ACTION_SET_FREE_IN_AS_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_ACTION_SET_FREE_IN_AS_ID_OFST 0 +#define MC_CMD_MAE_ACTION_SET_FREE_IN_AS_ID_LEN 4 +#define MC_CMD_MAE_ACTION_SET_FREE_IN_AS_ID_MINNUM 1 +#define MC_CMD_MAE_ACTION_SET_FREE_IN_AS_ID_MAXNUM 32 +#define MC_CMD_MAE_ACTION_SET_FREE_IN_AS_ID_MAXNUM_MCDI2 32 + +/* MC_CMD_MAE_ACTION_SET_FREE_OUT msgresponse */ +#define MC_CMD_MAE_ACTION_SET_FREE_OUT_LENMIN 4 +#define MC_CMD_MAE_ACTION_SET_FREE_OUT_LENMAX 128 +#define MC_CMD_MAE_ACTION_SET_FREE_OUT_LENMAX_MCDI2 128 +#define MC_CMD_MAE_ACTION_SET_FREE_OUT_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_ACTION_SET_FREE_OUT_FREED_AS_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_ACTION_SET_FREE_OUT_FREED_AS_ID_OFST 0 +#define MC_CMD_MAE_ACTION_SET_FREE_OUT_FREED_AS_ID_LEN 4 +#define MC_CMD_MAE_ACTION_SET_FREE_OUT_FREED_AS_ID_MINNUM 1 +#define MC_CMD_MAE_ACTION_SET_FREE_OUT_FREED_AS_ID_MAXNUM 32 +#define MC_CMD_MAE_ACTION_SET_FREE_OUT_FREED_AS_ID_MAXNUM_MCDI2 32 + + +/***********************************/ +/* MC_CMD_MAE_ACTION_SET_LIST_ALLOC + * Allocate an action set list (ASL) that can be referenced by an ID. The ASL + * ID can be used when inserting an action rule, so that for each packet + * matching the rule every action set in the list is applied. If the maximum + * number of ASLs have already been allocated then the command will fail with + * MC_CMD_ERR_ENOSPC. + */ +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC 0x14f +#undef MC_CMD_0x14f_PRIVILEGE_CTG + +#define MC_CMD_0x14f_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN msgrequest */ +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_LENMIN 8 +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_LENMAX 252 +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_LENMAX_MCDI2 1020 +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_LEN(num) (4+4*(num)) +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_AS_IDS_NUM(len) (((len)-4)/4) +/* Number of elements in the AS_IDS field. */ +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_COUNT_OFST 0 +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_COUNT_LEN 4 +/* The IDs of the action sets in this list. The last element of this list may + * be the ID of an already allocated ASL. In this case the action sets from the + * already allocated ASL will be applied after the action sets supplied by this + * request. This mechanism can be used to reduce resource usage in the case + * where one ASL is a sublist of another ASL. The sublist should be allocated + * first, then the superlist should be allocated by supplying all required + * action set IDs that are not in the sublist followed by the ID of the + * sublist. One sublist can be referenced by multiple superlists. + */ +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_AS_IDS_OFST 4 +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_AS_IDS_LEN 4 +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_AS_IDS_MINNUM 1 +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_AS_IDS_MAXNUM 62 +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_IN_AS_IDS_MAXNUM_MCDI2 254 + +/* MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT msgresponse */ +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_LEN 4 +/* The MSB of the ASL_ID is guaranteed to be set. This allows an ASL_ID to be + * distinguished from an AS_ID returned from MC_CMD_MAE_ACTION_SET_ALLOC. + */ +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ASL_ID_OFST 0 +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ASL_ID_LEN 4 +/* enum: An action set list ID that is guaranteed never to represent an action + * set list + */ +#define MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL 0xffffffff + + +/***********************************/ +/* MC_CMD_MAE_ACTION_SET_LIST_FREE + * Free match-action-engine redirect_lists + */ +#define MC_CMD_MAE_ACTION_SET_LIST_FREE 0x150 +#undef MC_CMD_0x150_PRIVILEGE_CTG + +#define MC_CMD_0x150_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_ACTION_SET_LIST_FREE_IN msgrequest */ +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_LENMIN 4 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_LENMAX 128 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_LENMAX_MCDI2 128 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_ASL_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_ASL_ID_OFST 0 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_ASL_ID_LEN 4 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_ASL_ID_MINNUM 1 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_ASL_ID_MAXNUM 32 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_IN_ASL_ID_MAXNUM_MCDI2 32 + +/* MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT msgresponse */ +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_LENMIN 4 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_LENMAX 128 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_LENMAX_MCDI2 128 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_FREED_ASL_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_FREED_ASL_ID_OFST 0 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_FREED_ASL_ID_LEN 4 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_FREED_ASL_ID_MINNUM 1 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_FREED_ASL_ID_MAXNUM 32 +#define MC_CMD_MAE_ACTION_SET_LIST_FREE_OUT_FREED_ASL_ID_MAXNUM_MCDI2 32 + + +/***********************************/ +/* MC_CMD_MAE_OUTER_RULE_INSERT + * Inserts an Outer Rule, which controls encapsulation parsing, and may + * influence the Lookup Sequence. If the maximum number of rules have already + * been inserted then the command will fail with MC_CMD_ERR_ENOSPC. + */ +#define MC_CMD_MAE_OUTER_RULE_INSERT 0x15a +#undef MC_CMD_0x15a_PRIVILEGE_CTG + +#define MC_CMD_0x15a_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_OUTER_RULE_INSERT_IN msgrequest */ +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_LENMIN 16 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_LENMAX 252 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_LENMAX_MCDI2 1020 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_LEN(num) (16+1*(num)) +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_FIELD_MATCH_CRITERIA_NUM(len) (((len)-16)/1) +/* Packets matching the rule will be parsed with this encapsulation. */ +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_ENCAP_TYPE_OFST 0 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_ENCAP_TYPE_LEN 4 +/* Enum values, see field(s): */ +/* MAE_MCDI_ENCAP_TYPE */ +/* Match priority. Lower values have higher priority. Must be less than + * MC_CMD_MAE_GET_CAPS_OUT.ENCAP_PRIOS If a packet matches two filters with + * equal priority then it is unspecified which takes priority. + */ +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_PRIO_OFST 4 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_PRIO_LEN 4 +/* Deprecated alias for ACTION_CONTROL. */ +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_LOOKUP_CONTROL_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_LOOKUP_CONTROL_LEN 4 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_DO_CT_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_DO_CT_LBN 0 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_DO_CT_WIDTH 1 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_CT_VNI_MODE_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_CT_VNI_MODE_LBN 1 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_CT_VNI_MODE_WIDTH 2 +/* Enum values, see field(s): */ +/* MAE_CT_VNI_MODE */ +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_DO_COUNT_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_DO_COUNT_LBN 3 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_DO_COUNT_WIDTH 1 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_CT_TCP_FLAGS_INHIBIT_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_CT_TCP_FLAGS_INHIBIT_LBN 4 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_CT_TCP_FLAGS_INHIBIT_WIDTH 1 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_RECIRC_ID_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_RECIRC_ID_LBN 8 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_RECIRC_ID_WIDTH 8 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_CT_DOMAIN_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_CT_DOMAIN_LBN 16 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_CT_DOMAIN_WIDTH 16 +/* This field controls the actions that are performed when a rule is hit. */ +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_ACTION_CONTROL_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_ACTION_CONTROL_LEN 4 +/* ID of counter to increment when the rule is hit. Only used if the DO_COUNT + * flag is set. The ID must have been allocated with COUNTER_TYPE=OR. + */ +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_COUNTER_ID_OFST 12 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_COUNTER_ID_LEN 4 +/* Structure of the format MAE_ENC_FIELD_PAIRS. */ +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_FIELD_MATCH_CRITERIA_OFST 16 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_FIELD_MATCH_CRITERIA_LEN 1 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_FIELD_MATCH_CRITERIA_MINNUM 0 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_FIELD_MATCH_CRITERIA_MAXNUM 236 +#define MC_CMD_MAE_OUTER_RULE_INSERT_IN_FIELD_MATCH_CRITERIA_MAXNUM_MCDI2 1004 + +/* MC_CMD_MAE_OUTER_RULE_INSERT_OUT msgresponse */ +#define MC_CMD_MAE_OUTER_RULE_INSERT_OUT_LEN 4 +#define MC_CMD_MAE_OUTER_RULE_INSERT_OUT_OR_ID_OFST 0 +#define MC_CMD_MAE_OUTER_RULE_INSERT_OUT_OR_ID_LEN 4 +/* enum: An outer match ID that is guaranteed never to represent an outer match + */ +#define MC_CMD_MAE_OUTER_RULE_INSERT_OUT_OUTER_RULE_ID_NULL 0xffffffff + + +/***********************************/ +/* MC_CMD_MAE_OUTER_RULE_REMOVE + */ +#define MC_CMD_MAE_OUTER_RULE_REMOVE 0x15b +#undef MC_CMD_0x15b_PRIVILEGE_CTG + +#define MC_CMD_0x15b_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_OUTER_RULE_REMOVE_IN msgrequest */ +#define MC_CMD_MAE_OUTER_RULE_REMOVE_IN_LENMIN 4 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_IN_LENMAX 128 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_IN_LENMAX_MCDI2 128 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_IN_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_OUTER_RULE_REMOVE_IN_OR_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_OUTER_RULE_REMOVE_IN_OR_ID_OFST 0 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_IN_OR_ID_LEN 4 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_IN_OR_ID_MINNUM 1 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_IN_OR_ID_MAXNUM 32 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_IN_OR_ID_MAXNUM_MCDI2 32 + +/* MC_CMD_MAE_OUTER_RULE_REMOVE_OUT msgresponse */ +#define MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_LENMIN 4 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_LENMAX 128 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_LENMAX_MCDI2 128 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_REMOVED_OR_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_REMOVED_OR_ID_OFST 0 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_REMOVED_OR_ID_LEN 4 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_REMOVED_OR_ID_MINNUM 1 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_REMOVED_OR_ID_MAXNUM 32 +#define MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_REMOVED_OR_ID_MAXNUM_MCDI2 32 + + +/***********************************/ +/* MC_CMD_MAE_OUTER_RULE_UPDATE + * Atomically change the response of an Outer Rule. + */ +#define MC_CMD_MAE_OUTER_RULE_UPDATE 0x17d +#undef MC_CMD_0x17d_PRIVILEGE_CTG + +#define MC_CMD_0x17d_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_OUTER_RULE_UPDATE_IN msgrequest */ +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_LEN 16 +/* ID of outer rule to update */ +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_OR_ID_OFST 0 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_OR_ID_LEN 4 +/* Packets matching the rule will be parsed with this encapsulation. */ +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_ENCAP_TYPE_OFST 4 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_ENCAP_TYPE_LEN 4 +/* Enum values, see field(s): */ +/* MAE_MCDI_ENCAP_TYPE */ +/* This field controls the actions that are performed when a rule is hit. */ +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_ACTION_CONTROL_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_ACTION_CONTROL_LEN 4 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_DO_CT_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_DO_CT_LBN 0 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_DO_CT_WIDTH 1 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_CT_VNI_MODE_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_CT_VNI_MODE_LBN 1 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_CT_VNI_MODE_WIDTH 2 +/* Enum values, see field(s): */ +/* MAE_CT_VNI_MODE */ +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_DO_COUNT_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_DO_COUNT_LBN 3 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_DO_COUNT_WIDTH 1 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_CT_TCP_FLAGS_INHIBIT_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_CT_TCP_FLAGS_INHIBIT_LBN 4 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_CT_TCP_FLAGS_INHIBIT_WIDTH 1 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_RECIRC_ID_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_RECIRC_ID_LBN 8 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_RECIRC_ID_WIDTH 8 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_CT_DOMAIN_OFST 8 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_CT_DOMAIN_LBN 16 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_CT_DOMAIN_WIDTH 16 +/* ID of counter to increment when the rule is hit. Only used if the DO_COUNT + * flag is set. The ID must have been allocated with COUNTER_TYPE=OR. + */ +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_COUNTER_ID_OFST 12 +#define MC_CMD_MAE_OUTER_RULE_UPDATE_IN_COUNTER_ID_LEN 4 + +/* MC_CMD_MAE_OUTER_RULE_UPDATE_OUT msgresponse */ +#define MC_CMD_MAE_OUTER_RULE_UPDATE_OUT_LEN 0 + +/* MAE_ACTION_RULE_RESPONSE structuredef */ +#define MAE_ACTION_RULE_RESPONSE_LEN 16 +#define MAE_ACTION_RULE_RESPONSE_ASL_ID_OFST 0 +#define MAE_ACTION_RULE_RESPONSE_ASL_ID_LEN 4 +#define MAE_ACTION_RULE_RESPONSE_ASL_ID_LBN 0 +#define MAE_ACTION_RULE_RESPONSE_ASL_ID_WIDTH 32 +/* Only one of ASL_ID or AS_ID may have a non-NULL value. */ +#define MAE_ACTION_RULE_RESPONSE_AS_ID_OFST 4 +#define MAE_ACTION_RULE_RESPONSE_AS_ID_LEN 4 +#define MAE_ACTION_RULE_RESPONSE_AS_ID_LBN 32 +#define MAE_ACTION_RULE_RESPONSE_AS_ID_WIDTH 32 +/* Controls lookup flow when this rule is hit. See sub-fields for details. More + * info on the lookup sequence can be found in SF-122976-TC. It is an error to + * set both DO_CT and DO_RECIRC. + */ +#define MAE_ACTION_RULE_RESPONSE_LOOKUP_CONTROL_OFST 8 +#define MAE_ACTION_RULE_RESPONSE_LOOKUP_CONTROL_LEN 4 +#define MAE_ACTION_RULE_RESPONSE_DO_CT_OFST 8 +#define MAE_ACTION_RULE_RESPONSE_DO_CT_LBN 0 +#define MAE_ACTION_RULE_RESPONSE_DO_CT_WIDTH 1 +#define MAE_ACTION_RULE_RESPONSE_DO_RECIRC_OFST 8 +#define MAE_ACTION_RULE_RESPONSE_DO_RECIRC_LBN 1 +#define MAE_ACTION_RULE_RESPONSE_DO_RECIRC_WIDTH 1 +#define MAE_ACTION_RULE_RESPONSE_CT_VNI_MODE_OFST 8 +#define MAE_ACTION_RULE_RESPONSE_CT_VNI_MODE_LBN 2 +#define MAE_ACTION_RULE_RESPONSE_CT_VNI_MODE_WIDTH 2 +/* Enum values, see field(s): */ +/* MAE_CT_VNI_MODE */ +#define MAE_ACTION_RULE_RESPONSE_RECIRC_ID_OFST 8 +#define MAE_ACTION_RULE_RESPONSE_RECIRC_ID_LBN 8 +#define MAE_ACTION_RULE_RESPONSE_RECIRC_ID_WIDTH 8 +#define MAE_ACTION_RULE_RESPONSE_CT_DOMAIN_OFST 8 +#define MAE_ACTION_RULE_RESPONSE_CT_DOMAIN_LBN 16 +#define MAE_ACTION_RULE_RESPONSE_CT_DOMAIN_WIDTH 16 +#define MAE_ACTION_RULE_RESPONSE_LOOKUP_CONTROL_LBN 64 +#define MAE_ACTION_RULE_RESPONSE_LOOKUP_CONTROL_WIDTH 32 +/* Counter ID to increment if DO_CT or DO_RECIRC is set. Must be set to + * COUNTER_ID_NULL otherwise. Counter ID must have been allocated with + * COUNTER_TYPE=AR. + */ +#define MAE_ACTION_RULE_RESPONSE_COUNTER_ID_OFST 12 +#define MAE_ACTION_RULE_RESPONSE_COUNTER_ID_LEN 4 +#define MAE_ACTION_RULE_RESPONSE_COUNTER_ID_LBN 96 +#define MAE_ACTION_RULE_RESPONSE_COUNTER_ID_WIDTH 32 + + +/***********************************/ +/* MC_CMD_MAE_ACTION_RULE_INSERT + * Insert a rule specify that packets matching a filter be processed according + * to a previous allocated action. Masks can be set as indicated by + * MC_CMD_MAE_GET_MATCH_FIELD_CAPABILITIES. If the maximum number of rules have + * already been inserted then the command will fail with MC_CMD_ERR_ENOSPC. + */ +#define MC_CMD_MAE_ACTION_RULE_INSERT 0x15c +#undef MC_CMD_0x15c_PRIVILEGE_CTG + +#define MC_CMD_0x15c_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_ACTION_RULE_INSERT_IN msgrequest */ +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_LENMIN 28 +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_LENMAX 252 +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_LENMAX_MCDI2 1020 +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_LEN(num) (28+1*(num)) +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_MATCH_CRITERIA_NUM(len) (((len)-28)/1) +/* See MC_CMD_MAE_OUTER_RULE_REGISTER_IN/PRIO. */ +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_PRIO_OFST 0 +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_PRIO_LEN 4 +/* Structure of the format MAE_ACTION_RULE_RESPONSE */ +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_RESPONSE_OFST 4 +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_RESPONSE_LEN 20 +/* Reserved for future use. Must be set to zero. */ +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_RSVD_OFST 24 +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_RSVD_LEN 4 +/* Structure of the format MAE_FIELD_MASK_VALUE_PAIRS */ +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_MATCH_CRITERIA_OFST 28 +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_MATCH_CRITERIA_LEN 1 +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_MATCH_CRITERIA_MINNUM 0 +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_MATCH_CRITERIA_MAXNUM 224 +#define MC_CMD_MAE_ACTION_RULE_INSERT_IN_MATCH_CRITERIA_MAXNUM_MCDI2 992 + +/* MC_CMD_MAE_ACTION_RULE_INSERT_OUT msgresponse */ +#define MC_CMD_MAE_ACTION_RULE_INSERT_OUT_LEN 4 +#define MC_CMD_MAE_ACTION_RULE_INSERT_OUT_AR_ID_OFST 0 +#define MC_CMD_MAE_ACTION_RULE_INSERT_OUT_AR_ID_LEN 4 +/* enum: An action rule ID that is guaranteed never to represent an action rule + */ +#define MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL 0xffffffff + + +/***********************************/ +/* MC_CMD_MAE_ACTION_RULE_UPDATE + * Atomically change the response of an action rule. Firmware may return + * ENOTSUP, in which case the driver should DELETE/INSERT. + */ +#define MC_CMD_MAE_ACTION_RULE_UPDATE 0x15d +#undef MC_CMD_0x15d_PRIVILEGE_CTG + +#define MC_CMD_0x15d_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_ACTION_RULE_UPDATE_IN msgrequest */ +#define MC_CMD_MAE_ACTION_RULE_UPDATE_IN_LEN 24 +/* ID of action rule to update */ +#define MC_CMD_MAE_ACTION_RULE_UPDATE_IN_AR_ID_OFST 0 +#define MC_CMD_MAE_ACTION_RULE_UPDATE_IN_AR_ID_LEN 4 +/* Structure of the format MAE_ACTION_RULE_RESPONSE */ +#define MC_CMD_MAE_ACTION_RULE_UPDATE_IN_RESPONSE_OFST 4 +#define MC_CMD_MAE_ACTION_RULE_UPDATE_IN_RESPONSE_LEN 20 + +/* MC_CMD_MAE_ACTION_RULE_UPDATE_OUT msgresponse */ +#define MC_CMD_MAE_ACTION_RULE_UPDATE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_MAE_ACTION_RULE_DELETE + */ +#define MC_CMD_MAE_ACTION_RULE_DELETE 0x155 +#undef MC_CMD_0x155_PRIVILEGE_CTG + +#define MC_CMD_0x155_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_ACTION_RULE_DELETE_IN msgrequest */ +#define MC_CMD_MAE_ACTION_RULE_DELETE_IN_LENMIN 4 +#define MC_CMD_MAE_ACTION_RULE_DELETE_IN_LENMAX 128 +#define MC_CMD_MAE_ACTION_RULE_DELETE_IN_LENMAX_MCDI2 128 +#define MC_CMD_MAE_ACTION_RULE_DELETE_IN_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_ACTION_RULE_DELETE_IN_AR_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_ACTION_RULE_DELETE_IN_AR_ID_OFST 0 +#define MC_CMD_MAE_ACTION_RULE_DELETE_IN_AR_ID_LEN 4 +#define MC_CMD_MAE_ACTION_RULE_DELETE_IN_AR_ID_MINNUM 1 +#define MC_CMD_MAE_ACTION_RULE_DELETE_IN_AR_ID_MAXNUM 32 +#define MC_CMD_MAE_ACTION_RULE_DELETE_IN_AR_ID_MAXNUM_MCDI2 32 + +/* MC_CMD_MAE_ACTION_RULE_DELETE_OUT msgresponse */ +#define MC_CMD_MAE_ACTION_RULE_DELETE_OUT_LENMIN 4 +#define MC_CMD_MAE_ACTION_RULE_DELETE_OUT_LENMAX 128 +#define MC_CMD_MAE_ACTION_RULE_DELETE_OUT_LENMAX_MCDI2 128 +#define MC_CMD_MAE_ACTION_RULE_DELETE_OUT_LEN(num) (0+4*(num)) +#define MC_CMD_MAE_ACTION_RULE_DELETE_OUT_DELETED_AR_ID_NUM(len) (((len)-0)/4) +/* Same semantics as MC_CMD_MAE_COUNTER_FREE */ +#define MC_CMD_MAE_ACTION_RULE_DELETE_OUT_DELETED_AR_ID_OFST 0 +#define MC_CMD_MAE_ACTION_RULE_DELETE_OUT_DELETED_AR_ID_LEN 4 +#define MC_CMD_MAE_ACTION_RULE_DELETE_OUT_DELETED_AR_ID_MINNUM 1 +#define MC_CMD_MAE_ACTION_RULE_DELETE_OUT_DELETED_AR_ID_MAXNUM 32 +#define MC_CMD_MAE_ACTION_RULE_DELETE_OUT_DELETED_AR_ID_MAXNUM_MCDI2 32 + + +/***********************************/ +/* MC_CMD_MAE_MPORT_LOOKUP + * Return the m-port corresponding to a selector. + */ +#define MC_CMD_MAE_MPORT_LOOKUP 0x160 +#undef MC_CMD_0x160_PRIVILEGE_CTG + +#define MC_CMD_0x160_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_MAE_MPORT_LOOKUP_IN msgrequest */ +#define MC_CMD_MAE_MPORT_LOOKUP_IN_LEN 4 +#define MC_CMD_MAE_MPORT_LOOKUP_IN_MPORT_SELECTOR_OFST 0 +#define MC_CMD_MAE_MPORT_LOOKUP_IN_MPORT_SELECTOR_LEN 4 + +/* MC_CMD_MAE_MPORT_LOOKUP_OUT msgresponse */ +#define MC_CMD_MAE_MPORT_LOOKUP_OUT_LEN 4 +#define MC_CMD_MAE_MPORT_LOOKUP_OUT_MPORT_ID_OFST 0 +#define MC_CMD_MAE_MPORT_LOOKUP_OUT_MPORT_ID_LEN 4 + + +/***********************************/ +/* MC_CMD_MAE_MPORT_ALLOC + * Allocates a m-port, which can subsequently be used in action rules as a + * match or delivery argument. + */ +#define MC_CMD_MAE_MPORT_ALLOC 0x163 +#undef MC_CMD_0x163_PRIVILEGE_CTG + +#define MC_CMD_0x163_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_MPORT_ALLOC_IN msgrequest */ +#define MC_CMD_MAE_MPORT_ALLOC_IN_LEN 20 +/* The type of m-port to allocate. Firmware may return ENOTSUP for certain + * types. + */ +#define MC_CMD_MAE_MPORT_ALLOC_IN_TYPE_OFST 0 +#define MC_CMD_MAE_MPORT_ALLOC_IN_TYPE_LEN 4 +/* enum: Traffic can be sent to this type of m-port using an override + * descriptor. Traffic received on this type of m-port will go to the VNIC on a + * nominated m-port, and will be delivered with metadata identifying the alias + * m-port. + */ +#define MC_CMD_MAE_MPORT_ALLOC_IN_MPORT_TYPE_ALIAS 0x1 +/* enum: This type of m-port has a VNIC attached. Queues can be created on this + * VNIC by specifying the created m-port as an m-port selector at queue + * creation time. + */ +#define MC_CMD_MAE_MPORT_ALLOC_IN_MPORT_TYPE_VNIC 0x2 +/* 128-bit value for use by the driver. */ +#define MC_CMD_MAE_MPORT_ALLOC_IN_UUID_OFST 4 +#define MC_CMD_MAE_MPORT_ALLOC_IN_UUID_LEN 16 + +/* MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN msgrequest */ +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_LEN 24 +/* The type of m-port to allocate. Firmware may return ENOTSUP for certain + * types. + */ +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_TYPE_OFST 0 +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_TYPE_LEN 4 +/* enum: Traffic can be sent to this type of m-port using an override + * descriptor. Traffic received on this type of m-port will go to the VNIC on a + * nominated m-port, and will be delivered with metadata identifying the alias + * m-port. + */ +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_MPORT_TYPE_ALIAS 0x1 +/* enum: This type of m-port has a VNIC attached. Queues can be created on this + * VNIC by specifying the created m-port as an m-port selector at queue + * creation time. + */ +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_MPORT_TYPE_VNIC 0x2 +/* 128-bit value for use by the driver. */ +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_UUID_OFST 4 +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_UUID_LEN 16 +/* An m-port selector identifying the VNIC to which traffic should be + * delivered. This must currently be set to MAE_MPORT_SELECTOR_ASSIGNED (i.e. + * the m-port assigned to the calling client). + */ +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_DELIVER_MPORT_OFST 20 +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_IN_DELIVER_MPORT_LEN 4 + +/* MC_CMD_MAE_MPORT_ALLOC_VNIC_IN msgrequest */ +#define MC_CMD_MAE_MPORT_ALLOC_VNIC_IN_LEN 20 +/* The type of m-port to allocate. Firmware may return ENOTSUP for certain + * types. + */ +#define MC_CMD_MAE_MPORT_ALLOC_VNIC_IN_TYPE_OFST 0 +#define MC_CMD_MAE_MPORT_ALLOC_VNIC_IN_TYPE_LEN 4 +/* enum: Traffic can be sent to this type of m-port using an override + * descriptor. Traffic received on this type of m-port will go to the VNIC on a + * nominated m-port, and will be delivered with metadata identifying the alias + * m-port. + */ +#define MC_CMD_MAE_MPORT_ALLOC_VNIC_IN_MPORT_TYPE_ALIAS 0x1 +/* enum: This type of m-port has a VNIC attached. Queues can be created on this + * VNIC by specifying the created m-port as an m-port selector at queue + * creation time. + */ +#define MC_CMD_MAE_MPORT_ALLOC_VNIC_IN_MPORT_TYPE_VNIC 0x2 +/* 128-bit value for use by the driver. */ +#define MC_CMD_MAE_MPORT_ALLOC_VNIC_IN_UUID_OFST 4 +#define MC_CMD_MAE_MPORT_ALLOC_VNIC_IN_UUID_LEN 16 + +/* MC_CMD_MAE_MPORT_ALLOC_OUT msgresponse */ +#define MC_CMD_MAE_MPORT_ALLOC_OUT_LEN 4 +/* ID of newly-allocated m-port. */ +#define MC_CMD_MAE_MPORT_ALLOC_OUT_MPORT_ID_OFST 0 +#define MC_CMD_MAE_MPORT_ALLOC_OUT_MPORT_ID_LEN 4 + +/* MC_CMD_MAE_MPORT_ALLOC_ALIAS_OUT msgrequest */ +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_OUT_LEN 24 +/* ID of newly-allocated m-port. */ +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_OUT_MPORT_ID_OFST 0 +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_OUT_MPORT_ID_LEN 4 +/* A value that will appear in the packet metadata for any packets delivered + * using an alias type m-port. This value is guaranteed unique on the VNIC + * being delivered to, and is guaranteed not to exceed the range of values + * representable in the relevant metadata field. + */ +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_OUT_LABEL_OFST 20 +#define MC_CMD_MAE_MPORT_ALLOC_ALIAS_OUT_LABEL_LEN 4 + +/* MC_CMD_MAE_MPORT_ALLOC_VNIC_OUT msgrequest */ +#define MC_CMD_MAE_MPORT_ALLOC_VNIC_OUT_LEN 4 +/* ID of newly-allocated m-port. */ +#define MC_CMD_MAE_MPORT_ALLOC_VNIC_OUT_MPORT_ID_OFST 0 +#define MC_CMD_MAE_MPORT_ALLOC_VNIC_OUT_MPORT_ID_LEN 4 + + +/***********************************/ +/* MC_CMD_MAE_MPORT_FREE + * Free a m-port which was previously allocated by the driver. + */ +#define MC_CMD_MAE_MPORT_FREE 0x164 +#undef MC_CMD_0x164_PRIVILEGE_CTG + +#define MC_CMD_0x164_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_MPORT_FREE_IN msgrequest */ +#define MC_CMD_MAE_MPORT_FREE_IN_LEN 4 +/* MPORT_ID as returned by MC_CMD_MAE_MPORT_ALLOC. */ +#define MC_CMD_MAE_MPORT_FREE_IN_MPORT_ID_OFST 0 +#define MC_CMD_MAE_MPORT_FREE_IN_MPORT_ID_LEN 4 + +/* MC_CMD_MAE_MPORT_FREE_OUT msgresponse */ +#define MC_CMD_MAE_MPORT_FREE_OUT_LEN 0 + +/* MAE_MPORT_DESC structuredef */ +#define MAE_MPORT_DESC_LEN 52 +#define MAE_MPORT_DESC_MPORT_ID_OFST 0 +#define MAE_MPORT_DESC_MPORT_ID_LEN 4 +#define MAE_MPORT_DESC_MPORT_ID_LBN 0 +#define MAE_MPORT_DESC_MPORT_ID_WIDTH 32 +/* Reserved for future purposes, contains information independent of caller */ +#define MAE_MPORT_DESC_FLAGS_OFST 4 +#define MAE_MPORT_DESC_FLAGS_LEN 4 +#define MAE_MPORT_DESC_FLAGS_LBN 32 +#define MAE_MPORT_DESC_FLAGS_WIDTH 32 +#define MAE_MPORT_DESC_CALLER_FLAGS_OFST 8 +#define MAE_MPORT_DESC_CALLER_FLAGS_LEN 4 +#define MAE_MPORT_DESC_CAN_RECEIVE_ON_OFST 8 +#define MAE_MPORT_DESC_CAN_RECEIVE_ON_LBN 0 +#define MAE_MPORT_DESC_CAN_RECEIVE_ON_WIDTH 1 +#define MAE_MPORT_DESC_CAN_DELIVER_TO_OFST 8 +#define MAE_MPORT_DESC_CAN_DELIVER_TO_LBN 1 +#define MAE_MPORT_DESC_CAN_DELIVER_TO_WIDTH 1 +#define MAE_MPORT_DESC_CAN_DELETE_OFST 8 +#define MAE_MPORT_DESC_CAN_DELETE_LBN 2 +#define MAE_MPORT_DESC_CAN_DELETE_WIDTH 1 +#define MAE_MPORT_DESC_IS_ZOMBIE_OFST 8 +#define MAE_MPORT_DESC_IS_ZOMBIE_LBN 3 +#define MAE_MPORT_DESC_IS_ZOMBIE_WIDTH 1 +#define MAE_MPORT_DESC_CALLER_FLAGS_LBN 64 +#define MAE_MPORT_DESC_CALLER_FLAGS_WIDTH 32 +/* Not the ideal name; it's really the type of thing connected to the m-port */ +#define MAE_MPORT_DESC_MPORT_TYPE_OFST 12 +#define MAE_MPORT_DESC_MPORT_TYPE_LEN 4 +/* enum: Connected to a MAC... */ +#define MAE_MPORT_DESC_MPORT_TYPE_NET_PORT 0x0 +/* enum: Adds metadata and delivers to another m-port */ +#define MAE_MPORT_DESC_MPORT_TYPE_ALIAS 0x1 +/* enum: Connected to a VNIC. */ +#define MAE_MPORT_DESC_MPORT_TYPE_VNIC 0x2 +#define MAE_MPORT_DESC_MPORT_TYPE_LBN 96 +#define MAE_MPORT_DESC_MPORT_TYPE_WIDTH 32 +/* 128-bit value available to drivers for m-port identification. */ +#define MAE_MPORT_DESC_UUID_OFST 16 +#define MAE_MPORT_DESC_UUID_LEN 16 +#define MAE_MPORT_DESC_UUID_LBN 128 +#define MAE_MPORT_DESC_UUID_WIDTH 128 +/* Big wadge of space reserved for other common properties */ +#define MAE_MPORT_DESC_RESERVED_OFST 32 +#define MAE_MPORT_DESC_RESERVED_LEN 8 +#define MAE_MPORT_DESC_RESERVED_LO_OFST 32 +#define MAE_MPORT_DESC_RESERVED_LO_LEN 4 +#define MAE_MPORT_DESC_RESERVED_LO_LBN 256 +#define MAE_MPORT_DESC_RESERVED_LO_WIDTH 32 +#define MAE_MPORT_DESC_RESERVED_HI_OFST 36 +#define MAE_MPORT_DESC_RESERVED_HI_LEN 4 +#define MAE_MPORT_DESC_RESERVED_HI_LBN 288 +#define MAE_MPORT_DESC_RESERVED_HI_WIDTH 32 +#define MAE_MPORT_DESC_RESERVED_LBN 256 +#define MAE_MPORT_DESC_RESERVED_WIDTH 64 +/* Logical port index. Only valid when type NET Port. */ +#define MAE_MPORT_DESC_NET_PORT_IDX_OFST 40 +#define MAE_MPORT_DESC_NET_PORT_IDX_LEN 4 +#define MAE_MPORT_DESC_NET_PORT_IDX_LBN 320 +#define MAE_MPORT_DESC_NET_PORT_IDX_WIDTH 32 +/* The m-port delivered to */ +#define MAE_MPORT_DESC_ALIAS_DELIVER_MPORT_ID_OFST 40 +#define MAE_MPORT_DESC_ALIAS_DELIVER_MPORT_ID_LEN 4 +#define MAE_MPORT_DESC_ALIAS_DELIVER_MPORT_ID_LBN 320 +#define MAE_MPORT_DESC_ALIAS_DELIVER_MPORT_ID_WIDTH 32 +/* The type of thing that owns the VNIC */ +#define MAE_MPORT_DESC_VNIC_CLIENT_TYPE_OFST 40 +#define MAE_MPORT_DESC_VNIC_CLIENT_TYPE_LEN 4 +#define MAE_MPORT_DESC_VNIC_CLIENT_TYPE_FUNCTION 0x1 /* enum */ +#define MAE_MPORT_DESC_VNIC_CLIENT_TYPE_PLUGIN 0x2 /* enum */ +#define MAE_MPORT_DESC_VNIC_CLIENT_TYPE_LBN 320 +#define MAE_MPORT_DESC_VNIC_CLIENT_TYPE_WIDTH 32 +/* The PCIe interface on which the function lives. CJK: We need an enumeration + * of interfaces that we extend as new interface (types) appear. This belongs + * elsewhere and should be referenced from here + */ +#define MAE_MPORT_DESC_VNIC_FUNCTION_INTERFACE_OFST 44 +#define MAE_MPORT_DESC_VNIC_FUNCTION_INTERFACE_LEN 4 +#define MAE_MPORT_DESC_VNIC_FUNCTION_INTERFACE_LBN 352 +#define MAE_MPORT_DESC_VNIC_FUNCTION_INTERFACE_WIDTH 32 +#define MAE_MPORT_DESC_VNIC_FUNCTION_PF_IDX_OFST 48 +#define MAE_MPORT_DESC_VNIC_FUNCTION_PF_IDX_LEN 2 +#define MAE_MPORT_DESC_VNIC_FUNCTION_PF_IDX_LBN 384 +#define MAE_MPORT_DESC_VNIC_FUNCTION_PF_IDX_WIDTH 16 +#define MAE_MPORT_DESC_VNIC_FUNCTION_VF_IDX_OFST 50 +#define MAE_MPORT_DESC_VNIC_FUNCTION_VF_IDX_LEN 2 +/* enum: Indicates that the function is a PF */ +#define MAE_MPORT_DESC_VF_IDX_NULL 0xffff +#define MAE_MPORT_DESC_VNIC_FUNCTION_VF_IDX_LBN 400 +#define MAE_MPORT_DESC_VNIC_FUNCTION_VF_IDX_WIDTH 16 +/* Reserved. Should be ignored for now. */ +#define MAE_MPORT_DESC_VNIC_PLUGIN_TBD_OFST 44 +#define MAE_MPORT_DESC_VNIC_PLUGIN_TBD_LEN 4 +#define MAE_MPORT_DESC_VNIC_PLUGIN_TBD_LBN 352 +#define MAE_MPORT_DESC_VNIC_PLUGIN_TBD_WIDTH 32 + +/* MAE_MPORT_DESC_V2 structuredef */ +#define MAE_MPORT_DESC_V2_LEN 56 +#define MAE_MPORT_DESC_V2_MPORT_ID_OFST 0 +#define MAE_MPORT_DESC_V2_MPORT_ID_LEN 4 +#define MAE_MPORT_DESC_V2_MPORT_ID_LBN 0 +#define MAE_MPORT_DESC_V2_MPORT_ID_WIDTH 32 +/* Reserved for future purposes, contains information independent of caller */ +#define MAE_MPORT_DESC_V2_FLAGS_OFST 4 +#define MAE_MPORT_DESC_V2_FLAGS_LEN 4 +#define MAE_MPORT_DESC_V2_FLAGS_LBN 32 +#define MAE_MPORT_DESC_V2_FLAGS_WIDTH 32 +#define MAE_MPORT_DESC_V2_CALLER_FLAGS_OFST 8 +#define MAE_MPORT_DESC_V2_CALLER_FLAGS_LEN 4 +#define MAE_MPORT_DESC_V2_CAN_RECEIVE_ON_OFST 8 +#define MAE_MPORT_DESC_V2_CAN_RECEIVE_ON_LBN 0 +#define MAE_MPORT_DESC_V2_CAN_RECEIVE_ON_WIDTH 1 +#define MAE_MPORT_DESC_V2_CAN_DELIVER_TO_OFST 8 +#define MAE_MPORT_DESC_V2_CAN_DELIVER_TO_LBN 1 +#define MAE_MPORT_DESC_V2_CAN_DELIVER_TO_WIDTH 1 +#define MAE_MPORT_DESC_V2_CAN_DELETE_OFST 8 +#define MAE_MPORT_DESC_V2_CAN_DELETE_LBN 2 +#define MAE_MPORT_DESC_V2_CAN_DELETE_WIDTH 1 +#define MAE_MPORT_DESC_V2_IS_ZOMBIE_OFST 8 +#define MAE_MPORT_DESC_V2_IS_ZOMBIE_LBN 3 +#define MAE_MPORT_DESC_V2_IS_ZOMBIE_WIDTH 1 +#define MAE_MPORT_DESC_V2_CALLER_FLAGS_LBN 64 +#define MAE_MPORT_DESC_V2_CALLER_FLAGS_WIDTH 32 +/* Not the ideal name; it's really the type of thing connected to the m-port */ +#define MAE_MPORT_DESC_V2_MPORT_TYPE_OFST 12 +#define MAE_MPORT_DESC_V2_MPORT_TYPE_LEN 4 +/* enum: Connected to a MAC... */ +#define MAE_MPORT_DESC_V2_MPORT_TYPE_NET_PORT 0x0 +/* enum: Adds metadata and delivers to another m-port */ +#define MAE_MPORT_DESC_V2_MPORT_TYPE_ALIAS 0x1 +/* enum: Connected to a VNIC. */ +#define MAE_MPORT_DESC_V2_MPORT_TYPE_VNIC 0x2 +#define MAE_MPORT_DESC_V2_MPORT_TYPE_LBN 96 +#define MAE_MPORT_DESC_V2_MPORT_TYPE_WIDTH 32 +/* 128-bit value available to drivers for m-port identification. */ +#define MAE_MPORT_DESC_V2_UUID_OFST 16 +#define MAE_MPORT_DESC_V2_UUID_LEN 16 +#define MAE_MPORT_DESC_V2_UUID_LBN 128 +#define MAE_MPORT_DESC_V2_UUID_WIDTH 128 +/* Big wadge of space reserved for other common properties */ +#define MAE_MPORT_DESC_V2_RESERVED_OFST 32 +#define MAE_MPORT_DESC_V2_RESERVED_LEN 8 +#define MAE_MPORT_DESC_V2_RESERVED_LO_OFST 32 +#define MAE_MPORT_DESC_V2_RESERVED_LO_LEN 4 +#define MAE_MPORT_DESC_V2_RESERVED_LO_LBN 256 +#define MAE_MPORT_DESC_V2_RESERVED_LO_WIDTH 32 +#define MAE_MPORT_DESC_V2_RESERVED_HI_OFST 36 +#define MAE_MPORT_DESC_V2_RESERVED_HI_LEN 4 +#define MAE_MPORT_DESC_V2_RESERVED_HI_LBN 288 +#define MAE_MPORT_DESC_V2_RESERVED_HI_WIDTH 32 +#define MAE_MPORT_DESC_V2_RESERVED_LBN 256 +#define MAE_MPORT_DESC_V2_RESERVED_WIDTH 64 +/* Logical port index. Only valid when type NET Port. */ +#define MAE_MPORT_DESC_V2_NET_PORT_IDX_OFST 40 +#define MAE_MPORT_DESC_V2_NET_PORT_IDX_LEN 4 +#define MAE_MPORT_DESC_V2_NET_PORT_IDX_LBN 320 +#define MAE_MPORT_DESC_V2_NET_PORT_IDX_WIDTH 32 +/* The m-port delivered to */ +#define MAE_MPORT_DESC_V2_ALIAS_DELIVER_MPORT_ID_OFST 40 +#define MAE_MPORT_DESC_V2_ALIAS_DELIVER_MPORT_ID_LEN 4 +#define MAE_MPORT_DESC_V2_ALIAS_DELIVER_MPORT_ID_LBN 320 +#define MAE_MPORT_DESC_V2_ALIAS_DELIVER_MPORT_ID_WIDTH 32 +/* The type of thing that owns the VNIC */ +#define MAE_MPORT_DESC_V2_VNIC_CLIENT_TYPE_OFST 40 +#define MAE_MPORT_DESC_V2_VNIC_CLIENT_TYPE_LEN 4 +#define MAE_MPORT_DESC_V2_VNIC_CLIENT_TYPE_FUNCTION 0x1 /* enum */ +#define MAE_MPORT_DESC_V2_VNIC_CLIENT_TYPE_PLUGIN 0x2 /* enum */ +#define MAE_MPORT_DESC_V2_VNIC_CLIENT_TYPE_LBN 320 +#define MAE_MPORT_DESC_V2_VNIC_CLIENT_TYPE_WIDTH 32 +/* The PCIe interface on which the function lives. CJK: We need an enumeration + * of interfaces that we extend as new interface (types) appear. This belongs + * elsewhere and should be referenced from here + */ +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_INTERFACE_OFST 44 +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_INTERFACE_LEN 4 +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_INTERFACE_LBN 352 +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_INTERFACE_WIDTH 32 +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_PF_IDX_OFST 48 +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_PF_IDX_LEN 2 +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_PF_IDX_LBN 384 +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_PF_IDX_WIDTH 16 +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_VF_IDX_OFST 50 +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_VF_IDX_LEN 2 +/* enum: Indicates that the function is a PF */ +#define MAE_MPORT_DESC_V2_VF_IDX_NULL 0xffff +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_VF_IDX_LBN 400 +#define MAE_MPORT_DESC_V2_VNIC_FUNCTION_VF_IDX_WIDTH 16 +/* Reserved. Should be ignored for now. */ +#define MAE_MPORT_DESC_V2_VNIC_PLUGIN_TBD_OFST 44 +#define MAE_MPORT_DESC_V2_VNIC_PLUGIN_TBD_LEN 4 +#define MAE_MPORT_DESC_V2_VNIC_PLUGIN_TBD_LBN 352 +#define MAE_MPORT_DESC_V2_VNIC_PLUGIN_TBD_WIDTH 32 +/* A client handle for the VNIC's owner. Only valid for type VNIC. */ +#define MAE_MPORT_DESC_V2_VNIC_CLIENT_HANDLE_OFST 52 +#define MAE_MPORT_DESC_V2_VNIC_CLIENT_HANDLE_LEN 4 +#define MAE_MPORT_DESC_V2_VNIC_CLIENT_HANDLE_LBN 416 +#define MAE_MPORT_DESC_V2_VNIC_CLIENT_HANDLE_WIDTH 32 + + +/***********************************/ +/* MC_CMD_MAE_MPORT_ENUMERATE + * Deprecated in favour of MAE_MPORT_READ_JOURNAL. Support for this command + * will be removed at some future point. + */ +#define MC_CMD_MAE_MPORT_ENUMERATE 0x17c +#undef MC_CMD_0x17c_PRIVILEGE_CTG + +#define MC_CMD_0x17c_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_MAE_MPORT_ENUMERATE_IN msgrequest */ +#define MC_CMD_MAE_MPORT_ENUMERATE_IN_LEN 0 + +/* MC_CMD_MAE_MPORT_ENUMERATE_OUT msgresponse */ +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_LENMIN 8 +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_LENMAX 252 +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_LEN(num) (8+1*(num)) +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_MPORT_DESC_DATA_NUM(len) (((len)-8)/1) +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_MPORT_DESC_COUNT_OFST 0 +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_MPORT_DESC_COUNT_LEN 4 +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_SIZEOF_MPORT_DESC_OFST 4 +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_SIZEOF_MPORT_DESC_LEN 4 +/* Any array of MAE_MPORT_DESC structures. The MAE_MPORT_DESC structure may + * grow in future version of this command. Drivers should use a stride of + * SIZEOF_MPORT_DESC. Fields beyond SIZEOF_MPORT_DESC are not present. + */ +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_MPORT_DESC_DATA_OFST 8 +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_MPORT_DESC_DATA_LEN 1 +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_MPORT_DESC_DATA_MINNUM 0 +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_MPORT_DESC_DATA_MAXNUM 244 +#define MC_CMD_MAE_MPORT_ENUMERATE_OUT_MPORT_DESC_DATA_MAXNUM_MCDI2 1012 + + +/***********************************/ +/* MC_CMD_MAE_MPORT_READ_JOURNAL + * Firmware maintains a per-client journal of mport creations and deletions. + * This journal is clear-on-read, i.e. repeated calls of this command will + * drain the buffer. Whenever the caller resets its function via FLR or + * MC_CMD_ENTITY_RESET, the journal is regenerated from a blank start. + */ +#define MC_CMD_MAE_MPORT_READ_JOURNAL 0x147 +#undef MC_CMD_0x147_PRIVILEGE_CTG + +#define MC_CMD_0x147_PRIVILEGE_CTG SRIOV_CTG_MAE + +/* MC_CMD_MAE_MPORT_READ_JOURNAL_IN msgrequest */ +#define MC_CMD_MAE_MPORT_READ_JOURNAL_IN_LEN 4 +/* Any unused flags are reserved and must be set to zero. */ +#define MC_CMD_MAE_MPORT_READ_JOURNAL_IN_FLAGS_OFST 0 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_IN_FLAGS_LEN 4 + +/* MC_CMD_MAE_MPORT_READ_JOURNAL_OUT msgresponse */ +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_LENMIN 12 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_LENMAX 252 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_LEN(num) (12+1*(num)) +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MPORT_DESC_DATA_NUM(len) (((len)-12)/1) +/* Any unused flags are reserved and must be ignored. */ +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_FLAGS_OFST 0 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_FLAGS_LEN 4 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MORE_OFST 0 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MORE_LBN 0 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MORE_WIDTH 1 +/* The number of MAE_MPORT_DESC structures in MPORT_DESC_DATA. May be zero. */ +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MPORT_DESC_COUNT_OFST 4 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MPORT_DESC_COUNT_LEN 4 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_SIZEOF_MPORT_DESC_OFST 8 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_SIZEOF_MPORT_DESC_LEN 4 +/* Any array of MAE_MPORT_DESC structures. The MAE_MPORT_DESC structure may + * grow in future version of this command. Drivers should use a stride of + * SIZEOF_MPORT_DESC. Fields beyond SIZEOF_MPORT_DESC are not present. + */ +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MPORT_DESC_DATA_OFST 12 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MPORT_DESC_DATA_LEN 1 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MPORT_DESC_DATA_MINNUM 0 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MPORT_DESC_DATA_MAXNUM 240 +#define MC_CMD_MAE_MPORT_READ_JOURNAL_OUT_MPORT_DESC_DATA_MAXNUM_MCDI2 1008 + +/* TABLE_FIELD_DESCR structuredef: An individual table field descriptor. This + * describes the location and properties of one N-bit field within a wider + * M-bit key/mask/response value. + */ +#define TABLE_FIELD_DESCR_LEN 8 +/* Identifier for this field. */ +#define TABLE_FIELD_DESCR_FIELD_ID_OFST 0 +#define TABLE_FIELD_DESCR_FIELD_ID_LEN 2 +/* Enum values, see field(s): */ +/* TABLE_FIELD_ID */ +#define TABLE_FIELD_DESCR_FIELD_ID_LBN 0 +#define TABLE_FIELD_DESCR_FIELD_ID_WIDTH 16 +/* Lowest (least significant) bit number of the bits of this field. */ +#define TABLE_FIELD_DESCR_LBN_OFST 2 +#define TABLE_FIELD_DESCR_LBN_LEN 2 +#define TABLE_FIELD_DESCR_LBN_LBN 16 +#define TABLE_FIELD_DESCR_LBN_WIDTH 16 +/* Width of this field in bits. */ +#define TABLE_FIELD_DESCR_WIDTH_OFST 4 +#define TABLE_FIELD_DESCR_WIDTH_LEN 2 +#define TABLE_FIELD_DESCR_WIDTH_LBN 32 +#define TABLE_FIELD_DESCR_WIDTH_WIDTH 16 +/* The mask type for this field. (Note that masking is relevant to keys; fields + * of responses are always reported with the EXACT type.) + */ +#define TABLE_FIELD_DESCR_MASK_TYPE_OFST 6 +#define TABLE_FIELD_DESCR_MASK_TYPE_LEN 1 +/* enum: Field must never be selected in the mask. */ +#define TABLE_FIELD_DESCR_MASK_NEVER 0x0 +/* enum: Exact match: field must always be selected in the mask. */ +#define TABLE_FIELD_DESCR_MASK_EXACT 0x1 +/* enum: Ternary match: arbitrary mask bits are allowed. */ +#define TABLE_FIELD_DESCR_MASK_TERNARY 0x2 +/* enum: Whole field match: mask must be all 1 bits, or all 0 bits. */ +#define TABLE_FIELD_DESCR_MASK_WHOLE_FIELD 0x3 +/* enum: Longest prefix match: mask must be 1 bit(s) followed by 0 bit(s). */ +#define TABLE_FIELD_DESCR_MASK_LPM 0x4 +#define TABLE_FIELD_DESCR_MASK_TYPE_LBN 48 +#define TABLE_FIELD_DESCR_MASK_TYPE_WIDTH 8 +/* A version code that allows field semantics to be extended. All fields + * currently use version 0. + */ +#define TABLE_FIELD_DESCR_SCHEME_OFST 7 +#define TABLE_FIELD_DESCR_SCHEME_LEN 1 +#define TABLE_FIELD_DESCR_SCHEME_LBN 56 +#define TABLE_FIELD_DESCR_SCHEME_WIDTH 8 + + +/***********************************/ +/* MC_CMD_TABLE_LIST + * Return the list of tables which may be accessed via this table API. + */ +#define MC_CMD_TABLE_LIST 0x1c9 +#undef MC_CMD_0x1c9_PRIVILEGE_CTG + +#define MC_CMD_0x1c9_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TABLE_LIST_IN msgrequest */ +#define MC_CMD_TABLE_LIST_IN_LEN 4 +/* Index of the first item to be returned in the TABLE_ID sequence. (Set to 0 + * for the first call; further calls are only required if the whole sequence + * does not fit within the maximum MCDI message size.) + */ +#define MC_CMD_TABLE_LIST_IN_FIRST_TABLE_ID_INDEX_OFST 0 +#define MC_CMD_TABLE_LIST_IN_FIRST_TABLE_ID_INDEX_LEN 4 + +/* MC_CMD_TABLE_LIST_OUT msgresponse */ +#define MC_CMD_TABLE_LIST_OUT_LENMIN 4 +#define MC_CMD_TABLE_LIST_OUT_LENMAX 252 +#define MC_CMD_TABLE_LIST_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_TABLE_LIST_OUT_LEN(num) (4+4*(num)) +#define MC_CMD_TABLE_LIST_OUT_TABLE_ID_NUM(len) (((len)-4)/4) +/* The total number of tables. */ +#define MC_CMD_TABLE_LIST_OUT_N_TABLES_OFST 0 +#define MC_CMD_TABLE_LIST_OUT_N_TABLES_LEN 4 +/* A sequence of table identifiers. If all N_TABLES items do not fit, further + * items can be obtained by repeating the call with a non-zero + * FIRST_TABLE_ID_INDEX. + */ +#define MC_CMD_TABLE_LIST_OUT_TABLE_ID_OFST 4 +#define MC_CMD_TABLE_LIST_OUT_TABLE_ID_LEN 4 +#define MC_CMD_TABLE_LIST_OUT_TABLE_ID_MINNUM 0 +#define MC_CMD_TABLE_LIST_OUT_TABLE_ID_MAXNUM 62 +#define MC_CMD_TABLE_LIST_OUT_TABLE_ID_MAXNUM_MCDI2 254 +/* Enum values, see field(s): */ +/* TABLE_ID */ + + +/***********************************/ +/* MC_CMD_TABLE_DESCRIPTOR + * Request the table descriptor for a particular table. This describes + * properties of the table and the format of the key and response. May return + * EINVAL for unknown table ID. + */ +#define MC_CMD_TABLE_DESCRIPTOR 0x1ca +#undef MC_CMD_0x1ca_PRIVILEGE_CTG + +#define MC_CMD_0x1ca_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TABLE_DESCRIPTOR_IN msgrequest */ +#define MC_CMD_TABLE_DESCRIPTOR_IN_LEN 8 +/* Identifier for this field. */ +#define MC_CMD_TABLE_DESCRIPTOR_IN_TABLE_ID_OFST 0 +#define MC_CMD_TABLE_DESCRIPTOR_IN_TABLE_ID_LEN 4 +/* Enum values, see field(s): */ +/* TABLE_ID */ +/* Index of the first item to be returned in the FIELDS sequence. (Set to 0 for + * the first call; further calls are only required if the whole sequence does + * not fit within the maximum MCDI message size.) + */ +#define MC_CMD_TABLE_DESCRIPTOR_IN_FIRST_FIELDS_INDEX_OFST 4 +#define MC_CMD_TABLE_DESCRIPTOR_IN_FIRST_FIELDS_INDEX_LEN 4 + +/* MC_CMD_TABLE_DESCRIPTOR_OUT msgresponse */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_LENMIN 28 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_LENMAX 252 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_LENMAX_MCDI2 1020 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_LEN(num) (20+8*(num)) +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_NUM(len) (((len)-20)/8) +/* Maximum number of entries in this table. */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_MAX_ENTRIES_OFST 0 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_MAX_ENTRIES_LEN 4 +/* The type of table. (This is really just informational; the important + * properties of a table that affect programming can be deduced from other + * items in the table or field descriptor.) + */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_TYPE_OFST 4 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_TYPE_LEN 2 +/* enum: Direct table (essentially just an array). Behaves like a BCAM for + * programming purposes, where the fact that the key is actually used as an + * array index is really just an implementation detail. + */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_TYPE_DIRECT 0x1 +/* enum: BCAM (binary CAM) table: exact match on all key fields." */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_TYPE_BCAM 0x2 +/* enum: TCAM (ternary CAM) table: matches fields with a mask. Each entry may + * have its own different mask. + */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_TYPE_TCAM 0x3 +/* enum: STCAM (semi-TCAM) table: like a TCAM but entries shared a limited + * number of unique masks. + */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_TYPE_STCAM 0x4 +/* Width of key (and corresponding mask, for TCAM or STCAM) in bits. */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_KEY_WIDTH_OFST 6 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_KEY_WIDTH_LEN 2 +/* Width of response in bits. */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_RESP_WIDTH_OFST 8 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_RESP_WIDTH_LEN 2 +/* The total number of fields in the key. */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_N_KEY_FIELDS_OFST 10 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_N_KEY_FIELDS_LEN 2 +/* The total number of fields in the response. */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_N_RESP_FIELDS_OFST 12 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_N_RESP_FIELDS_LEN 2 +/* Number of priorities for STCAM or TCAM; otherwise 0. The priority of a table + * entry (relevant when more than one masked entry matches) ranges from + * 0=highest to N_PRIORITIES-1=lowest. + */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_N_PRIORITIES_OFST 14 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_N_PRIORITIES_LEN 2 +/* Maximum number of masks for STCAM; otherwise 0. */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_MAX_MASKS_OFST 16 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_MAX_MASKS_LEN 2 +/* Flags. */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FLAGS_OFST 18 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FLAGS_LEN 1 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_ALLOC_MASKS_OFST 18 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_ALLOC_MASKS_LBN 0 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_ALLOC_MASKS_WIDTH 1 +/* Access scheme version code, allowing the method of accessing table entries + * to change semantics in future. A client which does not understand the value + * of this field should assume that it cannot program this table. Currently + * always set to 0 indicating the original MC_CMD_TABLE_INSERT/UPDATE/DELETE + * semantics. + */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_SCHEME_OFST 19 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_SCHEME_LEN 1 +/* A sequence of TABLE_FIELD_DESCR structures: N_KEY_FIELDS items describing + * the key, followed by N_RESP_FIELDS items describing the response. If all + * N_KEY_FIELDS+N_RESP_FIELDS items do not fit, further items can be obtained + * by repeating the call with a non-zero FIRST_FIELDS_INDEX. + */ +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_OFST 20 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_LEN 8 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_LO_OFST 20 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_LO_LEN 4 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_LO_LBN 160 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_LO_WIDTH 32 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_HI_OFST 24 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_HI_LEN 4 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_HI_LBN 192 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_HI_WIDTH 32 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_MINNUM 1 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_MAXNUM 29 +#define MC_CMD_TABLE_DESCRIPTOR_OUT_FIELDS_MAXNUM_MCDI2 125 + + +/***********************************/ +/* MC_CMD_TABLE_INSERT + * Insert a new entry into a table. The entry must not currently exist. May + * return EINVAL for unknown table ID or other bad request parameters, EEXIST + * if the entry already exists, ENOSPC if there is no space or EPERM if the + * operation is not permitted. In case of an error, the additional MCDI error + * argument field returns the raw error code from the underlying CAM driver. + */ +#define MC_CMD_TABLE_INSERT 0x1cd +#undef MC_CMD_0x1cd_PRIVILEGE_CTG + +#define MC_CMD_0x1cd_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TABLE_INSERT_IN msgrequest */ +#define MC_CMD_TABLE_INSERT_IN_LENMIN 16 +#define MC_CMD_TABLE_INSERT_IN_LENMAX 252 +#define MC_CMD_TABLE_INSERT_IN_LENMAX_MCDI2 1020 +#define MC_CMD_TABLE_INSERT_IN_LEN(num) (12+4*(num)) +#define MC_CMD_TABLE_INSERT_IN_DATA_NUM(len) (((len)-12)/4) +/* Table identifier. */ +#define MC_CMD_TABLE_INSERT_IN_TABLE_ID_OFST 0 +#define MC_CMD_TABLE_INSERT_IN_TABLE_ID_LEN 4 +/* Enum values, see field(s): */ +/* TABLE_ID */ +/* Width in bits of supplied key data (must match table properties). */ +#define MC_CMD_TABLE_INSERT_IN_KEY_WIDTH_OFST 4 +#define MC_CMD_TABLE_INSERT_IN_KEY_WIDTH_LEN 2 +/* Width in bits of supplied mask data (0 for direct/BCAM tables, or for STCAM + * when allocated MASK_ID is used instead). + */ +#define MC_CMD_TABLE_INSERT_IN_MASK_WIDTH_OFST 6 +#define MC_CMD_TABLE_INSERT_IN_MASK_WIDTH_LEN 2 +/* Width in bits of supplied response data (for INSERT and UPDATE operations + * this must match the table properties; for DELETE operations, no response + * data is required and this must be 0). + */ +#define MC_CMD_TABLE_INSERT_IN_RESP_WIDTH_OFST 8 +#define MC_CMD_TABLE_INSERT_IN_RESP_WIDTH_LEN 2 +/* Mask ID for STCAM table - used instead of mask data if the table descriptor + * reports ALLOC_MASKS==1. Otherwise set to 0. + */ +#define MC_CMD_TABLE_INSERT_IN_MASK_ID_OFST 6 +#define MC_CMD_TABLE_INSERT_IN_MASK_ID_LEN 2 +/* Priority for TCAM or STCAM, in range 0..N_PRIORITIES-1, otherwise 0. */ +#define MC_CMD_TABLE_INSERT_IN_PRIORITY_OFST 8 +#define MC_CMD_TABLE_INSERT_IN_PRIORITY_LEN 2 +/* (32-bit alignment padding - set to 0) */ +#define MC_CMD_TABLE_INSERT_IN_RESERVED_OFST 10 +#define MC_CMD_TABLE_INSERT_IN_RESERVED_LEN 2 +/* Sequence of key, mask (if MASK_WIDTH > 0), and response (if RESP_WIDTH > 0) + * data values. Each of these items is logically treated as a single wide N-bit + * value, in which the individual fields have been placed within that value per + * the LBN and WIDTH information from the table field descriptors. The wide + * N-bit value is padded with 0 bits at the MSB end if necessary to make a + * multiple of 32 bits. The value is then packed into this command as a + * sequence of 32-bit words, bits [31:0] first, then bits [63:32], etc. + */ +#define MC_CMD_TABLE_INSERT_IN_DATA_OFST 12 +#define MC_CMD_TABLE_INSERT_IN_DATA_LEN 4 +#define MC_CMD_TABLE_INSERT_IN_DATA_MINNUM 1 +#define MC_CMD_TABLE_INSERT_IN_DATA_MAXNUM 60 +#define MC_CMD_TABLE_INSERT_IN_DATA_MAXNUM_MCDI2 252 + +/* MC_CMD_TABLE_INSERT_OUT msgresponse */ +#define MC_CMD_TABLE_INSERT_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_TABLE_UPDATE + * Update an existing entry in a table with a new response value. May return + * EINVAL for unknown table ID or other bad request parameters, ENOENT if the + * entry does not already exist, or EPERM if the operation is not permitted. In + * case of an error, the additional MCDI error argument field returns the raw + * error code from the underlying CAM driver. + */ +#define MC_CMD_TABLE_UPDATE 0x1ce +#undef MC_CMD_0x1ce_PRIVILEGE_CTG + +#define MC_CMD_0x1ce_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TABLE_UPDATE_IN msgrequest */ +#define MC_CMD_TABLE_UPDATE_IN_LENMIN 16 +#define MC_CMD_TABLE_UPDATE_IN_LENMAX 252 +#define MC_CMD_TABLE_UPDATE_IN_LENMAX_MCDI2 1020 +#define MC_CMD_TABLE_UPDATE_IN_LEN(num) (12+4*(num)) +#define MC_CMD_TABLE_UPDATE_IN_DATA_NUM(len) (((len)-12)/4) +/* Table identifier. */ +#define MC_CMD_TABLE_UPDATE_IN_TABLE_ID_OFST 0 +#define MC_CMD_TABLE_UPDATE_IN_TABLE_ID_LEN 4 +/* Enum values, see field(s): */ +/* TABLE_ID */ +/* Width in bits of supplied key data (must match table properties). */ +#define MC_CMD_TABLE_UPDATE_IN_KEY_WIDTH_OFST 4 +#define MC_CMD_TABLE_UPDATE_IN_KEY_WIDTH_LEN 2 +/* Width in bits of supplied mask data (0 for direct/BCAM tables, or for STCAM + * when allocated MASK_ID is used instead). + */ +#define MC_CMD_TABLE_UPDATE_IN_MASK_WIDTH_OFST 6 +#define MC_CMD_TABLE_UPDATE_IN_MASK_WIDTH_LEN 2 +/* Width in bits of supplied response data (for INSERT and UPDATE operations + * this must match the table properties; for DELETE operations, no response + * data is required and this must be 0). + */ +#define MC_CMD_TABLE_UPDATE_IN_RESP_WIDTH_OFST 8 +#define MC_CMD_TABLE_UPDATE_IN_RESP_WIDTH_LEN 2 +/* Mask ID for STCAM table - used instead of mask data if the table descriptor + * reports ALLOC_MASKS==1. Otherwise set to 0. + */ +#define MC_CMD_TABLE_UPDATE_IN_MASK_ID_OFST 6 +#define MC_CMD_TABLE_UPDATE_IN_MASK_ID_LEN 2 +/* Priority for TCAM or STCAM, in range 0..N_PRIORITIES-1, otherwise 0. */ +#define MC_CMD_TABLE_UPDATE_IN_PRIORITY_OFST 8 +#define MC_CMD_TABLE_UPDATE_IN_PRIORITY_LEN 2 +/* (32-bit alignment padding - set to 0) */ +#define MC_CMD_TABLE_UPDATE_IN_RESERVED_OFST 10 +#define MC_CMD_TABLE_UPDATE_IN_RESERVED_LEN 2 +/* Sequence of key, mask (if MASK_WIDTH > 0), and response (if RESP_WIDTH > 0) + * data values. Each of these items is logically treated as a single wide N-bit + * value, in which the individual fields have been placed within that value per + * the LBN and WIDTH information from the table field descriptors. The wide + * N-bit value is padded with 0 bits at the MSB end if necessary to make a + * multiple of 32 bits. The value is then packed into this command as a + * sequence of 32-bit words, bits [31:0] first, then bits [63:32], etc. + */ +#define MC_CMD_TABLE_UPDATE_IN_DATA_OFST 12 +#define MC_CMD_TABLE_UPDATE_IN_DATA_LEN 4 +#define MC_CMD_TABLE_UPDATE_IN_DATA_MINNUM 1 +#define MC_CMD_TABLE_UPDATE_IN_DATA_MAXNUM 60 +#define MC_CMD_TABLE_UPDATE_IN_DATA_MAXNUM_MCDI2 252 + +/* MC_CMD_TABLE_UPDATE_OUT msgresponse */ +#define MC_CMD_TABLE_UPDATE_OUT_LEN 0 + + +/***********************************/ +/* MC_CMD_TABLE_DELETE + * Delete an existing entry in a table. May return EINVAL for unknown table ID + * or other bad request parameters, ENOENT if the entry does not exist, or + * EPERM if the operation is not permitted. In case of an error, the additional + * MCDI error argument field returns the raw error code from the underlying CAM + * driver. + */ +#define MC_CMD_TABLE_DELETE 0x1cf +#undef MC_CMD_0x1cf_PRIVILEGE_CTG + +#define MC_CMD_0x1cf_PRIVILEGE_CTG SRIOV_CTG_GENERAL + +/* MC_CMD_TABLE_DELETE_IN msgrequest */ +#define MC_CMD_TABLE_DELETE_IN_LENMIN 16 +#define MC_CMD_TABLE_DELETE_IN_LENMAX 252 +#define MC_CMD_TABLE_DELETE_IN_LENMAX_MCDI2 1020 +#define MC_CMD_TABLE_DELETE_IN_LEN(num) (12+4*(num)) +#define MC_CMD_TABLE_DELETE_IN_DATA_NUM(len) (((len)-12)/4) +/* Table identifier. */ +#define MC_CMD_TABLE_DELETE_IN_TABLE_ID_OFST 0 +#define MC_CMD_TABLE_DELETE_IN_TABLE_ID_LEN 4 +/* Enum values, see field(s): */ +/* TABLE_ID */ +/* Width in bits of supplied key data (must match table properties). */ +#define MC_CMD_TABLE_DELETE_IN_KEY_WIDTH_OFST 4 +#define MC_CMD_TABLE_DELETE_IN_KEY_WIDTH_LEN 2 +/* Width in bits of supplied mask data (0 for direct/BCAM tables, or for STCAM + * when allocated MASK_ID is used instead). + */ +#define MC_CMD_TABLE_DELETE_IN_MASK_WIDTH_OFST 6 +#define MC_CMD_TABLE_DELETE_IN_MASK_WIDTH_LEN 2 +/* Width in bits of supplied response data (for INSERT and UPDATE operations + * this must match the table properties; for DELETE operations, no response + * data is required and this must be 0). + */ +#define MC_CMD_TABLE_DELETE_IN_RESP_WIDTH_OFST 8 +#define MC_CMD_TABLE_DELETE_IN_RESP_WIDTH_LEN 2 +/* Mask ID for STCAM table - used instead of mask data if the table descriptor + * reports ALLOC_MASKS==1. Otherwise set to 0. + */ +#define MC_CMD_TABLE_DELETE_IN_MASK_ID_OFST 6 +#define MC_CMD_TABLE_DELETE_IN_MASK_ID_LEN 2 +/* Priority for TCAM or STCAM, in range 0..N_PRIORITIES-1, otherwise 0. */ +#define MC_CMD_TABLE_DELETE_IN_PRIORITY_OFST 8 +#define MC_CMD_TABLE_DELETE_IN_PRIORITY_LEN 2 +/* (32-bit alignment padding - set to 0) */ +#define MC_CMD_TABLE_DELETE_IN_RESERVED_OFST 10 +#define MC_CMD_TABLE_DELETE_IN_RESERVED_LEN 2 +/* Sequence of key, mask (if MASK_WIDTH > 0), and response (if RESP_WIDTH > 0) + * data values. Each of these items is logically treated as a single wide N-bit + * value, in which the individual fields have been placed within that value per + * the LBN and WIDTH information from the table field descriptors. The wide + * N-bit value is padded with 0 bits at the MSB end if necessary to make a + * multiple of 32 bits. The value is then packed into this command as a + * sequence of 32-bit words, bits [31:0] first, then bits [63:32], etc. + */ +#define MC_CMD_TABLE_DELETE_IN_DATA_OFST 12 +#define MC_CMD_TABLE_DELETE_IN_DATA_LEN 4 +#define MC_CMD_TABLE_DELETE_IN_DATA_MINNUM 1 +#define MC_CMD_TABLE_DELETE_IN_DATA_MAXNUM 60 +#define MC_CMD_TABLE_DELETE_IN_DATA_MAXNUM_MCDI2 252 + +/* MC_CMD_TABLE_DELETE_OUT msgresponse */ +#define MC_CMD_TABLE_DELETE_OUT_LEN 0 #endif /* MCDI_PCOL_H */ diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 2228c88a7f31..4cde54cf77b9 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -178,6 +178,7 @@ struct efx_tx_buffer { #define EFX_TX_BUF_OPTION 0x10 /* empty buffer for option descriptor */ #define EFX_TX_BUF_XDP 0x20 /* buffer was sent with XDP */ #define EFX_TX_BUF_TSO_V3 0x40 /* empty buffer for a TSO_V3 descriptor */ +#define EFX_TX_BUF_EFV 0x100 /* buffer was sent from representor */ /** * struct efx_tx_queue - An Efx TX queue @@ -966,6 +967,8 @@ enum efx_xdp_tx_queues_mode { * @vf_count: Number of VFs intended to be enabled. * @vf_init_count: Number of VFs that have been fully initialised. * @vi_scale: log2 number of vnics per VF. + * @vf_reps_lock: Protects vf_reps list + * @vf_reps: local VF reps * @ptp_data: PTP state data * @ptp_warned: has this NIC seen and warned about unexpected PTP events? * @vpd_sn: Serial number read from VPD @@ -1145,6 +1148,8 @@ struct efx_nic { unsigned vf_init_count; unsigned vi_scale; #endif + spinlock_t vf_reps_lock; + struct list_head vf_reps; struct efx_ptp_data *ptp_data; bool ptp_warned; diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c index 79cc0bb76321..d12474042c84 100644 --- a/drivers/net/ethernet/sfc/tx.c +++ b/drivers/net/ethernet/sfc/tx.c @@ -559,6 +559,7 @@ netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, void efx_xmit_done_single(struct efx_tx_queue *tx_queue) { unsigned int pkts_compl = 0, bytes_compl = 0; + unsigned int efv_pkts_compl = 0; unsigned int read_ptr; bool finished = false; @@ -580,7 +581,8 @@ void efx_xmit_done_single(struct efx_tx_queue *tx_queue) /* Need to check the flag before dequeueing. */ if (buffer->flags & EFX_TX_BUF_SKB) finished = true; - efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); + efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl, + &efv_pkts_compl); ++tx_queue->read_count; read_ptr = tx_queue->read_count & tx_queue->ptr_mask; @@ -589,7 +591,7 @@ void efx_xmit_done_single(struct efx_tx_queue *tx_queue) tx_queue->pkts_compl += pkts_compl; tx_queue->bytes_compl += bytes_compl; - EFX_WARN_ON_PARANOID(pkts_compl != 1); + EFX_WARN_ON_PARANOID(pkts_compl + efv_pkts_compl != 1); efx_xmit_done_check_empty(tx_queue); } diff --git a/drivers/net/ethernet/sfc/tx_common.c b/drivers/net/ethernet/sfc/tx_common.c index 658ea2d34070..67e789b96c43 100644 --- a/drivers/net/ethernet/sfc/tx_common.c +++ b/drivers/net/ethernet/sfc/tx_common.c @@ -109,9 +109,11 @@ void efx_fini_tx_queue(struct efx_tx_queue *tx_queue) /* Free any buffers left in the ring */ while (tx_queue->read_count != tx_queue->write_count) { unsigned int pkts_compl = 0, bytes_compl = 0; + unsigned int efv_pkts_compl = 0; buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask]; - efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); + efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl, + &efv_pkts_compl); ++tx_queue->read_count; } @@ -146,7 +148,8 @@ void efx_remove_tx_queue(struct efx_tx_queue *tx_queue) void efx_dequeue_buffer(struct efx_tx_queue *tx_queue, struct efx_tx_buffer *buffer, unsigned int *pkts_compl, - unsigned int *bytes_compl) + unsigned int *bytes_compl, + unsigned int *efv_pkts_compl) { if (buffer->unmap_len) { struct device *dma_dev = &tx_queue->efx->pci_dev->dev; @@ -164,9 +167,15 @@ void efx_dequeue_buffer(struct efx_tx_queue *tx_queue, if (buffer->flags & EFX_TX_BUF_SKB) { struct sk_buff *skb = (struct sk_buff *)buffer->skb; - EFX_WARN_ON_PARANOID(!pkts_compl || !bytes_compl); - (*pkts_compl)++; - (*bytes_compl) += skb->len; + if (unlikely(buffer->flags & EFX_TX_BUF_EFV)) { + EFX_WARN_ON_PARANOID(!efv_pkts_compl); + (*efv_pkts_compl)++; + } else { + EFX_WARN_ON_PARANOID(!pkts_compl || !bytes_compl); + (*pkts_compl)++; + (*bytes_compl) += skb->len; + } + if (tx_queue->timestamping && (tx_queue->completed_timestamp_major || tx_queue->completed_timestamp_minor)) { @@ -199,7 +208,8 @@ void efx_dequeue_buffer(struct efx_tx_queue *tx_queue, static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue, unsigned int index, unsigned int *pkts_compl, - unsigned int *bytes_compl) + unsigned int *bytes_compl, + unsigned int *efv_pkts_compl) { struct efx_nic *efx = tx_queue->efx; unsigned int stop_index, read_ptr; @@ -218,7 +228,8 @@ static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue, return; } - efx_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl); + efx_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl, + efv_pkts_compl); ++tx_queue->read_count; read_ptr = tx_queue->read_count & tx_queue->ptr_mask; @@ -241,15 +252,17 @@ void efx_xmit_done_check_empty(struct efx_tx_queue *tx_queue) void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) { unsigned int fill_level, pkts_compl = 0, bytes_compl = 0; + unsigned int efv_pkts_compl = 0; struct efx_nic *efx = tx_queue->efx; EFX_WARN_ON_ONCE_PARANOID(index > tx_queue->ptr_mask); - efx_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl); + efx_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl, + &efv_pkts_compl); tx_queue->pkts_compl += pkts_compl; tx_queue->bytes_compl += bytes_compl; - if (pkts_compl > 1) + if (pkts_compl + efv_pkts_compl > 1) ++tx_queue->merge_events; /* See if we need to restart the netif queue. This memory @@ -274,6 +287,7 @@ void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) void efx_enqueue_unwind(struct efx_tx_queue *tx_queue, unsigned int insert_count) { + unsigned int efv_pkts_compl = 0; struct efx_tx_buffer *buffer; unsigned int bytes_compl = 0; unsigned int pkts_compl = 0; @@ -282,7 +296,8 @@ void efx_enqueue_unwind(struct efx_tx_queue *tx_queue, while (tx_queue->insert_count != insert_count) { --tx_queue->insert_count; buffer = __efx_tx_queue_get_insert_buffer(tx_queue); - efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl); + efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl, + &efv_pkts_compl); } } diff --git a/drivers/net/ethernet/sfc/tx_common.h b/drivers/net/ethernet/sfc/tx_common.h index bbab7f248250..d87aecbc7bf1 100644 --- a/drivers/net/ethernet/sfc/tx_common.h +++ b/drivers/net/ethernet/sfc/tx_common.h @@ -19,7 +19,8 @@ void efx_remove_tx_queue(struct efx_tx_queue *tx_queue); void efx_dequeue_buffer(struct efx_tx_queue *tx_queue, struct efx_tx_buffer *buffer, unsigned int *pkts_compl, - unsigned int *bytes_compl); + unsigned int *bytes_compl, + unsigned int *efv_pkts_compl); static inline bool efx_tx_buffer_in_use(struct efx_tx_buffer *buffer) { diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index d0e82cb5ae03..52f9ed8db9c9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -297,6 +297,11 @@ static void get_arttime(struct mii_bus *mii, int intel_adhoc_addr, *art_time = ns; } +static int stmmac_cross_ts_isr(struct stmmac_priv *priv) +{ + return (readl(priv->ioaddr + GMAC_INT_STATUS) & GMAC_INT_TSIE); +} + static int intel_crosststamp(ktime_t *device, struct system_counterval_t *system, void *ctx) @@ -312,8 +317,6 @@ static int intel_crosststamp(ktime_t *device, u32 num_snapshot; u32 gpio_value; u32 acr_value; - int ret; - u32 v; int i; if (!boot_cpu_has(X86_FEATURE_ART)) @@ -327,6 +330,8 @@ static int intel_crosststamp(ktime_t *device, if (priv->plat->ext_snapshot_en) return -EBUSY; + priv->plat->int_snapshot_en = 1; + mutex_lock(&priv->aux_ts_lock); /* Enable Internal snapshot trigger */ acr_value = readl(ptpaddr + PTP_ACR); @@ -346,6 +351,7 @@ static int intel_crosststamp(ktime_t *device, break; default: mutex_unlock(&priv->aux_ts_lock); + priv->plat->int_snapshot_en = 0; return -EINVAL; } writel(acr_value, ptpaddr + PTP_ACR); @@ -367,13 +373,12 @@ static int intel_crosststamp(ktime_t *device, gpio_value |= GMAC_GPO1; writel(gpio_value, ioaddr + GMAC_GPIO_STATUS); - /* Poll for time sync operation done */ - ret = readl_poll_timeout(priv->ioaddr + GMAC_INT_STATUS, v, - (v & GMAC_INT_TSIE), 100, 10000); - - if (ret == -ETIMEDOUT) { - pr_err("%s: Wait for time sync operation timeout\n", __func__); - return ret; + /* Time sync done Indication - Interrupt method */ + if (!wait_event_interruptible_timeout(priv->tstamp_busy_wait, + stmmac_cross_ts_isr(priv), + HZ / 100)) { + priv->plat->int_snapshot_en = 0; + return -ETIMEDOUT; } num_snapshot = (readl(ioaddr + GMAC_TIMESTAMP_STATUS) & @@ -391,6 +396,7 @@ static int intel_crosststamp(ktime_t *device, } system->cycles *= intel_priv->crossts_adj; + priv->plat->int_snapshot_en = 0; return 0; } @@ -606,6 +612,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, plat->has_crossts = true; plat->crosststamp = intel_crosststamp; + plat->int_snapshot_en = 0; /* Setup MSI vector offset specific to Intel mGbE controller */ plat->msi_mac_vec = 29; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c index 6ff88df58767..ca8ab290013c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c @@ -576,32 +576,7 @@ static int mediatek_dwmac_init(struct platform_device *pdev, void *priv) } } - ret = clk_bulk_prepare_enable(variant->num_clks, plat->clks); - if (ret) { - dev_err(plat->dev, "failed to enable clks, err = %d\n", ret); - return ret; - } - - ret = clk_prepare_enable(plat->rmii_internal_clk); - if (ret) { - dev_err(plat->dev, "failed to enable rmii internal clk, err = %d\n", ret); - goto err_clk; - } - return 0; - -err_clk: - clk_bulk_disable_unprepare(variant->num_clks, plat->clks); - return ret; -} - -static void mediatek_dwmac_exit(struct platform_device *pdev, void *priv) -{ - struct mediatek_dwmac_plat_data *plat = priv; - const struct mediatek_dwmac_variant *variant = plat->variant; - - clk_disable_unprepare(plat->rmii_internal_clk); - clk_bulk_disable_unprepare(variant->num_clks, plat->clks); } static int mediatek_dwmac_clks_config(void *priv, bool enabled) @@ -643,7 +618,6 @@ static int mediatek_dwmac_common_data(struct platform_device *pdev, plat->addr64 = priv_plat->variant->dma_bit_mask; plat->bsp_priv = priv_plat; plat->init = mediatek_dwmac_init; - plat->exit = mediatek_dwmac_exit; plat->clks_config = mediatek_dwmac_clks_config; if (priv_plat->variant->dwmac_fix_mac_speed) plat->fix_mac_speed = priv_plat->variant->dwmac_fix_mac_speed; @@ -712,13 +686,32 @@ static int mediatek_dwmac_probe(struct platform_device *pdev) mediatek_dwmac_common_data(pdev, plat_dat, priv_plat); mediatek_dwmac_init(pdev, priv_plat); + ret = mediatek_dwmac_clks_config(priv_plat, true); + if (ret) + return ret; + ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); if (ret) { stmmac_remove_config_dt(pdev, plat_dat); - return ret; + goto err_drv_probe; } return 0; + +err_drv_probe: + mediatek_dwmac_clks_config(priv_plat, false); + return ret; +} + +static int mediatek_dwmac_remove(struct platform_device *pdev) +{ + struct mediatek_dwmac_plat_data *priv_plat = get_stmmac_bsp_priv(&pdev->dev); + int ret; + + ret = stmmac_pltfr_remove(pdev); + mediatek_dwmac_clks_config(priv_plat, false); + + return ret; } static const struct of_device_id mediatek_dwmac_match[] = { @@ -733,7 +726,7 @@ MODULE_DEVICE_TABLE(of, mediatek_dwmac_match); static struct platform_driver mediatek_dwmac_driver = { .probe = mediatek_dwmac_probe, - .remove = stmmac_pltfr_remove, + .remove = mediatek_dwmac_remove, .driver = { .name = "dwmac-mediatek", .pm = &stmmac_pltfr_pm_ops, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h index 462ca7ed095a..71dad409f78b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h @@ -150,7 +150,8 @@ #define GMAC_PCS_IRQ_DEFAULT (GMAC_INT_RGSMIIS | GMAC_INT_PCS_LINK | \ GMAC_INT_PCS_ANE) -#define GMAC_INT_DEFAULT_ENABLE (GMAC_INT_PMT_EN | GMAC_INT_LPI_EN) +#define GMAC_INT_DEFAULT_ENABLE (GMAC_INT_PMT_EN | GMAC_INT_LPI_EN | \ + GMAC_INT_TSIE) enum dwmac4_irq_status { time_stamp_irq = 0x00001000, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index fd41db65fe1d..d8f1fbc25bdd 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -23,6 +23,7 @@ static void dwmac4_core_init(struct mac_device_info *hw, struct net_device *dev) { + struct stmmac_priv *priv = netdev_priv(dev); void __iomem *ioaddr = hw->pcsr; u32 value = readl(ioaddr + GMAC_CONFIG); @@ -58,6 +59,9 @@ static void dwmac4_core_init(struct mac_device_info *hw, value |= GMAC_INT_FPE_EN; writel(value, ioaddr + GMAC_INT_EN); + + if (GMAC_INT_DEFAULT_ENABLE & GMAC_INT_TSIE) + init_waitqueue_head(&priv->tstamp_busy_wait); } static void dwmac4_rx_queue_enable(struct mac_device_info *hw, @@ -219,6 +223,9 @@ static void dwmac4_map_mtl_dma(struct mac_device_info *hw, u32 queue, u32 chan) if (queue == 0 || queue == 4) { value &= ~MTL_RXQ_DMA_Q04MDMACH_MASK; value |= MTL_RXQ_DMA_Q04MDMACH(chan); + } else if (queue > 4) { + value &= ~MTL_RXQ_DMA_QXMDMACH_MASK(queue - 4); + value |= MTL_RXQ_DMA_QXMDMACH(chan, queue - 4); } else { value &= ~MTL_RXQ_DMA_QXMDMACH_MASK(queue); value |= MTL_RXQ_DMA_QXMDMACH(chan, queue); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 57970ae2178d..f9e83964aa7e 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -266,6 +266,7 @@ struct stmmac_priv { rwlock_t ptp_lock; /* Protects auxiliary snapshot registers from concurrent access. */ struct mutex aux_ts_lock; + wait_queue_head_t tstamp_busy_wait; void __iomem *mmcaddr; void __iomem *ptpaddr; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index abfb3cd5958d..9c3055ee2608 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -803,14 +803,6 @@ static int stmmac_ethtool_op_set_eee(struct net_device *dev, netdev_warn(priv->dev, "Setting EEE tx-lpi is not supported\n"); - if (priv->hw->xpcs) { - ret = xpcs_config_eee(priv->hw->xpcs, - priv->plat->mult_fact_100ns, - edata->eee_enabled); - if (ret) - return ret; - } - if (!edata->eee_enabled) stmmac_disable_eee_mode(priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c index 92d32940aff0..764832f4dae1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_hwtstamp.c @@ -179,6 +179,11 @@ static void timestamp_interrupt(struct stmmac_priv *priv) u64 ptp_time; int i; + if (priv->plat->int_snapshot_en) { + wake_up(&priv->tstamp_busy_wait); + return; + } + tsync_int = readl(priv->ioaddr + GMAC_INT_STATUS) & GMAC_INT_TSIE; if (!tsync_int) diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 6f14b00c0b14..89eb3e51b22c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -834,19 +834,10 @@ int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) struct timespec64 now; u32 sec_inc = 0; u64 temp = 0; - int ret; if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) return -EOPNOTSUPP; - ret = clk_prepare_enable(priv->plat->clk_ptp_ref); - if (ret < 0) { - netdev_warn(priv->dev, - "failed to enable PTP reference clock: %pe\n", - ERR_PTR(ret)); - return ret; - } - stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); priv->systime_flags = systime_flags; @@ -3272,6 +3263,14 @@ static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) stmmac_mmc_setup(priv); + if (ptp_register) { + ret = clk_prepare_enable(priv->plat->clk_ptp_ref); + if (ret < 0) + netdev_warn(priv->dev, + "failed to enable PTP reference clock: %pe\n", + ERR_PTR(ret)); + } + ret = stmmac_init_ptp(priv); if (ret == -EOPNOTSUPP) netdev_info(priv->dev, "PTP not supported by HW\n"); @@ -7215,8 +7214,6 @@ int stmmac_dvr_remove(struct device *dev) netdev_info(priv->dev, "%s: removing driver", __func__); pm_runtime_get_sync(dev); - pm_runtime_disable(dev); - pm_runtime_put_noidle(dev); stmmac_stop_all_dma(priv); stmmac_mac_set(priv, priv->ioaddr, false); @@ -7243,6 +7240,9 @@ int stmmac_dvr_remove(struct device *dev) mutex_destroy(&priv->lock); bitmap_free(priv->af_xdp_zc_qps); + pm_runtime_disable(dev); + pm_runtime_put_noidle(dev); + return 0; } EXPORT_SYMBOL_GPL(stmmac_dvr_remove); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 11e1055e8260..9f5cac4000da 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -815,7 +815,13 @@ static int __maybe_unused stmmac_pltfr_noirq_resume(struct device *dev) if (ret) return ret; - stmmac_init_tstamp_counter(priv, priv->systime_flags); + ret = clk_prepare_enable(priv->plat->clk_ptp_ref); + if (ret < 0) { + netdev_warn(priv->dev, + "failed to enable PTP reference clock: %pe\n", + ERR_PTR(ret)); + return ret; + } } return 0; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index e45fb191d8e6..4d11980dcd64 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -175,11 +175,10 @@ static int stmmac_enable(struct ptp_clock_info *ptp, struct stmmac_priv *priv = container_of(ptp, struct stmmac_priv, ptp_clock_ops); void __iomem *ptpaddr = priv->ptpaddr; - void __iomem *ioaddr = priv->hw->pcsr; struct stmmac_pps_cfg *cfg; - u32 intr_value, acr_value; int ret = -EOPNOTSUPP; unsigned long flags; + u32 acr_value; switch (rq->type) { case PTP_CLK_REQ_PEROUT: @@ -213,19 +212,10 @@ static int stmmac_enable(struct ptp_clock_info *ptp, netdev_dbg(priv->dev, "Auxiliary Snapshot %d enabled.\n", priv->plat->ext_snapshot_num >> PTP_ACR_ATSEN_SHIFT); - /* Enable Timestamp Interrupt */ - intr_value = readl(ioaddr + GMAC_INT_EN); - intr_value |= GMAC_INT_TSIE; - writel(intr_value, ioaddr + GMAC_INT_EN); - } else { netdev_dbg(priv->dev, "Auxiliary Snapshot %d disabled.\n", priv->plat->ext_snapshot_num >> PTP_ACR_ATSEN_SHIFT); - /* Disable Timestamp Interrupt */ - intr_value = readl(ioaddr + GMAC_INT_EN); - intr_value &= ~GMAC_INT_TSIE; - writel(intr_value, ioaddr + GMAC_INT_EN); } writel(acr_value, ptpaddr + PTP_ACR); mutex_unlock(&priv->aux_ts_lock); diff --git a/drivers/net/ipa/Makefile b/drivers/net/ipa/Makefile index bdfb2430ab2c..8b2220eb6b92 100644 --- a/drivers/net/ipa/Makefile +++ b/drivers/net/ipa/Makefile @@ -1,3 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Qualcomm IPA driver. + +IPA_VERSIONS := 3.1 3.5.1 4.2 4.5 4.9 4.11 + obj-$(CONFIG_QCOM_IPA) += ipa.o ipa-y := ipa_main.o ipa_power.o ipa_reg.o ipa_mem.o \ @@ -7,6 +13,4 @@ ipa-y := ipa_main.o ipa_power.o ipa_reg.o ipa_mem.o \ ipa_resource.o ipa_qmi.o ipa_qmi_msg.o \ ipa_sysfs.o -ipa-y += ipa_data-v3.1.o ipa_data-v3.5.1.o \ - ipa_data-v4.2.o ipa_data-v4.5.o \ - ipa_data-v4.9.o ipa_data-v4.11.o +ipa-y += $(IPA_VERSIONS:%=data/ipa_data-v%.o) diff --git a/drivers/net/ipa/ipa_data-v3.1.c b/drivers/net/ipa/data/ipa_data-v3.1.c index 00f4e506e6e5..1c1895aea811 100644 --- a/drivers/net/ipa/ipa_data-v3.1.c +++ b/drivers/net/ipa/data/ipa_data-v3.1.c @@ -6,10 +6,10 @@ #include <linux/log2.h> -#include "gsi.h" -#include "ipa_data.h" -#include "ipa_endpoint.h" -#include "ipa_mem.h" +#include "../gsi.h" +#include "../ipa_data.h" +#include "../ipa_endpoint.h" +#include "../ipa_mem.h" /** enum ipa_resource_type - IPA resource types for an SoC having IPA v3.1 */ enum ipa_resource_type { diff --git a/drivers/net/ipa/ipa_data-v3.5.1.c b/drivers/net/ipa/data/ipa_data-v3.5.1.c index b7e32e87733e..58b708d2fc75 100644 --- a/drivers/net/ipa/ipa_data-v3.5.1.c +++ b/drivers/net/ipa/data/ipa_data-v3.5.1.c @@ -6,10 +6,10 @@ #include <linux/log2.h> -#include "gsi.h" -#include "ipa_data.h" -#include "ipa_endpoint.h" -#include "ipa_mem.h" +#include "../gsi.h" +#include "../ipa_data.h" +#include "../ipa_endpoint.h" +#include "../ipa_mem.h" /** enum ipa_resource_type - IPA resource types for an SoC having IPA v3.5.1 */ enum ipa_resource_type { diff --git a/drivers/net/ipa/ipa_data-v4.11.c b/drivers/net/ipa/data/ipa_data-v4.11.c index 1be823e5c5c2..a204e439c23d 100644 --- a/drivers/net/ipa/ipa_data-v4.11.c +++ b/drivers/net/ipa/data/ipa_data-v4.11.c @@ -4,10 +4,10 @@ #include <linux/log2.h> -#include "gsi.h" -#include "ipa_data.h" -#include "ipa_endpoint.h" -#include "ipa_mem.h" +#include "../gsi.h" +#include "../ipa_data.h" +#include "../ipa_endpoint.h" +#include "../ipa_mem.h" /** enum ipa_resource_type - IPA resource types for an SoC having IPA v4.11 */ enum ipa_resource_type { diff --git a/drivers/net/ipa/ipa_data-v4.2.c b/drivers/net/ipa/data/ipa_data-v4.2.c index 683f1f91042f..04f574fe006f 100644 --- a/drivers/net/ipa/ipa_data-v4.2.c +++ b/drivers/net/ipa/data/ipa_data-v4.2.c @@ -4,10 +4,10 @@ #include <linux/log2.h> -#include "gsi.h" -#include "ipa_data.h" -#include "ipa_endpoint.h" -#include "ipa_mem.h" +#include "../gsi.h" +#include "../ipa_data.h" +#include "../ipa_endpoint.h" +#include "../ipa_mem.h" /** enum ipa_resource_type - IPA resource types for an SoC having IPA v4.2 */ enum ipa_resource_type { diff --git a/drivers/net/ipa/ipa_data-v4.5.c b/drivers/net/ipa/data/ipa_data-v4.5.c index 79398f286a9c..684239e71f46 100644 --- a/drivers/net/ipa/ipa_data-v4.5.c +++ b/drivers/net/ipa/data/ipa_data-v4.5.c @@ -4,10 +4,10 @@ #include <linux/log2.h> -#include "gsi.h" -#include "ipa_data.h" -#include "ipa_endpoint.h" -#include "ipa_mem.h" +#include "../gsi.h" +#include "../ipa_data.h" +#include "../ipa_endpoint.h" +#include "../ipa_mem.h" /** enum ipa_resource_type - IPA resource types for an SoC having IPA v4.5 */ enum ipa_resource_type { diff --git a/drivers/net/ipa/ipa_data-v4.9.c b/drivers/net/ipa/data/ipa_data-v4.9.c index 4b96efd05cf2..2333e15f9533 100644 --- a/drivers/net/ipa/ipa_data-v4.9.c +++ b/drivers/net/ipa/data/ipa_data-v4.9.c @@ -4,10 +4,10 @@ #include <linux/log2.h> -#include "gsi.h" -#include "ipa_data.h" -#include "ipa_endpoint.h" -#include "ipa_mem.h" +#include "../gsi.h" +#include "../ipa_data.h" +#include "../ipa_endpoint.h" +#include "../ipa_mem.h" /** enum ipa_resource_type - IPA resource types for an SoC having IPA v4.9 */ enum ipa_resource_type { diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index 4e46974a69ec..9e307eebd33f 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -665,7 +665,8 @@ static void gsi_evt_ring_doorbell(struct gsi *gsi, u32 evt_ring_id, u32 index) static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id) { struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id]; - size_t size = evt_ring->ring.count * GSI_RING_ELEMENT_SIZE; + struct gsi_ring *ring = &evt_ring->ring; + size_t size; u32 val; /* We program all event rings as GPI type/protocol */ @@ -674,6 +675,7 @@ static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id) val |= u32_encode_bits(GSI_RING_ELEMENT_SIZE, EV_ELEMENT_SIZE_FMASK); iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_0_OFFSET(evt_ring_id)); + size = ring->count * GSI_RING_ELEMENT_SIZE; val = ev_r_length_encoded(gsi->version, size); iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_1_OFFSET(evt_ring_id)); @@ -681,9 +683,9 @@ static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id) * high-order 32 bits of the address of the event ring, * respectively. */ - val = lower_32_bits(evt_ring->ring.addr); + val = lower_32_bits(ring->addr); iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_2_OFFSET(evt_ring_id)); - val = upper_32_bits(evt_ring->ring.addr); + val = upper_32_bits(ring->addr); iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_3_OFFSET(evt_ring_id)); /* Enable interrupt moderation by setting the moderation delay */ @@ -700,8 +702,8 @@ static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id) iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_12_OFFSET(evt_ring_id)); iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_13_OFFSET(evt_ring_id)); - /* Finally, tell the hardware we've completed event 0 (arbitrary) */ - gsi_evt_ring_doorbell(gsi, evt_ring_id, 0); + /* Finally, tell the hardware our "last processed" event (arbitrary) */ + gsi_evt_ring_doorbell(gsi, evt_ring_id, ring->index); } /* Find the transaction whose completion indicates a channel is quiesced */ @@ -720,6 +722,9 @@ static struct gsi_trans *gsi_channel_trans_last(struct gsi_channel *channel) list = &trans_info->alloc; if (!list_empty(list)) goto done; + list = &trans_info->committed; + if (!list_empty(list)) + goto done; list = &trans_info->pending; if (!list_empty(list)) goto done; @@ -770,9 +775,6 @@ static void gsi_channel_program(struct gsi_channel *channel, bool doorbell) u32 wrr_weight = 0; u32 val; - /* Arbitrarily pick TRE 0 as the first channel element to use */ - channel->tre_ring.index = 0; - /* We program all channels as GPI type/protocol */ val = chtype_protocol_encoded(gsi->version, GSI_CHANNEL_TYPE_GPI); if (channel->toward_ipa) @@ -949,6 +951,8 @@ void gsi_channel_reset(struct gsi *gsi, u32 channel_id, bool doorbell) if (gsi->version < IPA_VERSION_4_0 && !channel->toward_ipa) gsi_channel_reset_command(channel); + /* Hardware assumes this is 0 following reset */ + channel->tre_ring.index = 0; gsi_channel_program(channel, doorbell); gsi_channel_trans_cancel_pending(channel); @@ -1363,7 +1367,7 @@ gsi_event_trans(struct gsi *gsi, struct gsi_event *event) * first *unfilled* event in the ring (following the last filled one). * * Events are sequential within the event ring, and transactions are - * sequential within the transaction pool. + * sequential within the transaction array. * * Note that @index always refers to an element *within* the event ring. */ @@ -1433,6 +1437,7 @@ static int gsi_ring_alloc(struct gsi *gsi, struct gsi_ring *ring, u32 count) ring->addr = addr; ring->count = count; + ring->index = 0; return 0; } diff --git a/drivers/net/ipa/gsi.h b/drivers/net/ipa/gsi.h index bad1a78a96ed..23de5f67374c 100644 --- a/drivers/net/ipa/gsi.h +++ b/drivers/net/ipa/gsi.h @@ -48,12 +48,13 @@ struct gsi_ring { * * A channel ring consists of TRE entries filled by the AP and passed * to the hardware for processing. For a channel ring, the ring index - * identifies the next unused entry to be filled by the AP. + * identifies the next unused entry to be filled by the AP. In this + * case the initial value is assumed by hardware to be 0. * * An event ring consists of event structures filled by the hardware * and passed to the AP. For event rings, the ring index identifies * the next ring entry that is not known to have been filled by the - * hardware. + * hardware. The initial value used is arbitrary (so we use 0). */ u32 index; }; @@ -82,13 +83,15 @@ struct gsi_trans_pool { struct gsi_trans_info { atomic_t tre_avail; /* TREs available for allocation */ struct gsi_trans_pool pool; /* transaction pool */ + struct gsi_trans **map; /* TRE -> transaction map */ + struct gsi_trans_pool sg_pool; /* scatterlist pool */ struct gsi_trans_pool cmd_pool; /* command payload DMA pool */ - struct gsi_trans **map; /* TRE -> transaction map */ spinlock_t spinlock; /* protects updates to the lists */ struct list_head alloc; /* allocated, not committed */ - struct list_head pending; /* committed, awaiting completion */ + struct list_head committed; /* committed, awaiting doorbell */ + struct list_head pending; /* pending, awaiting completion */ struct list_head complete; /* completed, awaiting poll */ struct list_head polled; /* returned by gsi_channel_poll_one() */ }; @@ -184,7 +187,7 @@ void gsi_teardown(struct gsi *gsi); * @gsi: GSI pointer * @channel_id: Channel whose limit is to be returned * - * Return: The maximum number of TREs oustanding on the channel + * Return: The maximum number of TREs outstanding on the channel */ u32 gsi_channel_tre_max(struct gsi *gsi, u32 channel_id); diff --git a/drivers/net/ipa/gsi_trans.c b/drivers/net/ipa/gsi_trans.c index 29496ca15825..18e7e8c405be 100644 --- a/drivers/net/ipa/gsi_trans.c +++ b/drivers/net/ipa/gsi_trans.c @@ -241,15 +241,31 @@ struct gsi_trans *gsi_channel_trans_complete(struct gsi_channel *channel) struct gsi_trans, links); } -/* Move a transaction from the allocated list to the pending list */ +/* Move a transaction from the allocated list to the committed list */ +static void gsi_trans_move_committed(struct gsi_trans *trans) +{ + struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; + struct gsi_trans_info *trans_info = &channel->trans_info; + + spin_lock_bh(&trans_info->spinlock); + + list_move_tail(&trans->links, &trans_info->committed); + + spin_unlock_bh(&trans_info->spinlock); +} + +/* Move transactions from the committed list to the pending list */ static void gsi_trans_move_pending(struct gsi_trans *trans) { struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; struct gsi_trans_info *trans_info = &channel->trans_info; + struct list_head list; spin_lock_bh(&trans_info->spinlock); - list_move_tail(&trans->links, &trans_info->pending); + /* Move this transaction and all predecessors to the pending list */ + list_cut_position(&list, &trans_info->committed, &trans->links); + list_splice_tail(&list, &trans_info->pending); spin_unlock_bh(&trans_info->spinlock); } @@ -346,7 +362,7 @@ struct gsi_trans *gsi_channel_trans_alloc(struct gsi *gsi, u32 channel_id, trans->rsvd_count = tre_count; init_completion(&trans->completion); - /* Allocate the scatterlist and (if requested) info entries. */ + /* Allocate the scatterlist */ trans->sgl = gsi_trans_pool_alloc(&trans_info->sg_pool, tre_count); sg_init_marker(trans->sgl, tre_count); @@ -388,7 +404,8 @@ void gsi_trans_free(struct gsi_trans *trans) if (!last) return; - ipa_gsi_trans_release(trans); + if (trans->used_count) + ipa_gsi_trans_release(trans); /* Releasing the reserved TREs implicitly frees the sgl[] and * (if present) info[] arrays, plus the transaction itself. @@ -581,13 +598,14 @@ static void __gsi_trans_commit(struct gsi_trans *trans, bool ring_db) if (channel->toward_ipa) gsi_trans_tx_committed(trans); - gsi_trans_move_pending(trans); + gsi_trans_move_committed(trans); /* Ring doorbell if requested, or if all TREs are allocated */ if (ring_db || !atomic_read(&channel->trans_info.tre_avail)) { /* Report what we're handing off to hardware for TX channels */ if (channel->toward_ipa) gsi_trans_tx_queued(trans); + gsi_trans_move_pending(trans); gsi_channel_doorbell(channel); } } @@ -692,6 +710,7 @@ void gsi_trans_read_byte_done(struct gsi *gsi, u32 channel_id) int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id) { struct gsi_channel *channel = &gsi->channel[channel_id]; + u32 tre_count = channel->tre_count; struct gsi_trans_info *trans_info; u32 tre_max; int ret; @@ -699,30 +718,40 @@ int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id) /* Ensure the size of a channel element is what's expected */ BUILD_BUG_ON(sizeof(struct gsi_tre) != GSI_RING_ELEMENT_SIZE); - /* The map array is used to determine what transaction is associated - * with a TRE that the hardware reports has completed. We need one - * map entry per TRE. - */ trans_info = &channel->trans_info; - trans_info->map = kcalloc(channel->tre_count, sizeof(*trans_info->map), - GFP_KERNEL); - if (!trans_info->map) - return -ENOMEM; - /* We can't use more TREs than there are available in the ring. - * This limits the number of transactions that can be oustanding. - * Worst case is one TRE per transaction (but we actually limit - * it to something a little less than that). We allocate resources - * for transactions (including transaction structures) based on - * this maximum number. + /* The tre_avail field is what ultimately limits the number of + * outstanding transactions and their resources. A transaction + * allocation succeeds only if the TREs available are sufficient + * for what the transaction might need. */ tre_max = gsi_channel_tre_max(channel->gsi, channel_id); + atomic_set(&trans_info->tre_avail, tre_max); - /* Transactions are allocated one at a time. */ + /* We can't use more TREs than the number available in the ring. + * This limits the number of transactions that can be outstanding. + * Worst case is one TRE per transaction (but we actually limit + * it to something a little less than that). By allocating a + * power-of-two number of transactions we can use an index + * modulo that number to determine the next one that's free. + * Transactions are allocated one at a time. + */ ret = gsi_trans_pool_init(&trans_info->pool, sizeof(struct gsi_trans), tre_max, 1); if (ret) - goto err_kfree; + return -ENOMEM; + + /* A completion event contains a pointer to the TRE that caused + * the event (which will be the last one used by the transaction). + * Each entry in this map records the transaction associated + * with a corresponding completed TRE. + */ + trans_info->map = kcalloc(tre_count, sizeof(*trans_info->map), + GFP_KERNEL); + if (!trans_info->map) { + ret = -ENOMEM; + goto err_trans_free; + } /* A transaction uses a scatterlist array to represent the data * transfers implemented by the transaction. Each scatterlist @@ -734,29 +763,21 @@ int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id) sizeof(struct scatterlist), tre_max, channel->trans_tre_max); if (ret) - goto err_trans_pool_exit; - - /* Finally, the tre_avail field is what ultimately limits the number - * of outstanding transactions and their resources. A transaction - * allocation succeeds only if the TREs available are sufficient for - * what the transaction might need. Transaction resource pools are - * sized based on the maximum number of outstanding TREs, so there - * will always be resources available if there are TREs available. - */ - atomic_set(&trans_info->tre_avail, tre_max); + goto err_map_free; spin_lock_init(&trans_info->spinlock); INIT_LIST_HEAD(&trans_info->alloc); + INIT_LIST_HEAD(&trans_info->committed); INIT_LIST_HEAD(&trans_info->pending); INIT_LIST_HEAD(&trans_info->complete); INIT_LIST_HEAD(&trans_info->polled); return 0; -err_trans_pool_exit: - gsi_trans_pool_exit(&trans_info->pool); -err_kfree: +err_map_free: kfree(trans_info->map); +err_trans_free: + gsi_trans_pool_exit(&trans_info->pool); dev_err(gsi->dev, "error %d initializing channel %u transactions\n", ret, channel_id); diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c index 3757ce3de2c5..32962d885acd 100644 --- a/drivers/net/ipa/ipa_main.c +++ b/drivers/net/ipa/ipa_main.c @@ -836,6 +836,8 @@ out_power_put: kfree(ipa); ipa_power_exit(power); + dev_info(dev, "IPA driver removed"); + return 0; } @@ -851,6 +853,7 @@ static void ipa_shutdown(struct platform_device *pdev) static const struct attribute_group *ipa_attribute_groups[] = { &ipa_attribute_group, &ipa_feature_attribute_group, + &ipa_endpoint_id_attribute_group, &ipa_modem_attribute_group, NULL, }; diff --git a/drivers/net/ipa/ipa_sysfs.c b/drivers/net/ipa/ipa_sysfs.c index ff61dbdd70d8..c0c8641cdd14 100644 --- a/drivers/net/ipa/ipa_sysfs.c +++ b/drivers/net/ipa/ipa_sysfs.c @@ -96,38 +96,71 @@ const struct attribute_group ipa_feature_attribute_group = { .attrs = ipa_feature_attrs, }; -static ssize_t -ipa_endpoint_id_show(struct ipa *ipa, char *buf, enum ipa_endpoint_name name) +static umode_t ipa_endpoint_id_is_visible(struct kobject *kobj, + struct attribute *attr, int n) { - u32 endpoint_id = ipa->name_map[name]->endpoint_id; + struct ipa *ipa = dev_get_drvdata(kobj_to_dev(kobj)); + struct device_attribute *dev_attr; + struct dev_ext_attribute *ea; + bool visible; + + /* An endpoint id attribute is only visible if it's defined */ + dev_attr = container_of(attr, struct device_attribute, attr); + ea = container_of(dev_attr, struct dev_ext_attribute, attr); - return scnprintf(buf, PAGE_SIZE, "%u\n", endpoint_id); + visible = !!ipa->name_map[(enum ipa_endpoint_name)(uintptr_t)ea->var]; + + return visible ? attr->mode : 0; } -static ssize_t rx_endpoint_id_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t endpoint_id_attr_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct ipa *ipa = dev_get_drvdata(dev); + struct ipa_endpoint *endpoint; + struct dev_ext_attribute *ea; + + ea = container_of(attr, struct dev_ext_attribute, attr); + endpoint = ipa->name_map[(enum ipa_endpoint_name)(uintptr_t)ea->var]; - return ipa_endpoint_id_show(ipa, buf, IPA_ENDPOINT_AP_MODEM_RX); + return sysfs_emit(buf, "%u\n", endpoint->endpoint_id); } -static DEVICE_ATTR_RO(rx_endpoint_id); +#define ENDPOINT_ID_ATTR(_n, _endpoint_name) \ + static struct dev_ext_attribute dev_attr_endpoint_id_ ## _n = { \ + .attr = __ATTR(_n, 0444, endpoint_id_attr_show, NULL), \ + .var = (void *)(_endpoint_name), \ + } -static ssize_t tx_endpoint_id_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct ipa *ipa = dev_get_drvdata(dev); +ENDPOINT_ID_ATTR(modem_rx, IPA_ENDPOINT_AP_MODEM_RX); +ENDPOINT_ID_ATTR(modem_tx, IPA_ENDPOINT_AP_MODEM_TX); - return ipa_endpoint_id_show(ipa, buf, IPA_ENDPOINT_AP_MODEM_TX); -} +static struct attribute *ipa_endpoint_id_attrs[] = { + &dev_attr_endpoint_id_modem_rx.attr.attr, + &dev_attr_endpoint_id_modem_tx.attr.attr, + NULL +}; -static DEVICE_ATTR_RO(tx_endpoint_id); +const struct attribute_group ipa_endpoint_id_attribute_group = { + .name = "endpoint_id", + .is_visible = ipa_endpoint_id_is_visible, + .attrs = ipa_endpoint_id_attrs, +}; + +/* Reuse endpoint ID attributes for the legacy modem endpoint IDs */ +#define MODEM_ATTR(_n, _endpoint_name) \ + static struct dev_ext_attribute dev_attr_modem_ ## _n = { \ + .attr = __ATTR(_n, 0444, endpoint_id_attr_show, NULL), \ + .var = (void *)(_endpoint_name), \ + } + +MODEM_ATTR(rx_endpoint_id, IPA_ENDPOINT_AP_MODEM_RX); +MODEM_ATTR(tx_endpoint_id, IPA_ENDPOINT_AP_MODEM_TX); static struct attribute *ipa_modem_attrs[] = { - &dev_attr_rx_endpoint_id.attr, - &dev_attr_tx_endpoint_id.attr, - NULL + &dev_attr_modem_rx_endpoint_id.attr.attr, + &dev_attr_modem_tx_endpoint_id.attr.attr, + NULL, }; const struct attribute_group ipa_modem_attribute_group = { diff --git a/drivers/net/ipa/ipa_sysfs.h b/drivers/net/ipa/ipa_sysfs.h index b34e5650bf8c..4a3ffd1e4e3f 100644 --- a/drivers/net/ipa/ipa_sysfs.h +++ b/drivers/net/ipa/ipa_sysfs.h @@ -10,6 +10,7 @@ struct attribute_group; extern const struct attribute_group ipa_attribute_group; extern const struct attribute_group ipa_feature_attribute_group; +extern const struct attribute_group ipa_endpoint_id_attribute_group; extern const struct attribute_group ipa_modem_attribute_group; #endif /* _IPA_SYSFS_H_ */ diff --git a/drivers/net/netdevsim/bus.c b/drivers/net/netdevsim/bus.c index 25cb2e600d53..b5f4df1a07a3 100644 --- a/drivers/net/netdevsim/bus.c +++ b/drivers/net/netdevsim/bus.c @@ -72,16 +72,7 @@ new_port_store(struct device *dev, struct device_attribute *attr, if (ret) return ret; - if (!mutex_trylock(&nsim_bus_dev->nsim_bus_reload_lock)) - return -EBUSY; - - if (nsim_bus_dev->in_reload) { - mutex_unlock(&nsim_bus_dev->nsim_bus_reload_lock); - return -EBUSY; - } - ret = nsim_drv_port_add(nsim_bus_dev, NSIM_DEV_PORT_TYPE_PF, port_index); - mutex_unlock(&nsim_bus_dev->nsim_bus_reload_lock); return ret ? ret : count; } @@ -102,16 +93,7 @@ del_port_store(struct device *dev, struct device_attribute *attr, if (ret) return ret; - if (!mutex_trylock(&nsim_bus_dev->nsim_bus_reload_lock)) - return -EBUSY; - - if (nsim_bus_dev->in_reload) { - mutex_unlock(&nsim_bus_dev->nsim_bus_reload_lock); - return -EBUSY; - } - ret = nsim_drv_port_del(nsim_bus_dev, NSIM_DEV_PORT_TYPE_PF, port_index); - mutex_unlock(&nsim_bus_dev->nsim_bus_reload_lock); return ret ? ret : count; } @@ -298,7 +280,6 @@ nsim_bus_dev_new(unsigned int id, unsigned int port_count, unsigned int num_queu nsim_bus_dev->num_queues = num_queues; nsim_bus_dev->initial_net = current->nsproxy->net_ns; nsim_bus_dev->max_vfs = NSIM_BUS_DEV_MAX_VFS; - mutex_init(&nsim_bus_dev->nsim_bus_reload_lock); /* Disallow using nsim_bus_dev */ smp_store_release(&nsim_bus_dev->init, false); diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c index 57a3ac893792..5802e80e8fe1 100644 --- a/drivers/net/netdevsim/dev.c +++ b/drivers/net/netdevsim/dev.c @@ -436,62 +436,62 @@ static int nsim_dev_resources_register(struct devlink *devlink) int err; /* Resources for IPv4 */ - err = devlink_resource_register(devlink, "IPv4", (u64)-1, - NSIM_RESOURCE_IPV4, - DEVLINK_RESOURCE_ID_PARENT_TOP, - ¶ms); + err = devl_resource_register(devlink, "IPv4", (u64)-1, + NSIM_RESOURCE_IPV4, + DEVLINK_RESOURCE_ID_PARENT_TOP, + ¶ms); if (err) { pr_err("Failed to register IPv4 top resource\n"); goto out; } - err = devlink_resource_register(devlink, "fib", (u64)-1, - NSIM_RESOURCE_IPV4_FIB, - NSIM_RESOURCE_IPV4, ¶ms); + err = devl_resource_register(devlink, "fib", (u64)-1, + NSIM_RESOURCE_IPV4_FIB, + NSIM_RESOURCE_IPV4, ¶ms); if (err) { pr_err("Failed to register IPv4 FIB resource\n"); return err; } - err = devlink_resource_register(devlink, "fib-rules", (u64)-1, - NSIM_RESOURCE_IPV4_FIB_RULES, - NSIM_RESOURCE_IPV4, ¶ms); + err = devl_resource_register(devlink, "fib-rules", (u64)-1, + NSIM_RESOURCE_IPV4_FIB_RULES, + NSIM_RESOURCE_IPV4, ¶ms); if (err) { pr_err("Failed to register IPv4 FIB rules resource\n"); return err; } /* Resources for IPv6 */ - err = devlink_resource_register(devlink, "IPv6", (u64)-1, - NSIM_RESOURCE_IPV6, - DEVLINK_RESOURCE_ID_PARENT_TOP, - ¶ms); + err = devl_resource_register(devlink, "IPv6", (u64)-1, + NSIM_RESOURCE_IPV6, + DEVLINK_RESOURCE_ID_PARENT_TOP, + ¶ms); if (err) { pr_err("Failed to register IPv6 top resource\n"); goto out; } - err = devlink_resource_register(devlink, "fib", (u64)-1, - NSIM_RESOURCE_IPV6_FIB, - NSIM_RESOURCE_IPV6, ¶ms); + err = devl_resource_register(devlink, "fib", (u64)-1, + NSIM_RESOURCE_IPV6_FIB, + NSIM_RESOURCE_IPV6, ¶ms); if (err) { pr_err("Failed to register IPv6 FIB resource\n"); return err; } - err = devlink_resource_register(devlink, "fib-rules", (u64)-1, - NSIM_RESOURCE_IPV6_FIB_RULES, - NSIM_RESOURCE_IPV6, ¶ms); + err = devl_resource_register(devlink, "fib-rules", (u64)-1, + NSIM_RESOURCE_IPV6_FIB_RULES, + NSIM_RESOURCE_IPV6, ¶ms); if (err) { pr_err("Failed to register IPv6 FIB rules resource\n"); return err; } /* Resources for nexthops */ - err = devlink_resource_register(devlink, "nexthops", (u64)-1, - NSIM_RESOURCE_NEXTHOPS, - DEVLINK_RESOURCE_ID_PARENT_TOP, - ¶ms); + err = devl_resource_register(devlink, "nexthops", (u64)-1, + NSIM_RESOURCE_NEXTHOPS, + DEVLINK_RESOURCE_ID_PARENT_TOP, + ¶ms); out: return err; @@ -557,15 +557,15 @@ static int nsim_dev_dummy_region_init(struct nsim_dev *nsim_dev, struct devlink *devlink) { nsim_dev->dummy_region = - devlink_region_create(devlink, &dummy_region_ops, - NSIM_DEV_DUMMY_REGION_SNAPSHOT_MAX, - NSIM_DEV_DUMMY_REGION_SIZE); + devl_region_create(devlink, &dummy_region_ops, + NSIM_DEV_DUMMY_REGION_SNAPSHOT_MAX, + NSIM_DEV_DUMMY_REGION_SIZE); return PTR_ERR_OR_ZERO(nsim_dev->dummy_region); } static void nsim_dev_dummy_region_exit(struct nsim_dev *nsim_dev) { - devlink_region_destroy(nsim_dev->dummy_region); + devl_region_destroy(nsim_dev->dummy_region); } static int @@ -832,7 +832,11 @@ static void nsim_dev_trap_report_work(struct work_struct *work) /* For each running port and enabled packet trap, generate a UDP * packet with a random 5-tuple and report it. */ - devl_lock(priv_to_devlink(nsim_dev)); + if (!devl_trylock(priv_to_devlink(nsim_dev))) { + schedule_delayed_work(&nsim_dev->trap_data->trap_report_dw, 0); + return; + } + list_for_each_entry(nsim_dev_port, &nsim_dev->port_list, list) { if (!netif_running(nsim_dev_port->ns->netdev)) continue; @@ -880,18 +884,18 @@ static int nsim_dev_traps_init(struct devlink *devlink) nsim_trap_data->nsim_dev = nsim_dev; nsim_dev->trap_data = nsim_trap_data; - err = devlink_trap_policers_register(devlink, nsim_trap_policers_arr, - policers_count); + err = devl_trap_policers_register(devlink, nsim_trap_policers_arr, + policers_count); if (err) goto err_trap_policers_cnt_free; - err = devlink_trap_groups_register(devlink, nsim_trap_groups_arr, - ARRAY_SIZE(nsim_trap_groups_arr)); + err = devl_trap_groups_register(devlink, nsim_trap_groups_arr, + ARRAY_SIZE(nsim_trap_groups_arr)); if (err) goto err_trap_policers_unregister; - err = devlink_traps_register(devlink, nsim_traps_arr, - ARRAY_SIZE(nsim_traps_arr), NULL); + err = devl_traps_register(devlink, nsim_traps_arr, + ARRAY_SIZE(nsim_traps_arr), NULL); if (err) goto err_trap_groups_unregister; @@ -903,11 +907,11 @@ static int nsim_dev_traps_init(struct devlink *devlink) return 0; err_trap_groups_unregister: - devlink_trap_groups_unregister(devlink, nsim_trap_groups_arr, - ARRAY_SIZE(nsim_trap_groups_arr)); + devl_trap_groups_unregister(devlink, nsim_trap_groups_arr, + ARRAY_SIZE(nsim_trap_groups_arr)); err_trap_policers_unregister: - devlink_trap_policers_unregister(devlink, nsim_trap_policers_arr, - ARRAY_SIZE(nsim_trap_policers_arr)); + devl_trap_policers_unregister(devlink, nsim_trap_policers_arr, + ARRAY_SIZE(nsim_trap_policers_arr)); err_trap_policers_cnt_free: kfree(nsim_trap_data->trap_policers_cnt_arr); err_trap_items_free: @@ -923,12 +927,12 @@ static void nsim_dev_traps_exit(struct devlink *devlink) /* caution, trap work takes devlink lock */ cancel_delayed_work_sync(&nsim_dev->trap_data->trap_report_dw); - devlink_traps_unregister(devlink, nsim_traps_arr, - ARRAY_SIZE(nsim_traps_arr)); - devlink_trap_groups_unregister(devlink, nsim_trap_groups_arr, - ARRAY_SIZE(nsim_trap_groups_arr)); - devlink_trap_policers_unregister(devlink, nsim_trap_policers_arr, - ARRAY_SIZE(nsim_trap_policers_arr)); + devl_traps_unregister(devlink, nsim_traps_arr, + ARRAY_SIZE(nsim_traps_arr)); + devl_trap_groups_unregister(devlink, nsim_trap_groups_arr, + ARRAY_SIZE(nsim_trap_groups_arr)); + devl_trap_policers_unregister(devlink, nsim_trap_policers_arr, + ARRAY_SIZE(nsim_trap_policers_arr)); kfree(nsim_dev->trap_data->trap_policers_cnt_arr); kfree(nsim_dev->trap_data->trap_items_arr); kfree(nsim_dev->trap_data); @@ -943,24 +947,19 @@ static int nsim_dev_reload_down(struct devlink *devlink, bool netns_change, struct netlink_ext_ack *extack) { struct nsim_dev *nsim_dev = devlink_priv(devlink); - struct nsim_bus_dev *nsim_bus_dev; - - nsim_bus_dev = nsim_dev->nsim_bus_dev; - if (!mutex_trylock(&nsim_bus_dev->nsim_bus_reload_lock)) - return -EOPNOTSUPP; + devl_lock(devlink); if (nsim_dev->dont_allow_reload) { /* For testing purposes, user set debugfs dont_allow_reload * value to true. So forbid it. */ NL_SET_ERR_MSG_MOD(extack, "User forbid the reload for testing purposes"); - mutex_unlock(&nsim_bus_dev->nsim_bus_reload_lock); + devl_unlock(devlink); return -EOPNOTSUPP; } - nsim_bus_dev->in_reload = true; nsim_dev_reload_destroy(nsim_dev); - mutex_unlock(&nsim_bus_dev->nsim_bus_reload_lock); + devl_unlock(devlink); return 0; } @@ -969,25 +968,21 @@ static int nsim_dev_reload_up(struct devlink *devlink, enum devlink_reload_actio struct netlink_ext_ack *extack) { struct nsim_dev *nsim_dev = devlink_priv(devlink); - struct nsim_bus_dev *nsim_bus_dev; int ret; - nsim_bus_dev = nsim_dev->nsim_bus_dev; - mutex_lock(&nsim_bus_dev->nsim_bus_reload_lock); - nsim_bus_dev->in_reload = false; - + devl_lock(devlink); if (nsim_dev->fail_reload) { /* For testing purposes, user set debugfs fail_reload * value to true. Fail right away. */ NL_SET_ERR_MSG_MOD(extack, "User setup the reload to fail for testing purposes"); - mutex_unlock(&nsim_bus_dev->nsim_bus_reload_lock); + devl_unlock(devlink); return -EINVAL; } *actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT); ret = nsim_dev_reload_create(nsim_dev, extack); - mutex_unlock(&nsim_bus_dev->nsim_bus_reload_lock); + devl_unlock(devlink); return ret; } @@ -1434,11 +1429,9 @@ static void nsim_dev_port_del_all(struct nsim_dev *nsim_dev) { struct nsim_dev_port *nsim_dev_port, *tmp; - devl_lock(priv_to_devlink(nsim_dev)); list_for_each_entry_safe(nsim_dev_port, tmp, &nsim_dev->port_list, list) __nsim_dev_port_del(nsim_dev_port); - devl_unlock(priv_to_devlink(nsim_dev)); } static int nsim_dev_port_add_all(struct nsim_dev *nsim_dev, @@ -1447,9 +1440,7 @@ static int nsim_dev_port_add_all(struct nsim_dev *nsim_dev, int i, err; for (i = 0; i < port_count; i++) { - devl_lock(priv_to_devlink(nsim_dev)); err = __nsim_dev_port_add(nsim_dev, NSIM_DEV_PORT_TYPE_PF, i); - devl_unlock(priv_to_devlink(nsim_dev)); if (err) goto err_port_del_all; } @@ -1537,6 +1528,7 @@ int nsim_drv_probe(struct nsim_bus_dev *nsim_bus_dev) nsim_bus_dev->initial_net, &nsim_bus_dev->dev); if (!devlink) return -ENOMEM; + devl_lock(devlink); nsim_dev = devlink_priv(devlink); nsim_dev->nsim_bus_dev = nsim_bus_dev; nsim_dev->switch_id.id_len = sizeof(nsim_dev->switch_id.id); @@ -1555,7 +1547,7 @@ int nsim_drv_probe(struct nsim_bus_dev *nsim_bus_dev) GFP_KERNEL | __GFP_NOWARN); if (!nsim_dev->vfconfigs) { err = -ENOMEM; - goto err_devlink_free; + goto err_devlink_unlock; } err = nsim_dev_resources_register(devlink); @@ -1608,6 +1600,7 @@ int nsim_drv_probe(struct nsim_bus_dev *nsim_bus_dev) nsim_dev->esw_mode = DEVLINK_ESWITCH_MODE_LEGACY; devlink_set_features(devlink, DEVLINK_F_RELOAD); + devl_unlock(devlink); devlink_register(devlink); return 0; @@ -1631,10 +1624,11 @@ err_params_unregister: devlink_params_unregister(devlink, nsim_devlink_params, ARRAY_SIZE(nsim_devlink_params)); err_dl_unregister: - devlink_resources_unregister(devlink); + devl_resources_unregister(devlink); err_vfc_free: kfree(nsim_dev->vfconfigs); -err_devlink_free: +err_devlink_unlock: + devl_unlock(devlink); devlink_free(devlink); dev_set_drvdata(&nsim_bus_dev->dev, NULL); return err; @@ -1648,13 +1642,11 @@ static void nsim_dev_reload_destroy(struct nsim_dev *nsim_dev) return; debugfs_remove(nsim_dev->take_snapshot); - devl_lock(devlink); if (nsim_dev_get_vfs(nsim_dev)) { nsim_bus_dev_set_vfs(nsim_dev->nsim_bus_dev, 0); if (nsim_esw_mode_is_switchdev(nsim_dev)) nsim_esw_legacy_enable(nsim_dev, NULL); } - devl_unlock(devlink); nsim_dev_port_del_all(nsim_dev); nsim_dev_hwstats_exit(nsim_dev); @@ -1671,14 +1663,16 @@ void nsim_drv_remove(struct nsim_bus_dev *nsim_bus_dev) struct devlink *devlink = priv_to_devlink(nsim_dev); devlink_unregister(devlink); + devl_lock(devlink); nsim_dev_reload_destroy(nsim_dev); nsim_bpf_dev_exit(nsim_dev); nsim_dev_debugfs_exit(nsim_dev); devlink_params_unregister(devlink, nsim_devlink_params, ARRAY_SIZE(nsim_devlink_params)); - devlink_resources_unregister(devlink); + devl_resources_unregister(devlink); kfree(nsim_dev->vfconfigs); + devl_unlock(devlink); devlink_free(devlink); dev_set_drvdata(&nsim_bus_dev->dev, NULL); } diff --git a/drivers/net/netdevsim/fib.c b/drivers/net/netdevsim/fib.c index c8f398f5bc5b..94e7512bef94 100644 --- a/drivers/net/netdevsim/fib.c +++ b/drivers/net/netdevsim/fib.c @@ -1453,7 +1453,7 @@ static void nsim_fib_set_max_all(struct nsim_fib_data *data, int err; u64 val; - err = devlink_resource_size_get(devlink, res_ids[i], &val); + err = devl_resource_size_get(devlink, res_ids[i], &val); if (err) val = (u64) -1; nsim_fib_set_max(data, res_ids[i], val); @@ -1562,26 +1562,26 @@ struct nsim_fib_data *nsim_fib_create(struct devlink *devlink, goto err_nexthop_nb_unregister; } - devlink_resource_occ_get_register(devlink, - NSIM_RESOURCE_IPV4_FIB, - nsim_fib_ipv4_resource_occ_get, - data); - devlink_resource_occ_get_register(devlink, - NSIM_RESOURCE_IPV4_FIB_RULES, - nsim_fib_ipv4_rules_res_occ_get, - data); - devlink_resource_occ_get_register(devlink, - NSIM_RESOURCE_IPV6_FIB, - nsim_fib_ipv6_resource_occ_get, - data); - devlink_resource_occ_get_register(devlink, - NSIM_RESOURCE_IPV6_FIB_RULES, - nsim_fib_ipv6_rules_res_occ_get, - data); - devlink_resource_occ_get_register(devlink, - NSIM_RESOURCE_NEXTHOPS, - nsim_fib_nexthops_res_occ_get, - data); + devl_resource_occ_get_register(devlink, + NSIM_RESOURCE_IPV4_FIB, + nsim_fib_ipv4_resource_occ_get, + data); + devl_resource_occ_get_register(devlink, + NSIM_RESOURCE_IPV4_FIB_RULES, + nsim_fib_ipv4_rules_res_occ_get, + data); + devl_resource_occ_get_register(devlink, + NSIM_RESOURCE_IPV6_FIB, + nsim_fib_ipv6_resource_occ_get, + data); + devl_resource_occ_get_register(devlink, + NSIM_RESOURCE_IPV6_FIB_RULES, + nsim_fib_ipv6_rules_res_occ_get, + data); + devl_resource_occ_get_register(devlink, + NSIM_RESOURCE_NEXTHOPS, + nsim_fib_nexthops_res_occ_get, + data); return data; err_nexthop_nb_unregister: @@ -1604,16 +1604,16 @@ err_data_free: void nsim_fib_destroy(struct devlink *devlink, struct nsim_fib_data *data) { - devlink_resource_occ_get_unregister(devlink, - NSIM_RESOURCE_NEXTHOPS); - devlink_resource_occ_get_unregister(devlink, - NSIM_RESOURCE_IPV6_FIB_RULES); - devlink_resource_occ_get_unregister(devlink, - NSIM_RESOURCE_IPV6_FIB); - devlink_resource_occ_get_unregister(devlink, - NSIM_RESOURCE_IPV4_FIB_RULES); - devlink_resource_occ_get_unregister(devlink, - NSIM_RESOURCE_IPV4_FIB); + devl_resource_occ_get_unregister(devlink, + NSIM_RESOURCE_NEXTHOPS); + devl_resource_occ_get_unregister(devlink, + NSIM_RESOURCE_IPV6_FIB_RULES); + devl_resource_occ_get_unregister(devlink, + NSIM_RESOURCE_IPV6_FIB); + devl_resource_occ_get_unregister(devlink, + NSIM_RESOURCE_IPV4_FIB_RULES); + devl_resource_occ_get_unregister(devlink, + NSIM_RESOURCE_IPV4_FIB); unregister_fib_notifier(devlink_net(devlink), &data->fib_nb); unregister_nexthop_notifier(devlink_net(devlink), &data->nexthop_nb); flush_work(&data->fib_event_work); diff --git a/drivers/net/netdevsim/netdevsim.h b/drivers/net/netdevsim/netdevsim.h index 0b122872b2c9..7d8ed8d8df5c 100644 --- a/drivers/net/netdevsim/netdevsim.h +++ b/drivers/net/netdevsim/netdevsim.h @@ -376,9 +376,6 @@ struct nsim_bus_dev { */ unsigned int max_vfs; unsigned int num_vfs; - /* Lock for devlink->reload_enabled in netdevsim module */ - struct mutex nsim_bus_reload_lock; - bool in_reload; bool init; }; diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c index ac2d400d1d6c..3b0e8f93e3c8 100644 --- a/drivers/net/usb/ax88179_178a.c +++ b/drivers/net/usb/ax88179_178a.c @@ -164,11 +164,15 @@ #define GMII_PHY_PGSEL_PAGE3 0x0003 #define GMII_PHY_PGSEL_PAGE5 0x0005 +static int ax88179_reset(struct usbnet *dev); + struct ax88179_data { u8 eee_enabled; u8 eee_active; u16 rxctl; - u16 reserved; + u8 in_pm; + u32 wol_supported; + u32 wolopts; }; struct ax88179_int_data { @@ -185,15 +189,29 @@ static const struct { {7, 0xcc, 0x4c, 0x18, 8}, }; +static void ax88179_set_pm_mode(struct usbnet *dev, bool pm_mode) +{ + struct ax88179_data *ax179_data = dev->driver_priv; + + ax179_data->in_pm = pm_mode; +} + +static int ax88179_in_pm(struct usbnet *dev) +{ + struct ax88179_data *ax179_data = dev->driver_priv; + + return ax179_data->in_pm; +} + static int __ax88179_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index, - u16 size, void *data, int in_pm) + u16 size, void *data) { int ret; int (*fn)(struct usbnet *, u8, u8, u16, u16, void *, u16); BUG_ON(!dev); - if (!in_pm) + if (!ax88179_in_pm(dev)) fn = usbnet_read_cmd; else fn = usbnet_read_cmd_nopm; @@ -209,14 +227,14 @@ static int __ax88179_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index, } static int __ax88179_write_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index, - u16 size, const void *data, int in_pm) + u16 size, const void *data) { int ret; int (*fn)(struct usbnet *, u8, u8, u16, u16, const void *, u16); BUG_ON(!dev); - if (!in_pm) + if (!ax88179_in_pm(dev)) fn = usbnet_write_cmd; else fn = usbnet_write_cmd_nopm; @@ -249,47 +267,6 @@ static void ax88179_write_cmd_async(struct usbnet *dev, u8 cmd, u16 value, } } -static int ax88179_read_cmd_nopm(struct usbnet *dev, u8 cmd, u16 value, - u16 index, u16 size, void *data) -{ - int ret; - - if (2 == size) { - u16 buf; - ret = __ax88179_read_cmd(dev, cmd, value, index, size, &buf, 1); - le16_to_cpus(&buf); - *((u16 *)data) = buf; - } else if (4 == size) { - u32 buf; - ret = __ax88179_read_cmd(dev, cmd, value, index, size, &buf, 1); - le32_to_cpus(&buf); - *((u32 *)data) = buf; - } else { - ret = __ax88179_read_cmd(dev, cmd, value, index, size, data, 1); - } - - return ret; -} - -static int ax88179_write_cmd_nopm(struct usbnet *dev, u8 cmd, u16 value, - u16 index, u16 size, const void *data) -{ - int ret; - - if (2 == size) { - u16 buf; - buf = *((u16 *)data); - cpu_to_le16s(&buf); - ret = __ax88179_write_cmd(dev, cmd, value, index, - size, &buf, 1); - } else { - ret = __ax88179_write_cmd(dev, cmd, value, index, - size, data, 1); - } - - return ret; -} - static int ax88179_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index, u16 size, void *data) { @@ -297,16 +274,16 @@ static int ax88179_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index, if (2 == size) { u16 buf = 0; - ret = __ax88179_read_cmd(dev, cmd, value, index, size, &buf, 0); + ret = __ax88179_read_cmd(dev, cmd, value, index, size, &buf); le16_to_cpus(&buf); *((u16 *)data) = buf; } else if (4 == size) { u32 buf = 0; - ret = __ax88179_read_cmd(dev, cmd, value, index, size, &buf, 0); + ret = __ax88179_read_cmd(dev, cmd, value, index, size, &buf); le32_to_cpus(&buf); *((u32 *)data) = buf; } else { - ret = __ax88179_read_cmd(dev, cmd, value, index, size, data, 0); + ret = __ax88179_read_cmd(dev, cmd, value, index, size, data); } return ret; @@ -322,10 +299,10 @@ static int ax88179_write_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index, buf = *((u16 *)data); cpu_to_le16s(&buf); ret = __ax88179_write_cmd(dev, cmd, value, index, - size, &buf, 0); + size, &buf); } else { ret = __ax88179_write_cmd(dev, cmd, value, index, - size, data, 0); + size, data); } return ret; @@ -425,55 +402,63 @@ ax88179_phy_write_mmd_indirect(struct usbnet *dev, u16 prtad, u16 devad, static int ax88179_suspend(struct usb_interface *intf, pm_message_t message) { struct usbnet *dev = usb_get_intfdata(intf); + struct ax88179_data *priv = dev->driver_priv; u16 tmp16; u8 tmp8; + ax88179_set_pm_mode(dev, true); + usbnet_suspend(intf, message); + /* Enable WoL */ + if (priv->wolopts) { + ax88179_read_cmd(dev, AX_ACCESS_MAC, AX_MONITOR_MOD, + 1, 1, &tmp8); + if (priv->wolopts & WAKE_PHY) + tmp8 |= AX_MONITOR_MODE_RWLC; + if (priv->wolopts & WAKE_MAGIC) + tmp8 |= AX_MONITOR_MODE_RWMP; + + ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_MONITOR_MOD, + 1, 1, &tmp8); + } + /* Disable RX path */ - ax88179_read_cmd_nopm(dev, AX_ACCESS_MAC, AX_MEDIUM_STATUS_MODE, - 2, 2, &tmp16); + ax88179_read_cmd(dev, AX_ACCESS_MAC, AX_MEDIUM_STATUS_MODE, + 2, 2, &tmp16); tmp16 &= ~AX_MEDIUM_RECEIVE_EN; - ax88179_write_cmd_nopm(dev, AX_ACCESS_MAC, AX_MEDIUM_STATUS_MODE, - 2, 2, &tmp16); + ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_MEDIUM_STATUS_MODE, + 2, 2, &tmp16); /* Force bulk-in zero length */ - ax88179_read_cmd_nopm(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, - 2, 2, &tmp16); + ax88179_read_cmd(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, + 2, 2, &tmp16); tmp16 |= AX_PHYPWR_RSTCTL_BZ | AX_PHYPWR_RSTCTL_IPRL; - ax88179_write_cmd_nopm(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, - 2, 2, &tmp16); + ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, + 2, 2, &tmp16); /* change clock */ tmp8 = 0; - ax88179_write_cmd_nopm(dev, AX_ACCESS_MAC, AX_CLK_SELECT, 1, 1, &tmp8); + ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_CLK_SELECT, 1, 1, &tmp8); /* Configure RX control register => stop operation */ tmp16 = AX_RX_CTL_STOP; - ax88179_write_cmd_nopm(dev, AX_ACCESS_MAC, AX_RX_CTL, 2, 2, &tmp16); + ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_RX_CTL, 2, 2, &tmp16); + + ax88179_set_pm_mode(dev, false); return 0; } /* This function is used to enable the autodetach function. */ /* This function is determined by offset 0x43 of EEPROM */ -static int ax88179_auto_detach(struct usbnet *dev, int in_pm) +static int ax88179_auto_detach(struct usbnet *dev) { u16 tmp16; u8 tmp8; - int (*fnr)(struct usbnet *, u8, u16, u16, u16, void *); - int (*fnw)(struct usbnet *, u8, u16, u16, u16, const void *); - - if (!in_pm) { - fnr = ax88179_read_cmd; - fnw = ax88179_write_cmd; - } else { - fnr = ax88179_read_cmd_nopm; - fnw = ax88179_write_cmd_nopm; - } - if (fnr(dev, AX_ACCESS_EEPROM, 0x43, 1, 2, &tmp16) < 0) + if (ax88179_read_cmd(dev, AX_ACCESS_EEPROM, 0x43, 1, 2, &tmp16) < 0) return 0; if ((tmp16 == 0xFFFF) || (!(tmp16 & 0x0100))) @@ -481,13 +466,13 @@ static int ax88179_auto_detach(struct usbnet *dev, int in_pm) /* Enable Auto Detach bit */ tmp8 = 0; - fnr(dev, AX_ACCESS_MAC, AX_CLK_SELECT, 1, 1, &tmp8); + ax88179_read_cmd(dev, AX_ACCESS_MAC, AX_CLK_SELECT, 1, 1, &tmp8); tmp8 |= AX_CLK_SELECT_ULR; - fnw(dev, AX_ACCESS_MAC, AX_CLK_SELECT, 1, 1, &tmp8); + ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_CLK_SELECT, 1, 1, &tmp8); - fnr(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, 2, 2, &tmp16); + ax88179_read_cmd(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, 2, 2, &tmp16); tmp16 |= AX_PHYPWR_RSTCTL_AT; - fnw(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, 2, 2, &tmp16); + ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, 2, 2, &tmp16); return 0; } @@ -495,35 +480,14 @@ static int ax88179_auto_detach(struct usbnet *dev, int in_pm) static int ax88179_resume(struct usb_interface *intf) { struct usbnet *dev = usb_get_intfdata(intf); - u16 tmp16; - u8 tmp8; - - usbnet_link_change(dev, 0, 0); - /* Power up ethernet PHY */ - tmp16 = 0; - ax88179_write_cmd_nopm(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, - 2, 2, &tmp16); - udelay(1000); - - tmp16 = AX_PHYPWR_RSTCTL_IPRL; - ax88179_write_cmd_nopm(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, - 2, 2, &tmp16); - msleep(200); + ax88179_set_pm_mode(dev, true); - /* Ethernet PHY Auto Detach*/ - ax88179_auto_detach(dev, 1); + usbnet_link_change(dev, 0, 0); - /* Enable clock */ - ax88179_read_cmd_nopm(dev, AX_ACCESS_MAC, AX_CLK_SELECT, 1, 1, &tmp8); - tmp8 |= AX_CLK_SELECT_ACS | AX_CLK_SELECT_BCS; - ax88179_write_cmd_nopm(dev, AX_ACCESS_MAC, AX_CLK_SELECT, 1, 1, &tmp8); - msleep(100); + ax88179_reset(dev); - /* Configure RX control register => start operation */ - tmp16 = AX_RX_CTL_DROPCRCERR | AX_RX_CTL_IPE | AX_RX_CTL_START | - AX_RX_CTL_AP | AX_RX_CTL_AMALL | AX_RX_CTL_AB; - ax88179_write_cmd_nopm(dev, AX_ACCESS_MAC, AX_RX_CTL, 2, 2, &tmp16); + ax88179_set_pm_mode(dev, false); return usbnet_resume(intf); } @@ -532,40 +496,22 @@ static void ax88179_get_wol(struct net_device *net, struct ethtool_wolinfo *wolinfo) { struct usbnet *dev = netdev_priv(net); - u8 opt; + struct ax88179_data *priv = dev->driver_priv; - if (ax88179_read_cmd(dev, AX_ACCESS_MAC, AX_MONITOR_MOD, - 1, 1, &opt) < 0) { - wolinfo->supported = 0; - wolinfo->wolopts = 0; - return; - } - - wolinfo->supported = WAKE_PHY | WAKE_MAGIC; - wolinfo->wolopts = 0; - if (opt & AX_MONITOR_MODE_RWLC) - wolinfo->wolopts |= WAKE_PHY; - if (opt & AX_MONITOR_MODE_RWMP) - wolinfo->wolopts |= WAKE_MAGIC; + wolinfo->supported = priv->wol_supported; + wolinfo->wolopts = priv->wolopts; } static int ax88179_set_wol(struct net_device *net, struct ethtool_wolinfo *wolinfo) { struct usbnet *dev = netdev_priv(net); - u8 opt = 0; + struct ax88179_data *priv = dev->driver_priv; - if (wolinfo->wolopts & ~(WAKE_PHY | WAKE_MAGIC)) + if (wolinfo->wolopts & ~(priv->wol_supported)) return -EINVAL; - if (wolinfo->wolopts & WAKE_PHY) - opt |= AX_MONITOR_MODE_RWLC; - if (wolinfo->wolopts & WAKE_MAGIC) - opt |= AX_MONITOR_MODE_RWMP; - - if (ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_MONITOR_MOD, - 1, 1, &opt) < 0) - return -EINVAL; + priv->wolopts = wolinfo->wolopts; return 0; } @@ -599,8 +545,7 @@ ax88179_get_eeprom(struct net_device *net, struct ethtool_eeprom *eeprom, /* ax88179/178A returns 2 bytes from eeprom on read */ for (i = first_word; i <= last_word; i++) { ret = __ax88179_read_cmd(dev, AX_ACCESS_EEPROM, i, 1, 2, - &eeprom_buff[i - first_word], - 0); + &eeprom_buff[i - first_word]); if (ret < 0) { kfree(eeprom_buff); return -EIO; @@ -745,7 +690,7 @@ ax88179_ethtool_set_eee(struct usbnet *dev, struct ethtool_eee *data) static int ax88179_chk_eee(struct usbnet *dev) { struct ethtool_cmd ecmd = { .cmd = ETHTOOL_GSET }; - struct ax88179_data *priv = (struct ax88179_data *)dev->data; + struct ax88179_data *priv = dev->driver_priv; mii_ethtool_gset(&dev->mii, &ecmd); @@ -848,7 +793,7 @@ static void ax88179_enable_eee(struct usbnet *dev) static int ax88179_get_eee(struct net_device *net, struct ethtool_eee *edata) { struct usbnet *dev = netdev_priv(net); - struct ax88179_data *priv = (struct ax88179_data *)dev->data; + struct ax88179_data *priv = dev->driver_priv; edata->eee_enabled = priv->eee_enabled; edata->eee_active = priv->eee_active; @@ -859,7 +804,7 @@ static int ax88179_get_eee(struct net_device *net, struct ethtool_eee *edata) static int ax88179_set_eee(struct net_device *net, struct ethtool_eee *edata) { struct usbnet *dev = netdev_priv(net); - struct ax88179_data *priv = (struct ax88179_data *)dev->data; + struct ax88179_data *priv = dev->driver_priv; int ret; priv->eee_enabled = edata->eee_enabled; @@ -910,8 +855,8 @@ static const struct ethtool_ops ax88179_ethtool_ops = { static void ax88179_set_multicast(struct net_device *net) { struct usbnet *dev = netdev_priv(net); - struct ax88179_data *data = (struct ax88179_data *)dev->data; - u8 *m_filter = ((u8 *)dev->data) + 12; + struct ax88179_data *data = dev->driver_priv; + u8 *m_filter = ((u8 *)dev->data); data->rxctl = (AX_RX_CTL_START | AX_RX_CTL_AB | AX_RX_CTL_IPE); @@ -923,7 +868,7 @@ static void ax88179_set_multicast(struct net_device *net) } else if (netdev_mc_empty(net)) { /* just broadcast and directed */ } else { - /* We use the 20 byte dev->data for our 8 byte filter buffer + /* We use dev->data for our 8 byte filter buffer * to avoid allocating memory that is tricky to free later */ u32 crc_bits; @@ -1069,7 +1014,7 @@ static int ax88179_check_eeprom(struct usbnet *dev) } while (buf & EEP_BUSY); __ax88179_read_cmd(dev, AX_ACCESS_MAC, AX_SROM_DATA_LOW, - 2, 2, &eeprom[i * 2], 0); + 2, 2, &eeprom[i * 2]); if ((i == 0) && (eeprom[0] == 0xFF)) return -EINVAL; @@ -1322,46 +1267,15 @@ static void ax88179_get_mac_addr(struct usbnet *dev) static int ax88179_bind(struct usbnet *dev, struct usb_interface *intf) { - u8 buf[5]; - u16 *tmp16; - u8 *tmp; - struct ax88179_data *ax179_data = (struct ax88179_data *)dev->data; - struct ethtool_eee eee_data; + struct ax88179_data *ax179_data; usbnet_get_endpoints(dev, intf); - tmp16 = (u16 *)buf; - tmp = (u8 *)buf; - - memset(ax179_data, 0, sizeof(*ax179_data)); - - /* Power up ethernet PHY */ - *tmp16 = 0; - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, 2, 2, tmp16); - *tmp16 = AX_PHYPWR_RSTCTL_IPRL; - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, 2, 2, tmp16); - msleep(200); - - *tmp = AX_CLK_SELECT_ACS | AX_CLK_SELECT_BCS; - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_CLK_SELECT, 1, 1, tmp); - msleep(100); - - /* Read MAC address from DTB or asix chip */ - ax88179_get_mac_addr(dev); - memcpy(dev->net->perm_addr, dev->net->dev_addr, ETH_ALEN); - - /* RX bulk configuration */ - memcpy(tmp, &AX88179_BULKIN_SIZE[0], 5); - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_RX_BULKIN_QCTRL, 5, 5, tmp); - - dev->rx_urb_size = 1024 * 20; + ax179_data = kzalloc(sizeof(*ax179_data), GFP_KERNEL); + if (!ax179_data) + return -ENOMEM; - *tmp = 0x34; - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PAUSE_WATERLVL_LOW, 1, 1, tmp); - - *tmp = 0x52; - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PAUSE_WATERLVL_HIGH, - 1, 1, tmp); + dev->driver_priv = ax179_data; dev->net->netdev_ops = &ax88179_netdev_ops; dev->net->ethtool_ops = &ax88179_ethtool_ops; @@ -1384,52 +1298,14 @@ static int ax88179_bind(struct usbnet *dev, struct usb_interface *intf) netif_set_tso_max_size(dev->net, 16384); - /* Enable checksum offload */ - *tmp = AX_RXCOE_IP | AX_RXCOE_TCP | AX_RXCOE_UDP | - AX_RXCOE_TCPV6 | AX_RXCOE_UDPV6; - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_RXCOE_CTL, 1, 1, tmp); - - *tmp = AX_TXCOE_IP | AX_TXCOE_TCP | AX_TXCOE_UDP | - AX_TXCOE_TCPV6 | AX_TXCOE_UDPV6; - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_TXCOE_CTL, 1, 1, tmp); - - /* Configure RX control register => start operation */ - *tmp16 = AX_RX_CTL_DROPCRCERR | AX_RX_CTL_IPE | AX_RX_CTL_START | - AX_RX_CTL_AP | AX_RX_CTL_AMALL | AX_RX_CTL_AB; - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_RX_CTL, 2, 2, tmp16); - - *tmp = AX_MONITOR_MODE_PMETYPE | AX_MONITOR_MODE_PMEPOL | - AX_MONITOR_MODE_RWMP; - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_MONITOR_MOD, 1, 1, tmp); - - /* Configure default medium type => giga */ - *tmp16 = AX_MEDIUM_RECEIVE_EN | AX_MEDIUM_TXFLOW_CTRLEN | - AX_MEDIUM_RXFLOW_CTRLEN | AX_MEDIUM_FULL_DUPLEX | - AX_MEDIUM_GIGAMODE; - ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_MEDIUM_STATUS_MODE, - 2, 2, tmp16); - - ax88179_led_setting(dev); - - ax179_data->eee_enabled = 0; - ax179_data->eee_active = 0; - - ax88179_disable_eee(dev); - - ax88179_ethtool_get_eee(dev, &eee_data); - eee_data.advertised = 0; - ax88179_ethtool_set_eee(dev, &eee_data); - - /* Restart autoneg */ - mii_nway_restart(&dev->mii); - - usbnet_link_change(dev, 0, 0); + ax88179_reset(dev); return 0; } static void ax88179_unbind(struct usbnet *dev, struct usb_interface *intf) { + struct ax88179_data *ax179_data = dev->driver_priv; u16 tmp16; /* Configure RX control register => stop operation */ @@ -1442,6 +1318,8 @@ static void ax88179_unbind(struct usbnet *dev, struct usb_interface *intf) /* Power down ethernet PHY */ tmp16 = 0; ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PHYPWR_RSTCTL, 2, 2, &tmp16); + + kfree(ax179_data); } static void @@ -1618,7 +1496,7 @@ ax88179_tx_fixup(struct usbnet *dev, struct sk_buff *skb, gfp_t flags) static int ax88179_link_reset(struct usbnet *dev) { - struct ax88179_data *ax179_data = (struct ax88179_data *)dev->data; + struct ax88179_data *ax179_data = dev->driver_priv; u8 tmp[5], link_sts; u16 mode, tmp16, delay = HZ / 10; u32 tmp32 = 0x40000000; @@ -1693,7 +1571,7 @@ static int ax88179_reset(struct usbnet *dev) u8 buf[5]; u16 *tmp16; u8 *tmp; - struct ax88179_data *ax179_data = (struct ax88179_data *)dev->data; + struct ax88179_data *ax179_data = dev->driver_priv; struct ethtool_eee eee_data; tmp16 = (u16 *)buf; @@ -1712,10 +1590,11 @@ static int ax88179_reset(struct usbnet *dev) msleep(100); /* Ethernet PHY Auto Detach*/ - ax88179_auto_detach(dev, 0); + ax88179_auto_detach(dev); /* Read MAC address from DTB or asix chip */ ax88179_get_mac_addr(dev); + memcpy(dev->net->perm_addr, dev->net->dev_addr, ETH_ALEN); /* RX bulk configuration */ memcpy(tmp, &AX88179_BULKIN_SIZE[0], 5); @@ -1730,12 +1609,6 @@ static int ax88179_reset(struct usbnet *dev) ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PAUSE_WATERLVL_HIGH, 1, 1, tmp); - dev->net->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | - NETIF_F_RXCSUM; - - dev->net->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | - NETIF_F_RXCSUM; - /* Enable checksum offload */ *tmp = AX_RXCOE_IP | AX_RXCOE_TCP | AX_RXCOE_UDP | AX_RXCOE_TCPV6 | AX_RXCOE_UDPV6; @@ -1761,6 +1634,12 @@ static int ax88179_reset(struct usbnet *dev) ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_MEDIUM_STATUS_MODE, 2, 2, tmp16); + /* Check if WoL is supported */ + ax179_data->wol_supported = 0; + if (ax88179_read_cmd(dev, AX_ACCESS_MAC, AX_MONITOR_MOD, + 1, 1, &tmp) > 0) + ax179_data->wol_supported = WAKE_MAGIC | WAKE_PHY; + ax88179_led_setting(dev); ax179_data->eee_enabled = 0; diff --git a/drivers/net/usb/cdc_ncm.c b/drivers/net/usb/cdc_ncm.c index d55f59ce4a31..8d5cbda33f66 100644 --- a/drivers/net/usb/cdc_ncm.c +++ b/drivers/net/usb/cdc_ncm.c @@ -1892,7 +1892,7 @@ static void cdc_ncm_status(struct usbnet *dev, struct urb *urb) } static const struct driver_info cdc_ncm_info = { - .description = "CDC NCM", + .description = "CDC NCM (NO ZLP)", .flags = FLAG_POINTTOPOINT | FLAG_NO_SETINT | FLAG_MULTI_PACKET | FLAG_LINK_INTR | FLAG_ETHER, .bind = cdc_ncm_bind, @@ -1904,6 +1904,19 @@ static const struct driver_info cdc_ncm_info = { .set_rx_mode = usbnet_cdc_update_filter, }; +/* Same as cdc_ncm_info, but with FLAG_SEND_ZLP */ +static const struct driver_info cdc_ncm_zlp_info = { + .description = "CDC NCM (SEND ZLP)", + .flags = FLAG_POINTTOPOINT | FLAG_NO_SETINT | FLAG_MULTI_PACKET + | FLAG_LINK_INTR | FLAG_ETHER | FLAG_SEND_ZLP, + .bind = cdc_ncm_bind, + .unbind = cdc_ncm_unbind, + .manage_power = usbnet_manage_power, + .status = cdc_ncm_status, + .rx_fixup = cdc_ncm_rx_fixup, + .tx_fixup = cdc_ncm_tx_fixup, +}; + /* Same as cdc_ncm_info, but with FLAG_WWAN */ static const struct driver_info wwan_info = { .description = "Mobile Broadband Network Device", @@ -2010,6 +2023,16 @@ static const struct usb_device_id cdc_devs[] = { .driver_info = (unsigned long)&wwan_info, }, + /* DisplayLink docking stations */ + { .match_flags = USB_DEVICE_ID_MATCH_INT_INFO + | USB_DEVICE_ID_MATCH_VENDOR, + .idVendor = 0x17e9, + .bInterfaceClass = USB_CLASS_COMM, + .bInterfaceSubClass = USB_CDC_SUBCLASS_NCM, + .bInterfaceProtocol = USB_CDC_PROTO_NONE, + .driver_info = (unsigned long)&cdc_ncm_zlp_info, + }, + /* Generic CDC-NCM devices */ { USB_INTERFACE_INFO(USB_CLASS_COMM, USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE), diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index b082819509e1..0f6efaabaa32 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -32,7 +32,7 @@ #define NETNEXT_VERSION "12" /* Information for net */ -#define NET_VERSION "12" +#define NET_VERSION "13" #define DRIVER_VERSION "v1." NETNEXT_VERSION "." NET_VERSION #define DRIVER_AUTHOR "Realtek linux nic maintainers <nic_swsd@realtek.com>" @@ -5917,7 +5917,8 @@ static void r8153_enter_oob(struct r8152 *tp) wait_oob_link_list_ready(tp); - ocp_write_word(tp, MCU_TYPE_PLA, PLA_RMS, mtu_to_size(tp->netdev->mtu)); + ocp_write_word(tp, MCU_TYPE_PLA, PLA_RMS, 1522); + ocp_write_byte(tp, MCU_TYPE_PLA, PLA_MTPS, MTPS_DEFAULT); switch (tp->version) { case RTL_VER_03: @@ -5953,6 +5954,10 @@ static void r8153_enter_oob(struct r8152 *tp) ocp_data |= NOW_IS_OOB | DIS_MCU_CLROOB; ocp_write_byte(tp, MCU_TYPE_PLA, PLA_OOB_CTRL, ocp_data); + ocp_data = ocp_read_word(tp, MCU_TYPE_PLA, PLA_SFF_STS_7); + ocp_data |= MCU_BORW_EN; + ocp_write_word(tp, MCU_TYPE_PLA, PLA_SFF_STS_7, ocp_data); + rxdy_gated_en(tp, false); ocp_data = ocp_read_dword(tp, MCU_TYPE_PLA, PLA_RCR); @@ -6555,6 +6560,9 @@ static void rtl8156_down(struct r8152 *tp) rtl_disable(tp); rtl_reset_bmu(tp); + ocp_write_word(tp, MCU_TYPE_PLA, PLA_RMS, 1522); + ocp_write_byte(tp, MCU_TYPE_PLA, PLA_MTPS, MTPS_DEFAULT); + /* Clear teredo wake event. bit[15:8] is the teredo wakeup * type. Set it to zero. bits[7:0] are the W1C bits about * the events. Set them to all 1 to clear them. @@ -6565,6 +6573,10 @@ static void rtl8156_down(struct r8152 *tp) ocp_data |= NOW_IS_OOB; ocp_write_byte(tp, MCU_TYPE_PLA, PLA_OOB_CTRL, ocp_data); + ocp_data = ocp_read_word(tp, MCU_TYPE_PLA, PLA_SFF_STS_7); + ocp_data |= MCU_BORW_EN; + ocp_write_word(tp, MCU_TYPE_PLA, PLA_SFF_STS_7, ocp_data); + rtl_rx_vlan_en(tp, true); rxdy_gated_en(tp, false); diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c index 19c414733747..dd831adbc1d1 100644 --- a/drivers/net/vmxnet3/vmxnet3_drv.c +++ b/drivers/net/vmxnet3/vmxnet3_drv.c @@ -1550,6 +1550,7 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, hash_type); } #endif + skb_record_rx_queue(ctx->skb, rq->qid); skb_put(ctx->skb, rcd->len); if (VMXNET3_VERSION_GE_2(adapter) && diff --git a/drivers/net/vmxnet3/vmxnet3_ethtool.c b/drivers/net/vmxnet3/vmxnet3_ethtool.c index c3eaf1b864ed..e2034adc3a1a 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethtool.c +++ b/drivers/net/vmxnet3/vmxnet3_ethtool.c @@ -1292,6 +1292,34 @@ done: return 0; } +static void vmxnet3_get_channels(struct net_device *netdev, + struct ethtool_channels *ec) +{ + struct vmxnet3_adapter *adapter = netdev_priv(netdev); + + if (IS_ENABLED(CONFIG_PCI_MSI) && adapter->intr.type == VMXNET3_IT_MSIX) { + if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) { + ec->combined_count = adapter->num_tx_queues; + } else { + ec->rx_count = adapter->num_rx_queues; + ec->tx_count = + adapter->share_intr == VMXNET3_INTR_TXSHARE ? + 1 : adapter->num_tx_queues; + } + } else { + ec->combined_count = 1; + } + + ec->other_count = 1; + + /* Number of interrupts cannot be changed on the fly */ + /* Just set maximums to actual values */ + ec->max_rx = ec->rx_count; + ec->max_tx = ec->tx_count; + ec->max_combined = ec->combined_count; + ec->max_other = ec->other_count; +} + static const struct ethtool_ops vmxnet3_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS | ETHTOOL_COALESCE_MAX_FRAMES | @@ -1317,6 +1345,7 @@ static const struct ethtool_ops vmxnet3_ethtool_ops = { .set_rxfh = vmxnet3_set_rss, #endif .get_link_ksettings = vmxnet3_get_link_ksettings, + .get_channels = vmxnet3_get_channels, }; void vmxnet3_set_ethtool_ops(struct net_device *netdev) diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index 40445a12c682..5df7a0abc39d 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -1077,7 +1077,7 @@ static int vrf_rtable_create(struct net_device *dev) return -ENOMEM; /* create a dst for routing packets out through a VRF device */ - rth = rt_dst_alloc(dev, 0, RTN_UNICAST, 1, 1); + rth = rt_dst_alloc(dev, 0, RTN_UNICAST, 1); if (!rth) return -ENOMEM; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index ec6ac298d8de..6a12a906a11e 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3786,7 +3786,7 @@ static int nvme_add_ns_cdev(struct nvme_ns *ns) } static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, - unsigned nsid, struct nvme_ns_ids *ids) + unsigned nsid, struct nvme_ns_ids *ids, bool is_shared) { struct nvme_ns_head *head; size_t size = sizeof(*head); @@ -3810,6 +3810,7 @@ static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, head->subsys = ctrl->subsys; head->ns_id = nsid; head->ids = *ids; + head->shared = is_shared; kref_init(&head->ref); if (head->ids.csi) { @@ -3891,12 +3892,11 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, nsid); goto out_unlock; } - head = nvme_alloc_ns_head(ctrl, nsid, ids); + head = nvme_alloc_ns_head(ctrl, nsid, ids, is_shared); if (IS_ERR(head)) { ret = PTR_ERR(head); goto out_unlock; } - head->shared = is_shared; } else { ret = -EINVAL; if (!is_shared || !head->shared) { diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 193b44755662..58c72d55769a 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -2690,8 +2690,13 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) struct pci_dev *pdev = to_pci_dev(dev->dev); mutex_lock(&dev->shutdown_lock); - if (pci_device_is_present(pdev) && pci_is_enabled(pdev)) { - u32 csts = readl(dev->bar + NVME_REG_CSTS); + if (pci_is_enabled(pdev)) { + u32 csts; + + if (pci_device_is_present(pdev)) + csts = readl(dev->bar + NVME_REG_CSTS); + else + csts = ~0; if (dev->ctrl.state == NVME_CTRL_LIVE || dev->ctrl.state == NVME_CTRL_RESETTING) { diff --git a/drivers/platform/x86/amd-pmc.c b/drivers/platform/x86/amd-pmc.c index f11d18beac18..700eb19e8450 100644 --- a/drivers/platform/x86/amd-pmc.c +++ b/drivers/platform/x86/amd-pmc.c @@ -91,6 +91,8 @@ #define AMD_CPU_ID_PCO AMD_CPU_ID_RV #define AMD_CPU_ID_CZN AMD_CPU_ID_RN #define AMD_CPU_ID_YC 0x14B5 +#define AMD_CPU_ID_CB 0x14D8 +#define AMD_CPU_ID_PS 0x14E8 #define PMC_MSG_DELAY_MIN_US 50 #define RESPONSE_REGISTER_LOOP_MAX 20000 @@ -318,6 +320,8 @@ static int amd_pmc_idlemask_read(struct amd_pmc_dev *pdev, struct device *dev, val = amd_pmc_reg_read(pdev, AMD_PMC_SCRATCH_REG_CZN); break; case AMD_CPU_ID_YC: + case AMD_CPU_ID_CB: + case AMD_CPU_ID_PS: val = amd_pmc_reg_read(pdev, AMD_PMC_SCRATCH_REG_YC); break; default: @@ -491,7 +495,8 @@ static void amd_pmc_dbgfs_register(struct amd_pmc_dev *dev) &amd_pmc_idlemask_fops); /* Enable STB only when the module_param is set */ if (enable_stb) { - if (dev->cpu_id == AMD_CPU_ID_YC) + if (dev->cpu_id == AMD_CPU_ID_YC || dev->cpu_id == AMD_CPU_ID_CB || + dev->cpu_id == AMD_CPU_ID_PS) debugfs_create_file("stb_read", 0644, dev->dbgfs_dir, dev, &amd_pmc_stb_debugfs_fops_v2); else @@ -615,6 +620,8 @@ static int amd_pmc_get_os_hint(struct amd_pmc_dev *dev) return MSG_OS_HINT_PCO; case AMD_CPU_ID_RN: case AMD_CPU_ID_YC: + case AMD_CPU_ID_CB: + case AMD_CPU_ID_PS: return MSG_OS_HINT_RN; } return -EINVAL; @@ -735,6 +742,8 @@ static struct acpi_s2idle_dev_ops amd_pmc_s2idle_dev_ops = { #endif static const struct pci_device_id pmc_pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_PS) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_CB) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_YC) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_CZN) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_RN) }, @@ -877,7 +886,7 @@ static int amd_pmc_probe(struct platform_device *pdev) mutex_init(&dev->lock); - if (enable_stb && dev->cpu_id == AMD_CPU_ID_YC) { + if (enable_stb && (dev->cpu_id == AMD_CPU_ID_YC || dev->cpu_id == AMD_CPU_ID_CB)) { err = amd_pmc_s2d_init(dev); if (err) return err; @@ -915,6 +924,7 @@ static const struct acpi_device_id amd_pmc_acpi_ids[] = { {"AMDI0005", 0}, {"AMDI0006", 0}, {"AMDI0007", 0}, + {"AMDI0008", 0}, {"AMD0004", 0}, {"AMD0005", 0}, { } diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c index 57a07db659cb..478dd300b9c9 100644 --- a/drivers/platform/x86/asus-nb-wmi.c +++ b/drivers/platform/x86/asus-nb-wmi.c @@ -522,6 +522,7 @@ static const struct key_entry asus_nb_wmi_keymap[] = { { KE_KEY, 0x31, { KEY_VOLUMEDOWN } }, { KE_KEY, 0x32, { KEY_MUTE } }, { KE_KEY, 0x35, { KEY_SCREENLOCK } }, + { KE_KEY, 0x38, { KEY_PROG3 } }, /* Armoury Crate */ { KE_KEY, 0x40, { KEY_PREVIOUSSONG } }, { KE_KEY, 0x41, { KEY_NEXTSONG } }, { KE_KEY, 0x43, { KEY_STOPCD } }, /* Stop/Eject */ @@ -574,6 +575,7 @@ static const struct key_entry asus_nb_wmi_keymap[] = { { KE_KEY, 0xA5, { KEY_SWITCHVIDEOMODE } }, /* SDSP LCD + TV + HDMI */ { KE_KEY, 0xA6, { KEY_SWITCHVIDEOMODE } }, /* SDSP CRT + TV + HDMI */ { KE_KEY, 0xA7, { KEY_SWITCHVIDEOMODE } }, /* SDSP LCD + CRT + TV + HDMI */ + { KE_KEY, 0xB3, { KEY_PROG4 } }, /* AURA */ { KE_KEY, 0xB5, { KEY_CALC } }, { KE_KEY, 0xC4, { KEY_KBDILLUMUP } }, { KE_KEY, 0xC5, { KEY_KBDILLUMDOWN } }, diff --git a/drivers/platform/x86/gigabyte-wmi.c b/drivers/platform/x86/gigabyte-wmi.c index 497ad2f64a51..5e7e6659a849 100644 --- a/drivers/platform/x86/gigabyte-wmi.c +++ b/drivers/platform/x86/gigabyte-wmi.c @@ -150,6 +150,7 @@ static const struct dmi_system_id gigabyte_wmi_known_working_platforms[] = { DMI_EXACT_MATCH_GIGABYTE_BOARD_NAME("B550M AORUS PRO-P"), DMI_EXACT_MATCH_GIGABYTE_BOARD_NAME("B550M DS3H"), DMI_EXACT_MATCH_GIGABYTE_BOARD_NAME("B660 GAMING X DDR4"), + DMI_EXACT_MATCH_GIGABYTE_BOARD_NAME("B660I AORUS PRO DDR4"), DMI_EXACT_MATCH_GIGABYTE_BOARD_NAME("Z390 I AORUS PRO WIFI-CF"), DMI_EXACT_MATCH_GIGABYTE_BOARD_NAME("Z490 AORUS ELITE AC"), DMI_EXACT_MATCH_GIGABYTE_BOARD_NAME("X570 AORUS ELITE"), diff --git a/drivers/platform/x86/intel/atomisp2/led.c b/drivers/platform/x86/intel/atomisp2/led.c index 5935dfca166f..10077a61d8c5 100644 --- a/drivers/platform/x86/intel/atomisp2/led.c +++ b/drivers/platform/x86/intel/atomisp2/led.c @@ -50,7 +50,8 @@ static const struct dmi_system_id atomisp2_led_systems[] __initconst = { { .matches = { DMI_EXACT_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), - DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "T100TA"), + /* Non exact match to also match T100TAF */ + DMI_MATCH(DMI_PRODUCT_NAME, "T100TA"), }, .driver_data = &asus_t100ta_lookup, }, diff --git a/drivers/platform/x86/intel/ifs/Kconfig b/drivers/platform/x86/intel/ifs/Kconfig index 7ce896434b8f..c341a27cc1a3 100644 --- a/drivers/platform/x86/intel/ifs/Kconfig +++ b/drivers/platform/x86/intel/ifs/Kconfig @@ -1,6 +1,9 @@ config INTEL_IFS tristate "Intel In Field Scan" depends on X86 && CPU_SUP_INTEL && 64BIT && SMP + # Discussion on the list has shown that the sysfs API needs a bit + # more work, mark this as broken for now + depends on BROKEN select INTEL_IFS_DEVICE help Enable support for the In Field Scan capability in select diff --git a/drivers/platform/x86/x86-android-tablets.c b/drivers/platform/x86/x86-android-tablets.c index f446be72e539..480375977435 100644 --- a/drivers/platform/x86/x86-android-tablets.c +++ b/drivers/platform/x86/x86-android-tablets.c @@ -27,8 +27,8 @@ #include <linux/pinctrl/machine.h> #include <linux/platform_data/lp855x.h> #include <linux/platform_device.h> -#include <linux/pm.h> #include <linux/power/bq24190_charger.h> +#include <linux/reboot.h> #include <linux/rmi.h> #include <linux/serdev.h> #include <linux/spi/spi.h> @@ -889,6 +889,7 @@ static const struct pinctrl_map lenovo_yoga_tab2_830_1050_codec_pinctrl_map = "INT33FC:02", "pmu_clk2_grp", "pmu_clk"); static struct pinctrl *lenovo_yoga_tab2_830_1050_codec_pinctrl; +static struct sys_off_handler *lenovo_yoga_tab2_830_1050_sys_off_handler; static int __init lenovo_yoga_tab2_830_1050_init_codec(void) { @@ -933,9 +934,11 @@ err_put_device: * followed by a normal 3 second press to recover. Avoid this by doing an EFI * poweroff instead. */ -static void lenovo_yoga_tab2_830_1050_power_off(void) +static int lenovo_yoga_tab2_830_1050_power_off(struct sys_off_data *data) { efi.reset_system(EFI_RESET_SHUTDOWN, EFI_SUCCESS, 0, NULL); + + return NOTIFY_DONE; } static int __init lenovo_yoga_tab2_830_1050_init(void) @@ -950,13 +953,19 @@ static int __init lenovo_yoga_tab2_830_1050_init(void) if (ret) return ret; - pm_power_off = lenovo_yoga_tab2_830_1050_power_off; + /* SYS_OFF_PRIO_FIRMWARE + 1 so that it runs before acpi_power_off */ + lenovo_yoga_tab2_830_1050_sys_off_handler = + register_sys_off_handler(SYS_OFF_MODE_POWER_OFF, SYS_OFF_PRIO_FIRMWARE + 1, + lenovo_yoga_tab2_830_1050_power_off, NULL); + if (IS_ERR(lenovo_yoga_tab2_830_1050_sys_off_handler)) + return PTR_ERR(lenovo_yoga_tab2_830_1050_sys_off_handler); + return 0; } static void lenovo_yoga_tab2_830_1050_exit(void) { - pm_power_off = NULL; /* Just turn poweroff into halt on module unload */ + unregister_sys_off_handler(lenovo_yoga_tab2_830_1050_sys_off_handler); if (lenovo_yoga_tab2_830_1050_codec_pinctrl) { pinctrl_put(lenovo_yoga_tab2_830_1050_codec_pinctrl); diff --git a/drivers/power/reset/arm-versatile-reboot.c b/drivers/power/reset/arm-versatile-reboot.c index 08d0a07b58ef..c7624d7611a7 100644 --- a/drivers/power/reset/arm-versatile-reboot.c +++ b/drivers/power/reset/arm-versatile-reboot.c @@ -146,6 +146,7 @@ static int __init versatile_reboot_probe(void) versatile_reboot_type = (enum versatile_reboot)reboot_id->data; syscon_regmap = syscon_node_to_regmap(np); + of_node_put(np); if (IS_ERR(syscon_regmap)) return PTR_ERR(syscon_regmap); diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c index ec8a404d71b4..4339fa9ff009 100644 --- a/drivers/power/supply/ab8500_fg.c +++ b/drivers/power/supply/ab8500_fg.c @@ -3148,6 +3148,7 @@ static int ab8500_fg_probe(struct platform_device *pdev) ret = ab8500_fg_init_hw_registers(di); if (ret) { dev_err(dev, "failed to initialize registers\n"); + destroy_workqueue(di->fg_wq); return ret; } @@ -3159,6 +3160,7 @@ static int ab8500_fg_probe(struct platform_device *pdev) di->fg_psy = devm_power_supply_register(dev, &ab8500_fg_desc, &psy_cfg); if (IS_ERR(di->fg_psy)) { dev_err(dev, "failed to register FG psy\n"); + destroy_workqueue(di->fg_wq); return PTR_ERR(di->fg_psy); } @@ -3174,8 +3176,10 @@ static int ab8500_fg_probe(struct platform_device *pdev) /* Register primary interrupt handlers */ for (i = 0; i < ARRAY_SIZE(ab8500_fg_irq); i++) { irq = platform_get_irq_byname(pdev, ab8500_fg_irq[i].name); - if (irq < 0) + if (irq < 0) { + destroy_workqueue(di->fg_wq); return irq; + } ret = devm_request_threaded_irq(dev, irq, NULL, ab8500_fg_irq[i].isr, @@ -3185,6 +3189,7 @@ static int ab8500_fg_probe(struct platform_device *pdev) if (ret != 0) { dev_err(dev, "failed to request %s IRQ %d: %d\n", ab8500_fg_irq[i].name, irq, ret); + destroy_workqueue(di->fg_wq); return ret; } dev_dbg(dev, "Requested %s IRQ %d: %d\n", @@ -3200,6 +3205,7 @@ static int ab8500_fg_probe(struct platform_device *pdev) ret = ab8500_fg_sysfs_init(di); if (ret) { dev_err(dev, "failed to create sysfs entry\n"); + destroy_workqueue(di->fg_wq); return ret; } @@ -3207,6 +3213,7 @@ static int ab8500_fg_probe(struct platform_device *pdev) if (ret) { dev_err(dev, "failed to create FG psy\n"); ab8500_fg_sysfs_exit(di); + destroy_workqueue(di->fg_wq); return ret; } diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c index fad5890c899e..470253c337c7 100644 --- a/drivers/power/supply/power_supply_core.c +++ b/drivers/power/supply/power_supply_core.c @@ -846,17 +846,17 @@ int power_supply_temp2resist_simple(struct power_supply_resistance_temp_table *t { int i, high, low; - /* Break loop at table_len - 1 because that is the highest index */ - for (i = 0; i < table_len - 1; i++) + for (i = 0; i < table_len; i++) if (temp > table[i].temp) break; /* The library function will deal with high == low */ - if ((i == 0) || (i == (table_len - 1))) - high = i; + if (i == 0) + high = low = i; + else if (i == table_len) + high = low = i - 1; else - high = i - 1; - low = i; + high = (low = i) - 1; return fixp_linear_interpolate(table[low].temp, table[low].resistance, @@ -958,17 +958,17 @@ int power_supply_ocv2cap_simple(struct power_supply_battery_ocv_table *table, { int i, high, low; - /* Break loop at table_len - 1 because that is the highest index */ - for (i = 0; i < table_len - 1; i++) + for (i = 0; i < table_len; i++) if (ocv > table[i].ocv) break; /* The library function will deal with high == low */ - if ((i == 0) || (i == (table_len - 1))) - high = i - 1; + if (i == 0) + high = low = i; + else if (i == table_len) + high = low = i - 1; else - high = i; /* i.e. i == 0 */ - low = i; + high = (low = i) - 1; return fixp_linear_interpolate(table[low].ocv, table[low].capacity, diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 5c13d2079d96..0a9045b49c50 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -1435,7 +1435,7 @@ static int __verify_queue_reservations(struct device_driver *drv, void *data) if (ap_drv->in_use) { rc = ap_drv->in_use(ap_perms.apm, newaqm); if (rc) - return -EBUSY; + rc = -EBUSY; } /* release the driver's module */ diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index c95360a3c186..0917b05059b4 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -3195,6 +3195,9 @@ static int megasas_map_queues(struct Scsi_Host *shost) qoff += map->nr_queues; offset += map->nr_queues; + /* we never use READ queue, so can't cheat blk-mq */ + shost->tag_set.map[HCTX_TYPE_READ].nr_queues = 0; + /* Setup Poll hctx */ map = &shost->tag_set.map[HCTX_TYPE_POLL]; map->nr_queues = instance->iopoll_q_count; diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c index f7466a895d3b..991eb01bb1e0 100644 --- a/drivers/scsi/pm8001/pm8001_hwi.c +++ b/drivers/scsi/pm8001/pm8001_hwi.c @@ -3145,15 +3145,6 @@ void pm8001_bytes_dmaed(struct pm8001_hba_info *pm8001_ha, int i) if (!phy->phy_attached) return; - if (sas_phy->phy) { - struct sas_phy *sphy = sas_phy->phy; - sphy->negotiated_linkrate = sas_phy->linkrate; - sphy->minimum_linkrate = phy->minimum_linkrate; - sphy->minimum_linkrate_hw = SAS_LINK_RATE_1_5_GBPS; - sphy->maximum_linkrate = phy->maximum_linkrate; - sphy->maximum_linkrate_hw = phy->maximum_linkrate; - } - if (phy->phy_type & PORT_TYPE_SAS) { struct sas_identify_frame *id; id = (struct sas_identify_frame *)phy->frame_rcvd; @@ -3177,26 +3168,22 @@ void pm8001_get_lrate_mode(struct pm8001_phy *phy, u8 link_rate) switch (link_rate) { case PHY_SPEED_120: phy->sas_phy.linkrate = SAS_LINK_RATE_12_0_GBPS; - phy->sas_phy.phy->negotiated_linkrate = SAS_LINK_RATE_12_0_GBPS; break; case PHY_SPEED_60: phy->sas_phy.linkrate = SAS_LINK_RATE_6_0_GBPS; - phy->sas_phy.phy->negotiated_linkrate = SAS_LINK_RATE_6_0_GBPS; break; case PHY_SPEED_30: phy->sas_phy.linkrate = SAS_LINK_RATE_3_0_GBPS; - phy->sas_phy.phy->negotiated_linkrate = SAS_LINK_RATE_3_0_GBPS; break; case PHY_SPEED_15: phy->sas_phy.linkrate = SAS_LINK_RATE_1_5_GBPS; - phy->sas_phy.phy->negotiated_linkrate = SAS_LINK_RATE_1_5_GBPS; break; } sas_phy->negotiated_linkrate = phy->sas_phy.linkrate; - sas_phy->maximum_linkrate_hw = SAS_LINK_RATE_6_0_GBPS; + sas_phy->maximum_linkrate_hw = phy->maximum_linkrate; sas_phy->minimum_linkrate_hw = SAS_LINK_RATE_1_5_GBPS; - sas_phy->maximum_linkrate = SAS_LINK_RATE_6_0_GBPS; - sas_phy->minimum_linkrate = SAS_LINK_RATE_1_5_GBPS; + sas_phy->maximum_linkrate = phy->maximum_linkrate; + sas_phy->minimum_linkrate = phy->minimum_linkrate; } /** diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c index 9b04f1a6a67d..01f2f41928eb 100644 --- a/drivers/scsi/pm8001/pm8001_init.c +++ b/drivers/scsi/pm8001/pm8001_init.c @@ -143,6 +143,8 @@ static void pm8001_phy_init(struct pm8001_hba_info *pm8001_ha, int phy_id) struct asd_sas_phy *sas_phy = &phy->sas_phy; phy->phy_state = PHY_LINK_DISABLE; phy->pm8001_ha = pm8001_ha; + phy->minimum_linkrate = SAS_LINK_RATE_1_5_GBPS; + phy->maximum_linkrate = SAS_LINK_RATE_6_0_GBPS; sas_phy->enabled = (phy_id < pm8001_ha->chip->n_phy) ? 1 : 0; sas_phy->class = SAS; sas_phy->iproto = SAS_PROTOCOL_ALL; diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c index 01c5e8ff4cc5..303cd05fec50 100644 --- a/drivers/scsi/pm8001/pm80xx_hwi.c +++ b/drivers/scsi/pm8001/pm80xx_hwi.c @@ -3723,8 +3723,12 @@ static int mpi_phy_stop_resp(struct pm8001_hba_info *pm8001_ha, void *piomb) pm8001_dbg(pm8001_ha, MSG, "phy:0x%x status:0x%x\n", phyid, status); if (status == PHY_STOP_SUCCESS || - status == PHY_STOP_ERR_DEVICE_ATTACHED) + status == PHY_STOP_ERR_DEVICE_ATTACHED) { phy->phy_state = PHY_LINK_DISABLE; + phy->sas_phy.phy->negotiated_linkrate = SAS_PHY_DISABLED; + phy->sas_phy.linkrate = SAS_PHY_DISABLED; + } + return 0; } diff --git a/drivers/spi/spi-amd.c b/drivers/spi/spi-amd.c index cba6a4486c24..efdcbe6c4c26 100644 --- a/drivers/spi/spi-amd.c +++ b/drivers/spi/spi-amd.c @@ -33,6 +33,7 @@ #define AMD_SPI_RX_COUNT_REG 0x4B #define AMD_SPI_STATUS_REG 0x4C +#define AMD_SPI_FIFO_SIZE 70 #define AMD_SPI_MEM_SIZE 200 /* M_CMD OP codes for SPI */ @@ -270,6 +271,11 @@ static int amd_spi_master_transfer(struct spi_master *master, return 0; } +static size_t amd_spi_max_transfer_size(struct spi_device *spi) +{ + return AMD_SPI_FIFO_SIZE; +} + static int amd_spi_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -302,6 +308,8 @@ static int amd_spi_probe(struct platform_device *pdev) master->flags = SPI_MASTER_HALF_DUPLEX; master->setup = amd_spi_master_setup; master->transfer_one_message = amd_spi_master_transfer; + master->max_transfer_size = amd_spi_max_transfer_size; + master->max_message_size = amd_spi_max_transfer_size; /* Register the controller with SPI framework */ err = devm_spi_register_master(dev, master); diff --git a/drivers/spi/spi-aspeed-smc.c b/drivers/spi/spi-aspeed-smc.c index 496f3e1e9079..3e891bf22470 100644 --- a/drivers/spi/spi-aspeed-smc.c +++ b/drivers/spi/spi-aspeed-smc.c @@ -558,6 +558,14 @@ static int aspeed_spi_dirmap_create(struct spi_mem_dirmap_desc *desc) u32 ctl_val; int ret = 0; + dev_dbg(aspi->dev, + "CE%d %s dirmap [ 0x%.8llx - 0x%.8llx ] OP %#x mode:%d.%d.%d.%d naddr:%#x ndummies:%#x\n", + chip->cs, op->data.dir == SPI_MEM_DATA_IN ? "read" : "write", + desc->info.offset, desc->info.offset + desc->info.length, + op->cmd.opcode, op->cmd.buswidth, op->addr.buswidth, + op->dummy.buswidth, op->data.buswidth, + op->addr.nbytes, op->dummy.nbytes); + chip->clk_freq = desc->mem->spi->max_speed_hz; /* Only for reads */ @@ -574,9 +582,11 @@ static int aspeed_spi_dirmap_create(struct spi_mem_dirmap_desc *desc) ctl_val = readl(chip->ctl) & ~CTRL_IO_CMD_MASK; ctl_val |= aspeed_spi_get_io_mode(op) | op->cmd.opcode << CTRL_COMMAND_SHIFT | - CTRL_IO_DUMMY_SET(op->dummy.nbytes / op->dummy.buswidth) | CTRL_IO_MODE_READ; + if (op->dummy.nbytes) + ctl_val |= CTRL_IO_DUMMY_SET(op->dummy.nbytes / op->dummy.buswidth); + /* Tune 4BYTE address mode */ if (op->addr.nbytes) { u32 addr_mode = readl(aspi->regs + CE_CTRL_REG); diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index 2b9fc8449a62..72b1a5a2298c 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -1578,8 +1578,7 @@ static int cqspi_probe(struct platform_device *pdev) ret = cqspi_of_get_pdata(cqspi); if (ret) { dev_err(dev, "Cannot get mandatory OF data.\n"); - ret = -ENODEV; - goto probe_master_put; + return -ENODEV; } /* Obtain QSPI clock. */ @@ -1587,7 +1586,7 @@ static int cqspi_probe(struct platform_device *pdev) if (IS_ERR(cqspi->clk)) { dev_err(dev, "Cannot claim QSPI clock.\n"); ret = PTR_ERR(cqspi->clk); - goto probe_master_put; + return ret; } /* Obtain and remap controller address. */ @@ -1596,7 +1595,7 @@ static int cqspi_probe(struct platform_device *pdev) if (IS_ERR(cqspi->iobase)) { dev_err(dev, "Cannot remap controller address.\n"); ret = PTR_ERR(cqspi->iobase); - goto probe_master_put; + return ret; } /* Obtain and remap AHB address. */ @@ -1605,7 +1604,7 @@ static int cqspi_probe(struct platform_device *pdev) if (IS_ERR(cqspi->ahb_base)) { dev_err(dev, "Cannot remap AHB address.\n"); ret = PTR_ERR(cqspi->ahb_base); - goto probe_master_put; + return ret; } cqspi->mmap_phys_base = (dma_addr_t)res_ahb->start; cqspi->ahb_size = resource_size(res_ahb); @@ -1614,15 +1613,13 @@ static int cqspi_probe(struct platform_device *pdev) /* Obtain IRQ line. */ irq = platform_get_irq(pdev, 0); - if (irq < 0) { - ret = -ENXIO; - goto probe_master_put; - } + if (irq < 0) + return -ENXIO; pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); if (ret < 0) - goto probe_master_put; + return ret; ret = clk_prepare_enable(cqspi->clk); if (ret) { @@ -1716,8 +1713,6 @@ probe_reset_failed: probe_clk_failed: pm_runtime_put_sync(dev); pm_runtime_disable(dev); -probe_master_put: - spi_master_put(master); return ret; } diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c index e68f1cc8ef98..6c8d8b051bfd 100644 --- a/drivers/target/target_core_file.c +++ b/drivers/target/target_core_file.c @@ -448,6 +448,9 @@ fd_execute_write_same(struct se_cmd *cmd) return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; } + if (!cmd->t_data_nents) + return TCM_INVALID_CDB_FIELD; + if (cmd->t_data_nents > 1 || cmd->t_data_sg[0].length != cmd->se_dev->dev_attrib.block_size) { pr_err("WRITE_SAME: Illegal SGL t_data_nents: %u length: %u" diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index 378c80313a0f..1ed9381751e6 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c @@ -494,6 +494,10 @@ iblock_execute_write_same(struct se_cmd *cmd) " backends not supported\n"); return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; } + + if (!cmd->t_data_nents) + return TCM_INVALID_CDB_FIELD; + sg = &cmd->t_data_sg[0]; if (cmd->t_data_nents > 1 || diff --git a/drivers/target/target_core_sbc.c b/drivers/target/target_core_sbc.c index ca1b2312d6e7..f6132836eb38 100644 --- a/drivers/target/target_core_sbc.c +++ b/drivers/target/target_core_sbc.c @@ -312,6 +312,12 @@ sbc_setup_write_same(struct se_cmd *cmd, unsigned char flags, struct sbc_ops *op pr_warn("WRITE SAME with ANCHOR not supported\n"); return TCM_INVALID_CDB_FIELD; } + + if (flags & 0x01) { + pr_warn("WRITE SAME with NDOB not supported\n"); + return TCM_INVALID_CDB_FIELD; + } + /* * Special case for WRITE_SAME w/ UNMAP=1 that ends up getting * translated into block discard requests within backend code. diff --git a/drivers/tee/optee/optee_smc.h b/drivers/tee/optee/optee_smc.h index c60896cf71cb..73b5e7760d10 100644 --- a/drivers/tee/optee/optee_smc.h +++ b/drivers/tee/optee/optee_smc.h @@ -189,7 +189,7 @@ struct optee_smc_call_get_os_revision_result { * Have config return register usage: * a0 OPTEE_SMC_RETURN_OK * a1 Physical address of start of SHM - * a2 Size of of SHM + * a2 Size of SHM * a3 Cache settings of memory, as defined by the * OPTEE_SMC_SHM_* values above * a4-7 Preserved diff --git a/drivers/tee/optee/smc_abi.c b/drivers/tee/optee/smc_abi.c index 385cb0aee610..a1c1fa1a9c28 100644 --- a/drivers/tee/optee/smc_abi.c +++ b/drivers/tee/optee/smc_abi.c @@ -884,8 +884,8 @@ static int optee_smc_do_call_with_arg(struct tee_context *ctx, rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params); rpc_arg = tee_shm_get_va(shm, offs + rpc_arg_offs); - if (IS_ERR(arg)) - return PTR_ERR(arg); + if (IS_ERR(rpc_arg)) + return PTR_ERR(rpc_arg); } if (rpc_arg && tee_shm_is_dynamic(shm)) { diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index af0f7c603fa4..98da206cd761 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -1073,7 +1073,7 @@ EXPORT_SYMBOL_GPL(tee_device_unregister); /** * tee_get_drvdata() - Return driver_data pointer * @teedev: Device containing the driver_data pointer - * @returns the driver_data pointer supplied to tee_register(). + * @returns the driver_data pointer supplied to tee_device_alloc(). */ void *tee_get_drvdata(struct tee_device *teedev) { diff --git a/drivers/tty/pty.c b/drivers/tty/pty.c index 74bfabe5b453..752dab3356d7 100644 --- a/drivers/tty/pty.c +++ b/drivers/tty/pty.c @@ -111,21 +111,11 @@ static void pty_unthrottle(struct tty_struct *tty) static int pty_write(struct tty_struct *tty, const unsigned char *buf, int c) { struct tty_struct *to = tty->link; - unsigned long flags; - if (tty->flow.stopped) + if (tty->flow.stopped || !c) return 0; - if (c > 0) { - spin_lock_irqsave(&to->port->lock, flags); - /* Stuff the data into the input queue of the other end */ - c = tty_insert_flip_string(to->port, buf, c); - spin_unlock_irqrestore(&to->port->lock, flags); - /* And shovel */ - if (c) - tty_flip_buffer_push(to->port); - } - return c; + return tty_insert_flip_string_and_push_buffer(to->port, buf, c); } /** diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index cfbd2de0ca6e..3f56dbc9432b 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -23,6 +23,7 @@ #include <linux/sysrq.h> #include <linux/delay.h> #include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include <linux/tty.h> #include <linux/ratelimit.h> #include <linux/tty_flip.h> @@ -559,6 +560,9 @@ serial8250_register_ports(struct uart_driver *drv, struct device *dev) up->port.dev = dev; + if (uart_console_enabled(&up->port)) + pm_runtime_get_sync(up->port.dev); + serial8250_apply_quirks(up); uart_add_one_port(drv, &up->port); } diff --git a/drivers/tty/serial/8250/8250_dma.c b/drivers/tty/serial/8250/8250_dma.c index 7133fceed35e..a8dba4a0a8fb 100644 --- a/drivers/tty/serial/8250/8250_dma.c +++ b/drivers/tty/serial/8250/8250_dma.c @@ -106,10 +106,10 @@ int serial8250_tx_dma(struct uart_8250_port *p) UART_XMIT_SIZE, DMA_TO_DEVICE); dma_async_issue_pending(dma->txchan); - if (dma->tx_err) { + serial8250_clear_THRI(p); + if (dma->tx_err) dma->tx_err = 0; - serial8250_clear_THRI(p); - } + return 0; err: dma->tx_err = 1; diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c index f57bbd32ef11..bb6aca07ab56 100644 --- a/drivers/tty/serial/8250/8250_dw.c +++ b/drivers/tty/serial/8250/8250_dw.c @@ -47,7 +47,7 @@ #define RZN1_UART_xDMACR_DMA_EN BIT(0) #define RZN1_UART_xDMACR_1_WORD_BURST (0 << 1) #define RZN1_UART_xDMACR_4_WORD_BURST (1 << 1) -#define RZN1_UART_xDMACR_8_WORD_BURST (3 << 1) +#define RZN1_UART_xDMACR_8_WORD_BURST (2 << 1) #define RZN1_UART_xDMACR_BLK_SZ(x) ((x) << 3) /* Quirks */ @@ -773,18 +773,18 @@ static const struct of_device_id dw8250_of_match[] = { MODULE_DEVICE_TABLE(of, dw8250_of_match); static const struct acpi_device_id dw8250_acpi_match[] = { - { "INT33C4", 0 }, - { "INT33C5", 0 }, - { "INT3434", 0 }, - { "INT3435", 0 }, - { "80860F0A", 0 }, - { "8086228A", 0 }, - { "APMC0D08", 0}, - { "AMD0020", 0 }, - { "AMDI0020", 0 }, - { "AMDI0022", 0 }, - { "BRCM2032", 0 }, - { "HISI0031", 0 }, + { "80860F0A", (kernel_ulong_t)&dw8250_dw_apb }, + { "8086228A", (kernel_ulong_t)&dw8250_dw_apb }, + { "AMD0020", (kernel_ulong_t)&dw8250_dw_apb }, + { "AMDI0020", (kernel_ulong_t)&dw8250_dw_apb }, + { "AMDI0022", (kernel_ulong_t)&dw8250_dw_apb }, + { "APMC0D08", (kernel_ulong_t)&dw8250_dw_apb}, + { "BRCM2032", (kernel_ulong_t)&dw8250_dw_apb }, + { "HISI0031", (kernel_ulong_t)&dw8250_dw_apb }, + { "INT33C4", (kernel_ulong_t)&dw8250_dw_apb }, + { "INT33C5", (kernel_ulong_t)&dw8250_dw_apb }, + { "INT3434", (kernel_ulong_t)&dw8250_dw_apb }, + { "INT3435", (kernel_ulong_t)&dw8250_dw_apb }, { }, }; MODULE_DEVICE_TABLE(acpi, dw8250_acpi_match); diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 8f32fe9e149e..3c36a06a20b0 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -1949,7 +1949,7 @@ int serial8250_handle_irq(struct uart_port *port, unsigned int iir) if ((status & UART_LSR_THRE) && (up->ier & UART_IER_THRI)) { if (!up->dma || up->dma->tx_err) serial8250_tx_chars(up); - else + else if (!up->dma->tx_running) __stop_tx(up); } @@ -2975,8 +2975,10 @@ static int serial8250_request_std_resource(struct uart_8250_port *up) case UPIO_MEM32BE: case UPIO_MEM16: case UPIO_MEM: - if (!port->mapbase) + if (!port->mapbase) { + ret = -EINVAL; break; + } if (!request_mem_region(port->mapbase, size, "serial")) { ret = -EBUSY; diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index 97ef41cb2721..16a21422ddce 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -1367,6 +1367,15 @@ static void pl011_stop_rx(struct uart_port *port) pl011_dma_rx_stop(uap); } +static void pl011_throttle_rx(struct uart_port *port) +{ + unsigned long flags; + + spin_lock_irqsave(&port->lock, flags); + pl011_stop_rx(port); + spin_unlock_irqrestore(&port->lock, flags); +} + static void pl011_enable_ms(struct uart_port *port) { struct uart_amba_port *uap = @@ -1788,9 +1797,10 @@ static int pl011_allocate_irq(struct uart_amba_port *uap) */ static void pl011_enable_interrupts(struct uart_amba_port *uap) { + unsigned long flags; unsigned int i; - spin_lock_irq(&uap->port.lock); + spin_lock_irqsave(&uap->port.lock, flags); /* Clear out any spuriously appearing RX interrupts */ pl011_write(UART011_RTIS | UART011_RXIS, uap, REG_ICR); @@ -1812,7 +1822,14 @@ static void pl011_enable_interrupts(struct uart_amba_port *uap) if (!pl011_dma_rx_running(uap)) uap->im |= UART011_RXIM; pl011_write(uap->im, uap, REG_IMSC); - spin_unlock_irq(&uap->port.lock); + spin_unlock_irqrestore(&uap->port.lock, flags); +} + +static void pl011_unthrottle_rx(struct uart_port *port) +{ + struct uart_amba_port *uap = container_of(port, struct uart_amba_port, port); + + pl011_enable_interrupts(uap); } static int pl011_startup(struct uart_port *port) @@ -2225,6 +2242,8 @@ static const struct uart_ops amba_pl011_pops = { .stop_tx = pl011_stop_tx, .start_tx = pl011_start_tx, .stop_rx = pl011_stop_rx, + .throttle = pl011_throttle_rx, + .unthrottle = pl011_unthrottle_rx, .enable_ms = pl011_enable_ms, .break_ctl = pl011_break_ctl, .startup = pl011_startup, diff --git a/drivers/tty/serial/mvebu-uart.c b/drivers/tty/serial/mvebu-uart.c index 0429c2a54290..93489fe334d0 100644 --- a/drivers/tty/serial/mvebu-uart.c +++ b/drivers/tty/serial/mvebu-uart.c @@ -470,14 +470,14 @@ static void mvebu_uart_shutdown(struct uart_port *port) } } -static int mvebu_uart_baud_rate_set(struct uart_port *port, unsigned int baud) +static unsigned int mvebu_uart_baud_rate_set(struct uart_port *port, unsigned int baud) { unsigned int d_divisor, m_divisor; unsigned long flags; u32 brdv, osamp; if (!port->uartclk) - return -EOPNOTSUPP; + return 0; /* * The baudrate is derived from the UART clock thanks to divisors: @@ -548,7 +548,7 @@ static int mvebu_uart_baud_rate_set(struct uart_port *port, unsigned int baud) (m_divisor << 16) | (m_divisor << 24); writel(osamp, port->membase + UART_OSAMP); - return 0; + return DIV_ROUND_CLOSEST(port->uartclk, d_divisor * m_divisor); } static void mvebu_uart_set_termios(struct uart_port *port, @@ -587,15 +587,11 @@ static void mvebu_uart_set_termios(struct uart_port *port, max_baud = port->uartclk / 80; baud = uart_get_baud_rate(port, termios, old, min_baud, max_baud); - if (mvebu_uart_baud_rate_set(port, baud)) { - /* No clock available, baudrate cannot be changed */ - if (old) - baud = uart_get_baud_rate(port, old, NULL, - min_baud, max_baud); - } else { - tty_termios_encode_baud_rate(termios, baud, baud); - uart_update_timeout(port, termios->c_cflag, baud); - } + baud = mvebu_uart_baud_rate_set(port, baud); + + /* In case baudrate cannot be changed, report previous old value */ + if (baud == 0 && old) + baud = tty_termios_baud_rate(old); /* Only the following flag changes are supported */ if (old) { @@ -606,6 +602,11 @@ static void mvebu_uart_set_termios(struct uart_port *port, termios->c_cflag |= CS8; } + if (baud != 0) { + tty_termios_encode_baud_rate(termios, baud, baud); + uart_update_timeout(port, termios->c_cflag, baud); + } + spin_unlock_irqrestore(&port->lock, flags); } diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index d5ca904def34..1afe47b62ad5 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -377,8 +377,7 @@ static void enable_tx_dma(struct s3c24xx_uart_port *ourport) /* Enable tx dma mode */ ucon = rd_regl(port, S3C2410_UCON); ucon &= ~(S3C64XX_UCON_TXBURST_MASK | S3C64XX_UCON_TXMODE_MASK); - ucon |= (dma_get_cache_alignment() >= 16) ? - S3C64XX_UCON_TXBURST_16 : S3C64XX_UCON_TXBURST_1; + ucon |= S3C64XX_UCON_TXBURST_1; ucon |= S3C64XX_UCON_TXMODE_DMA; wr_regl(port, S3C2410_UCON, ucon); @@ -674,7 +673,7 @@ static void enable_rx_dma(struct s3c24xx_uart_port *ourport) S3C64XX_UCON_DMASUS_EN | S3C64XX_UCON_TIMEOUT_EN | S3C64XX_UCON_RXMODE_MASK); - ucon |= S3C64XX_UCON_RXBURST_16 | + ucon |= S3C64XX_UCON_RXBURST_1 | 0xf << S3C64XX_UCON_TIMEOUT_SHIFT | S3C64XX_UCON_EMPTYINT_EN | S3C64XX_UCON_TIMEOUT_EN | diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 338ebadfd44b..3dc926d6c00a 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -1941,11 +1941,6 @@ static int uart_proc_show(struct seq_file *m, void *v) } #endif -static inline bool uart_console_enabled(struct uart_port *port) -{ - return uart_console(port) && (port->cons->flags & CON_ENABLED); -} - static void uart_port_spin_lock_init(struct uart_port *port) { spin_lock_init(&port->lock); diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c index b7b44f4050d4..0973b03eeeaa 100644 --- a/drivers/tty/serial/stm32-usart.c +++ b/drivers/tty/serial/stm32-usart.c @@ -72,6 +72,8 @@ static void stm32_usart_config_reg_rs485(u32 *cr1, u32 *cr3, u32 delay_ADE, *cr3 |= USART_CR3_DEM; over8 = *cr1 & USART_CR1_OVER8; + *cr1 &= ~(USART_CR1_DEDT_MASK | USART_CR1_DEAT_MASK); + if (over8) rs485_deat_dedt = delay_ADE * baud * 8; else diff --git a/drivers/tty/tty.h b/drivers/tty/tty.h index b710c5ef89ab..f310a8274df1 100644 --- a/drivers/tty/tty.h +++ b/drivers/tty/tty.h @@ -111,4 +111,7 @@ static inline void tty_audit_tiocsti(struct tty_struct *tty, char ch) ssize_t redirected_tty_write(struct kiocb *, struct iov_iter *); +int tty_insert_flip_string_and_push_buffer(struct tty_port *port, + const unsigned char *chars, size_t cnt); + #endif diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c index bfa431a8e690..595d8b49c745 100644 --- a/drivers/tty/tty_buffer.c +++ b/drivers/tty/tty_buffer.c @@ -532,6 +532,15 @@ static void flush_to_ldisc(struct work_struct *work) } +static inline void tty_flip_buffer_commit(struct tty_buffer *tail) +{ + /* + * Paired w/ acquire in flush_to_ldisc(); ensures flush_to_ldisc() sees + * buffer data. + */ + smp_store_release(&tail->commit, tail->used); +} + /** * tty_flip_buffer_push - push terminal buffers * @port: tty port to push @@ -546,16 +555,43 @@ void tty_flip_buffer_push(struct tty_port *port) { struct tty_bufhead *buf = &port->buf; - /* - * Paired w/ acquire in flush_to_ldisc(); ensures flush_to_ldisc() sees - * buffer data. - */ - smp_store_release(&buf->tail->commit, buf->tail->used); + tty_flip_buffer_commit(buf->tail); queue_work(system_unbound_wq, &buf->work); } EXPORT_SYMBOL(tty_flip_buffer_push); /** + * tty_insert_flip_string_and_push_buffer - add characters to the tty buffer and + * push + * @port: tty port + * @chars: characters + * @size: size + * + * The function combines tty_insert_flip_string() and tty_flip_buffer_push() + * with the exception of properly holding the @port->lock. + * + * To be used only internally (by pty currently). + * + * Returns: the number added. + */ +int tty_insert_flip_string_and_push_buffer(struct tty_port *port, + const unsigned char *chars, size_t size) +{ + struct tty_bufhead *buf = &port->buf; + unsigned long flags; + + spin_lock_irqsave(&port->lock, flags); + size = tty_insert_flip_string(port, chars, size); + if (size) + tty_flip_buffer_commit(buf->tail); + spin_unlock_irqrestore(&port->lock, flags); + + queue_work(system_unbound_wq, &buf->work); + + return size; +} + +/** * tty_buffer_init - prepare a tty buffer structure * @port: tty port to initialise * diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index f8c87c4d7399..dfc1f4b445f3 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -855,7 +855,7 @@ static void delete_char(struct vc_data *vc, unsigned int nr) unsigned short *p = (unsigned short *) vc->vc_pos; vc_uniscr_delete(vc, nr); - scr_memcpyw(p, p + nr, (vc->vc_cols - vc->state.x - nr) * 2); + scr_memmovew(p, p + nr, (vc->vc_cols - vc->state.x - nr) * 2); scr_memsetw(p + vc->vc_cols - vc->state.x - nr, vc->vc_video_erase_char, nr * 2); vc->vc_need_wrap = 0; diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index ce86d1b790c0..c7b337480e3e 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -5738,7 +5738,7 @@ int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable) } hba->dev_info.wb_enabled = enable; - dev_info(hba->dev, "%s Write Booster %s\n", + dev_dbg(hba->dev, "%s Write Booster %s\n", __func__, enable ? "enabled" : "disabled"); return ret; @@ -7253,7 +7253,7 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) hba->silence_err_logs = false; /* scale up clocks to max frequency before full reinitialization */ - ufshcd_set_clk_freq(hba, true); + ufshcd_scale_clks(hba, true); err = ufshcd_hba_enable(hba); diff --git a/drivers/usb/dwc3/dwc3-am62.c b/drivers/usb/dwc3/dwc3-am62.c index fea7aca35dc8..173cf3579c55 100644 --- a/drivers/usb/dwc3/dwc3-am62.c +++ b/drivers/usb/dwc3/dwc3-am62.c @@ -195,8 +195,7 @@ static int dwc3_ti_probe(struct platform_device *pdev) if (i == ARRAY_SIZE(dwc3_ti_rate_table)) { dev_err(dev, "unsupported usb2_refclk rate: %lu KHz\n", rate); - ret = -EINVAL; - goto err_clk_disable; + return -EINVAL; } data->rate_code = i; @@ -204,7 +203,7 @@ static int dwc3_ti_probe(struct platform_device *pdev) /* Read the syscon property and set the rate code */ ret = phy_syscon_pll_refclk(data); if (ret) - goto err_clk_disable; + return ret; /* VBUS divider select */ data->vbus_divider = device_property_read_bool(dev, "ti,vbus-divider"); @@ -245,8 +244,6 @@ err_pm_disable: clk_disable_unprepare(data->usb2_refclk); pm_runtime_disable(dev); pm_runtime_set_suspended(dev); -err_clk_disable: - clk_put(data->usb2_refclk); return ret; } @@ -276,7 +273,6 @@ static int dwc3_ti_remove(struct platform_device *pdev) pm_runtime_disable(dev); pm_runtime_set_suspended(dev); - clk_put(data->usb2_refclk); platform_set_drvdata(pdev, NULL); return 0; } diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 8716bece1072..0d89dfa6eef5 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -4249,7 +4249,6 @@ static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt) } evt->count = 0; - evt->flags &= ~DWC3_EVENT_PENDING; ret = IRQ_HANDLED; /* Unmask interrupt */ @@ -4261,6 +4260,9 @@ static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt) dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); } + /* Keep the clearing of DWC3_EVENT_PENDING at the end */ + evt->flags &= ~DWC3_EVENT_PENDING; + return ret; } diff --git a/drivers/usb/gadget/function/uvc_configfs.c b/drivers/usb/gadget/function/uvc_configfs.c index e5a6b6e36b3d..4303a3283ba0 100644 --- a/drivers/usb/gadget/function/uvc_configfs.c +++ b/drivers/usb/gadget/function/uvc_configfs.c @@ -2371,6 +2371,7 @@ static ssize_t f_uvc_opts_string_##cname##_store(struct config_item *item,\ const char *page, size_t len) \ { \ struct f_uvc_opts *opts = to_f_uvc_opts(item); \ + int size = min(sizeof(opts->aname), len + 1); \ int ret = 0; \ \ mutex_lock(&opts->lock); \ @@ -2379,8 +2380,9 @@ static ssize_t f_uvc_opts_string_##cname##_store(struct config_item *item,\ goto end; \ } \ \ - ret = snprintf(opts->aname, min(sizeof(opts->aname), len), \ - "%s", page); \ + ret = strscpy(opts->aname, page, size); \ + if (ret == -E2BIG) \ + ret = size - 1; \ \ end: \ mutex_unlock(&opts->lock); \ diff --git a/drivers/usb/host/ehci-fsl.c b/drivers/usb/host/ehci-fsl.c index 385be30baad3..896c0d107f72 100644 --- a/drivers/usb/host/ehci-fsl.c +++ b/drivers/usb/host/ehci-fsl.c @@ -76,14 +76,9 @@ static int fsl_ehci_drv_probe(struct platform_device *pdev) return -ENODEV; } - res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!res) { - dev_err(&pdev->dev, - "Found HC with no IRQ. Check %s setup!\n", - dev_name(&pdev->dev)); - return -ENODEV; - } - irq = res->start; + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; hcd = __usb_create_hcd(&fsl_ehci_hc_driver, pdev->dev.parent, &pdev->dev, dev_name(&pdev->dev), NULL); diff --git a/drivers/usb/host/fsl-mph-dr-of.c b/drivers/usb/host/fsl-mph-dr-of.c index 44a7e58a26e3..e5df17522892 100644 --- a/drivers/usb/host/fsl-mph-dr-of.c +++ b/drivers/usb/host/fsl-mph-dr-of.c @@ -112,6 +112,9 @@ static struct platform_device *fsl_usb2_device_register( goto error; } + pdev->dev.of_node = ofdev->dev.of_node; + pdev->dev.of_node_reused = true; + retval = platform_device_add(pdev); if (retval) goto error; diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c index b440d338a895..d5a3986dfee7 100644 --- a/drivers/usb/serial/ftdi_sio.c +++ b/drivers/usb/serial/ftdi_sio.c @@ -1023,6 +1023,9 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(FTDI_VID, CHETCO_SEASMART_DISPLAY_PID) }, { USB_DEVICE(FTDI_VID, CHETCO_SEASMART_LITE_PID) }, { USB_DEVICE(FTDI_VID, CHETCO_SEASMART_ANALOG_PID) }, + /* Belimo Automation devices */ + { USB_DEVICE(FTDI_VID, BELIMO_ZTH_PID) }, + { USB_DEVICE(FTDI_VID, BELIMO_ZIP_PID) }, /* ICP DAS I-756xU devices */ { USB_DEVICE(ICPDAS_VID, ICPDAS_I7560U_PID) }, { USB_DEVICE(ICPDAS_VID, ICPDAS_I7561U_PID) }, diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h index d1a9564697a4..4e92c165c86b 100644 --- a/drivers/usb/serial/ftdi_sio_ids.h +++ b/drivers/usb/serial/ftdi_sio_ids.h @@ -1569,6 +1569,12 @@ #define CHETCO_SEASMART_ANALOG_PID 0xA5AF /* SeaSmart Analog Adapter */ /* + * Belimo Automation + */ +#define BELIMO_ZTH_PID 0x8050 +#define BELIMO_ZIP_PID 0xC811 + +/* * Unjo AB */ #define UNJO_VID 0x22B7 diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index ee0e520707dd..c4724750c81a 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -1718,6 +1718,7 @@ void typec_set_pwr_opmode(struct typec_port *port, partner->usb_pd = 1; sysfs_notify(&partner_dev->kobj, NULL, "supports_usb_power_delivery"); + kobject_uevent(&partner_dev->kobj, KOBJ_CHANGE); } put_device(partner_dev); } diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 4b56c39f766d..84b143eef395 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -396,13 +396,15 @@ static void __unmap_grant_pages_done(int result, unsigned int offset = data->unmap_ops - map->unmap_ops; for (i = 0; i < data->count; i++) { - WARN_ON(map->unmap_ops[offset+i].status); + WARN_ON(map->unmap_ops[offset + i].status != GNTST_okay && + map->unmap_ops[offset + i].handle != INVALID_GRANT_HANDLE); pr_debug("unmap handle=%d st=%d\n", map->unmap_ops[offset+i].handle, map->unmap_ops[offset+i].status); map->unmap_ops[offset+i].handle = INVALID_GRANT_HANDLE; if (use_ptemod) { - WARN_ON(map->kunmap_ops[offset+i].status); + WARN_ON(map->kunmap_ops[offset + i].status != GNTST_okay && + map->kunmap_ops[offset + i].handle != INVALID_GRANT_HANDLE); pr_debug("kunmap handle=%u st=%d\n", map->kunmap_ops[offset+i].handle, map->kunmap_ops[offset+i].status); diff --git a/fs/afs/file.c b/fs/afs/file.c index 42118a4f3383..d1cfb235c4b9 100644 --- a/fs/afs/file.c +++ b/fs/afs/file.c @@ -375,7 +375,7 @@ static int afs_begin_cache_operation(struct netfs_io_request *rreq) } static int afs_check_write_begin(struct file *file, loff_t pos, unsigned len, - struct folio *folio, void **_fsdata) + struct folio **foliop, void **_fsdata) { struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 415bf1823fb3..9c21e214d29e 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -675,9 +675,8 @@ struct btrfs_fs_info { rwlock_t global_root_lock; struct rb_root global_root_tree; - /* The xarray that holds all the FS roots */ - spinlock_t fs_roots_lock; - struct xarray fs_roots; + spinlock_t fs_roots_radix_lock; + struct radix_tree_root fs_roots_radix; /* block group cache stuff */ rwlock_t block_group_cache_lock; @@ -995,10 +994,10 @@ struct btrfs_fs_info { struct btrfs_delayed_root *delayed_root; - /* Extent buffer xarray */ + /* Extent buffer radix tree */ spinlock_t buffer_lock; /* Entries are eb->start / sectorsize */ - struct xarray extent_buffers; + struct radix_tree_root buffer_radix; /* next backup root to be overwritten */ int backup_root_index; @@ -1119,8 +1118,7 @@ enum { */ BTRFS_ROOT_SHAREABLE, BTRFS_ROOT_TRACK_DIRTY, - /* The root is tracked in fs_info::fs_roots */ - BTRFS_ROOT_REGISTERED, + BTRFS_ROOT_IN_RADIX, BTRFS_ROOT_ORPHAN_ITEM_INSERTED, BTRFS_ROOT_DEFRAG_RUNNING, BTRFS_ROOT_FORCE_COW, @@ -1224,10 +1222,10 @@ struct btrfs_root { struct rb_root inode_tree; /* - * Xarray that keeps track of delayed nodes of every inode, protected - * by inode_lock + * radix tree that keeps track of delayed nodes of every inode, + * protected by inode_lock */ - struct xarray delayed_nodes; + struct radix_tree_root delayed_nodes_tree; /* * right now this just gets used so that a root has its own devid * for stat. It may be used for more later diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index 66779ab3ed4a..748bf6b0d860 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -78,7 +78,7 @@ static struct btrfs_delayed_node *btrfs_get_delayed_node( } spin_lock(&root->inode_lock); - node = xa_load(&root->delayed_nodes, ino); + node = radix_tree_lookup(&root->delayed_nodes_tree, ino); if (node) { if (btrfs_inode->delayed_node) { @@ -90,9 +90,9 @@ static struct btrfs_delayed_node *btrfs_get_delayed_node( /* * It's possible that we're racing into the middle of removing - * this node from the xarray. In this case, the refcount + * this node from the radix tree. In this case, the refcount * was zero and it should never go back to one. Just return - * NULL like it was never in the xarray at all; our release + * NULL like it was never in the radix at all; our release * function is in the process of removing it. * * Some implementations of refcount_inc refuse to bump the @@ -100,7 +100,7 @@ static struct btrfs_delayed_node *btrfs_get_delayed_node( * here, refcount_inc() may decide to just WARN_ONCE() instead * of actually bumping the refcount. * - * If this node is properly in the xarray, we want to bump the + * If this node is properly in the radix, we want to bump the * refcount twice, once for the inode and once for this get * operation. */ @@ -128,30 +128,36 @@ static struct btrfs_delayed_node *btrfs_get_or_create_delayed_node( u64 ino = btrfs_ino(btrfs_inode); int ret; - do { - node = btrfs_get_delayed_node(btrfs_inode); - if (node) - return node; +again: + node = btrfs_get_delayed_node(btrfs_inode); + if (node) + return node; - node = kmem_cache_zalloc(delayed_node_cache, GFP_NOFS); - if (!node) - return ERR_PTR(-ENOMEM); - btrfs_init_delayed_node(node, root, ino); + node = kmem_cache_zalloc(delayed_node_cache, GFP_NOFS); + if (!node) + return ERR_PTR(-ENOMEM); + btrfs_init_delayed_node(node, root, ino); - /* Cached in the inode and can be accessed */ - refcount_set(&node->refs, 2); + /* cached in the btrfs inode and can be accessed */ + refcount_set(&node->refs, 2); - spin_lock(&root->inode_lock); - ret = xa_insert(&root->delayed_nodes, ino, node, GFP_NOFS); - if (ret) { - spin_unlock(&root->inode_lock); - kmem_cache_free(delayed_node_cache, node); - if (ret != -EBUSY) - return ERR_PTR(ret); - } - } while (ret); + ret = radix_tree_preload(GFP_NOFS); + if (ret) { + kmem_cache_free(delayed_node_cache, node); + return ERR_PTR(ret); + } + + spin_lock(&root->inode_lock); + ret = radix_tree_insert(&root->delayed_nodes_tree, ino, node); + if (ret == -EEXIST) { + spin_unlock(&root->inode_lock); + kmem_cache_free(delayed_node_cache, node); + radix_tree_preload_end(); + goto again; + } btrfs_inode->delayed_node = node; spin_unlock(&root->inode_lock); + radix_tree_preload_end(); return node; } @@ -270,7 +276,8 @@ static void __btrfs_release_delayed_node( * back up. We can delete it now. */ ASSERT(refcount_read(&delayed_node->refs) == 0); - xa_erase(&root->delayed_nodes, delayed_node->inode_id); + radix_tree_delete(&root->delayed_nodes_tree, + delayed_node->inode_id); spin_unlock(&root->inode_lock); kmem_cache_free(delayed_node_cache, delayed_node); } @@ -1863,35 +1870,34 @@ void btrfs_kill_delayed_inode_items(struct btrfs_inode *inode) void btrfs_kill_all_delayed_nodes(struct btrfs_root *root) { - unsigned long index = 0; - struct btrfs_delayed_node *delayed_node; + u64 inode_id = 0; struct btrfs_delayed_node *delayed_nodes[8]; + int i, n; while (1) { - int n = 0; - spin_lock(&root->inode_lock); - if (xa_empty(&root->delayed_nodes)) { + n = radix_tree_gang_lookup(&root->delayed_nodes_tree, + (void **)delayed_nodes, inode_id, + ARRAY_SIZE(delayed_nodes)); + if (!n) { spin_unlock(&root->inode_lock); - return; + break; } - xa_for_each_start(&root->delayed_nodes, index, delayed_node, index) { + inode_id = delayed_nodes[n - 1]->inode_id + 1; + for (i = 0; i < n; i++) { /* * Don't increase refs in case the node is dead and * about to be removed from the tree in the loop below */ - if (refcount_inc_not_zero(&delayed_node->refs)) { - delayed_nodes[n] = delayed_node; - n++; - } - if (n >= ARRAY_SIZE(delayed_nodes)) - break; + if (!refcount_inc_not_zero(&delayed_nodes[i]->refs)) + delayed_nodes[i] = NULL; } - index++; spin_unlock(&root->inode_lock); - for (int i = 0; i < n; i++) { + for (i = 0; i < n; i++) { + if (!delayed_nodes[i]) + continue; __btrfs_kill_delayed_node(delayed_nodes[i]); btrfs_release_delayed_node(delayed_nodes[i]); } diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 4ba005c41983..de440ebf5648 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -5,6 +5,7 @@ #include <linux/fs.h> #include <linux/blkdev.h> +#include <linux/radix-tree.h> #include <linux/writeback.h> #include <linux/workqueue.h> #include <linux/kthread.h> @@ -485,7 +486,7 @@ static int csum_dirty_subpage_buffers(struct btrfs_fs_info *fs_info, uptodate = btrfs_subpage_test_uptodate(fs_info, page, cur, fs_info->nodesize); - /* A dirty eb shouldn't disappear from extent_buffers */ + /* A dirty eb shouldn't disappear from buffer_radix */ if (WARN_ON(!eb)) return -EUCLEAN; @@ -1158,7 +1159,7 @@ static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info, root->nr_delalloc_inodes = 0; root->nr_ordered_extents = 0; root->inode_tree = RB_ROOT; - xa_init_flags(&root->delayed_nodes, GFP_ATOMIC); + INIT_RADIX_TREE(&root->delayed_nodes_tree, GFP_ATOMIC); btrfs_init_root_block_rsv(root); @@ -1210,9 +1211,9 @@ static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info, btrfs_qgroup_init_swapped_blocks(&root->swapped_blocks); #ifdef CONFIG_BTRFS_DEBUG INIT_LIST_HEAD(&root->leak_list); - spin_lock(&fs_info->fs_roots_lock); + spin_lock(&fs_info->fs_roots_radix_lock); list_add_tail(&root->leak_list, &fs_info->allocated_roots); - spin_unlock(&fs_info->fs_roots_lock); + spin_unlock(&fs_info->fs_roots_radix_lock); #endif } @@ -1659,11 +1660,12 @@ static struct btrfs_root *btrfs_lookup_fs_root(struct btrfs_fs_info *fs_info, { struct btrfs_root *root; - spin_lock(&fs_info->fs_roots_lock); - root = xa_load(&fs_info->fs_roots, (unsigned long)root_id); + spin_lock(&fs_info->fs_roots_radix_lock); + root = radix_tree_lookup(&fs_info->fs_roots_radix, + (unsigned long)root_id); if (root) root = btrfs_grab_root(root); - spin_unlock(&fs_info->fs_roots_lock); + spin_unlock(&fs_info->fs_roots_radix_lock); return root; } @@ -1705,14 +1707,20 @@ int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info, { int ret; - spin_lock(&fs_info->fs_roots_lock); - ret = xa_insert(&fs_info->fs_roots, (unsigned long)root->root_key.objectid, - root, GFP_NOFS); + ret = radix_tree_preload(GFP_NOFS); + if (ret) + return ret; + + spin_lock(&fs_info->fs_roots_radix_lock); + ret = radix_tree_insert(&fs_info->fs_roots_radix, + (unsigned long)root->root_key.objectid, + root); if (ret == 0) { btrfs_grab_root(root); - set_bit(BTRFS_ROOT_REGISTERED, &root->state); + set_bit(BTRFS_ROOT_IN_RADIX, &root->state); } - spin_unlock(&fs_info->fs_roots_lock); + spin_unlock(&fs_info->fs_roots_radix_lock); + radix_tree_preload_end(); return ret; } @@ -2342,9 +2350,9 @@ void btrfs_put_root(struct btrfs_root *root) btrfs_drew_lock_destroy(&root->snapshot_lock); free_root_extent_buffers(root); #ifdef CONFIG_BTRFS_DEBUG - spin_lock(&root->fs_info->fs_roots_lock); + spin_lock(&root->fs_info->fs_roots_radix_lock); list_del_init(&root->leak_list); - spin_unlock(&root->fs_info->fs_roots_lock); + spin_unlock(&root->fs_info->fs_roots_radix_lock); #endif kfree(root); } @@ -2352,21 +2360,28 @@ void btrfs_put_root(struct btrfs_root *root) void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info) { - struct btrfs_root *root; - unsigned long index = 0; + int ret; + struct btrfs_root *gang[8]; + int i; while (!list_empty(&fs_info->dead_roots)) { - root = list_entry(fs_info->dead_roots.next, - struct btrfs_root, root_list); - list_del(&root->root_list); + gang[0] = list_entry(fs_info->dead_roots.next, + struct btrfs_root, root_list); + list_del(&gang[0]->root_list); - if (test_bit(BTRFS_ROOT_REGISTERED, &root->state)) - btrfs_drop_and_free_fs_root(fs_info, root); - btrfs_put_root(root); + if (test_bit(BTRFS_ROOT_IN_RADIX, &gang[0]->state)) + btrfs_drop_and_free_fs_root(fs_info, gang[0]); + btrfs_put_root(gang[0]); } - xa_for_each(&fs_info->fs_roots, index, root) { - btrfs_drop_and_free_fs_root(fs_info, root); + while (1) { + ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix, + (void **)gang, 0, + ARRAY_SIZE(gang)); + if (!ret) + break; + for (i = 0; i < ret; i++) + btrfs_drop_and_free_fs_root(fs_info, gang[i]); } } @@ -3134,8 +3149,8 @@ static int __cold init_tree_roots(struct btrfs_fs_info *fs_info) void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) { - xa_init_flags(&fs_info->fs_roots, GFP_ATOMIC); - xa_init_flags(&fs_info->extent_buffers, GFP_ATOMIC); + INIT_RADIX_TREE(&fs_info->fs_roots_radix, GFP_ATOMIC); + INIT_RADIX_TREE(&fs_info->buffer_radix, GFP_ATOMIC); INIT_LIST_HEAD(&fs_info->trans_list); INIT_LIST_HEAD(&fs_info->dead_roots); INIT_LIST_HEAD(&fs_info->delayed_iputs); @@ -3143,7 +3158,7 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) INIT_LIST_HEAD(&fs_info->caching_block_groups); spin_lock_init(&fs_info->delalloc_root_lock); spin_lock_init(&fs_info->trans_lock); - spin_lock_init(&fs_info->fs_roots_lock); + spin_lock_init(&fs_info->fs_roots_radix_lock); spin_lock_init(&fs_info->delayed_iput_lock); spin_lock_init(&fs_info->defrag_inodes_lock); spin_lock_init(&fs_info->super_lock); @@ -3374,7 +3389,7 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) /* * btrfs_find_orphan_roots() is responsible for finding all the dead * roots (with 0 refs), flag them with BTRFS_ROOT_DEAD_TREE and load - * them into the fs_info->fs_roots. This must be done before + * them into the fs_info->fs_roots_radix tree. This must be done before * calling btrfs_orphan_cleanup() on the tree root. If we don't do it * first, then btrfs_orphan_cleanup() will delete a dead root's orphan * item before the root's tree is deleted - this means that if we unmount @@ -4499,11 +4514,12 @@ void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info, { bool drop_ref = false; - spin_lock(&fs_info->fs_roots_lock); - xa_erase(&fs_info->fs_roots, (unsigned long)root->root_key.objectid); - if (test_and_clear_bit(BTRFS_ROOT_REGISTERED, &root->state)) + spin_lock(&fs_info->fs_roots_radix_lock); + radix_tree_delete(&fs_info->fs_roots_radix, + (unsigned long)root->root_key.objectid); + if (test_and_clear_bit(BTRFS_ROOT_IN_RADIX, &root->state)) drop_ref = true; - spin_unlock(&fs_info->fs_roots_lock); + spin_unlock(&fs_info->fs_roots_radix_lock); if (BTRFS_FS_ERROR(fs_info)) { ASSERT(root->log_root == NULL); @@ -4519,48 +4535,50 @@ void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info, int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info) { - struct btrfs_root *roots[8]; - unsigned long index = 0; - int i; + u64 root_objectid = 0; + struct btrfs_root *gang[8]; + int i = 0; int err = 0; - int grabbed; + unsigned int ret = 0; while (1) { - struct btrfs_root *root; - - spin_lock(&fs_info->fs_roots_lock); - if (!xa_find(&fs_info->fs_roots, &index, ULONG_MAX, XA_PRESENT)) { - spin_unlock(&fs_info->fs_roots_lock); - return err; + spin_lock(&fs_info->fs_roots_radix_lock); + ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix, + (void **)gang, root_objectid, + ARRAY_SIZE(gang)); + if (!ret) { + spin_unlock(&fs_info->fs_roots_radix_lock); + break; } + root_objectid = gang[ret - 1]->root_key.objectid + 1; - grabbed = 0; - xa_for_each_start(&fs_info->fs_roots, index, root, index) { - /* Avoid grabbing roots in dead_roots */ - if (btrfs_root_refs(&root->root_item) > 0) - roots[grabbed++] = btrfs_grab_root(root); - if (grabbed >= ARRAY_SIZE(roots)) - break; + for (i = 0; i < ret; i++) { + /* Avoid to grab roots in dead_roots */ + if (btrfs_root_refs(&gang[i]->root_item) == 0) { + gang[i] = NULL; + continue; + } + /* grab all the search result for later use */ + gang[i] = btrfs_grab_root(gang[i]); } - spin_unlock(&fs_info->fs_roots_lock); + spin_unlock(&fs_info->fs_roots_radix_lock); - for (i = 0; i < grabbed; i++) { - if (!roots[i]) + for (i = 0; i < ret; i++) { + if (!gang[i]) continue; - index = roots[i]->root_key.objectid; - err = btrfs_orphan_cleanup(roots[i]); + root_objectid = gang[i]->root_key.objectid; + err = btrfs_orphan_cleanup(gang[i]); if (err) - goto out; - btrfs_put_root(roots[i]); + break; + btrfs_put_root(gang[i]); } - index++; + root_objectid++; } -out: - /* Release the roots that remain uncleaned due to error */ - for (; i < grabbed; i++) { - if (roots[i]) - btrfs_put_root(roots[i]); + /* release the uncleaned roots due to error */ + for (; i < ret; i++) { + if (gang[i]) + btrfs_put_root(gang[i]); } return err; } @@ -4879,28 +4897,31 @@ static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info) static void btrfs_drop_all_logs(struct btrfs_fs_info *fs_info) { - unsigned long index = 0; - int grabbed = 0; - struct btrfs_root *roots[8]; + struct btrfs_root *gang[8]; + u64 root_objectid = 0; + int ret; + + spin_lock(&fs_info->fs_roots_radix_lock); + while ((ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix, + (void **)gang, root_objectid, + ARRAY_SIZE(gang))) != 0) { + int i; - spin_lock(&fs_info->fs_roots_lock); - while ((grabbed = xa_extract(&fs_info->fs_roots, (void **)roots, index, - ULONG_MAX, 8, XA_PRESENT))) { - for (int i = 0; i < grabbed; i++) - roots[i] = btrfs_grab_root(roots[i]); - spin_unlock(&fs_info->fs_roots_lock); + for (i = 0; i < ret; i++) + gang[i] = btrfs_grab_root(gang[i]); + spin_unlock(&fs_info->fs_roots_radix_lock); - for (int i = 0; i < grabbed; i++) { - if (!roots[i]) + for (i = 0; i < ret; i++) { + if (!gang[i]) continue; - index = roots[i]->root_key.objectid; - btrfs_free_log(NULL, roots[i]); - btrfs_put_root(roots[i]); + root_objectid = gang[i]->root_key.objectid; + btrfs_free_log(NULL, gang[i]); + btrfs_put_root(gang[i]); } - index++; - spin_lock(&fs_info->fs_roots_lock); + root_objectid++; + spin_lock(&fs_info->fs_roots_radix_lock); } - spin_unlock(&fs_info->fs_roots_lock); + spin_unlock(&fs_info->fs_roots_radix_lock); btrfs_free_log_root_tree(NULL, fs_info); } diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 4157ecc27d4b..a3afc15430ce 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -5829,7 +5829,7 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) btrfs_qgroup_convert_reserved_meta(root, INT_MAX); btrfs_qgroup_free_meta_all_pertrans(root); - if (test_bit(BTRFS_ROOT_REGISTERED, &root->state)) + if (test_bit(BTRFS_ROOT_IN_RADIX, &root->state)) btrfs_add_dropped_root(trans, root); else btrfs_put_root(root); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 04e36343da3a..f03ab5dbda7a 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2966,7 +2966,7 @@ static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page) } /* - * Find extent buffer for a given bytenr. + * Find extent buffer for a givne bytenr. * * This is for end_bio_extent_readpage(), thus we can't do any unsafe locking * in endio context. @@ -2985,9 +2985,11 @@ static struct extent_buffer *find_extent_buffer_readpage( return (struct extent_buffer *)page->private; } - /* For subpage case, we need to lookup extent buffer xarray */ - eb = xa_load(&fs_info->extent_buffers, - bytenr >> fs_info->sectorsize_bits); + /* For subpage case, we need to lookup buffer radix tree */ + rcu_read_lock(); + eb = radix_tree_lookup(&fs_info->buffer_radix, + bytenr >> fs_info->sectorsize_bits); + rcu_read_unlock(); ASSERT(eb); return eb; } @@ -4435,8 +4437,8 @@ static struct extent_buffer *find_extent_buffer_nolock( struct extent_buffer *eb; rcu_read_lock(); - eb = xa_load(&fs_info->extent_buffers, - start >> fs_info->sectorsize_bits); + eb = radix_tree_lookup(&fs_info->buffer_radix, + start >> fs_info->sectorsize_bits); if (eb && atomic_inc_not_zero(&eb->refs)) { rcu_read_unlock(); return eb; @@ -6129,22 +6131,24 @@ struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info, if (!eb) return ERR_PTR(-ENOMEM); eb->fs_info = fs_info; - - do { - ret = xa_insert(&fs_info->extent_buffers, - start >> fs_info->sectorsize_bits, - eb, GFP_NOFS); - if (ret == -ENOMEM) { - exists = ERR_PTR(ret); +again: + ret = radix_tree_preload(GFP_NOFS); + if (ret) { + exists = ERR_PTR(ret); + goto free_eb; + } + spin_lock(&fs_info->buffer_lock); + ret = radix_tree_insert(&fs_info->buffer_radix, + start >> fs_info->sectorsize_bits, eb); + spin_unlock(&fs_info->buffer_lock); + radix_tree_preload_end(); + if (ret == -EEXIST) { + exists = find_extent_buffer(fs_info, start); + if (exists) goto free_eb; - } - if (ret == -EBUSY) { - exists = find_extent_buffer(fs_info, start); - if (exists) - goto free_eb; - } - } while (ret); - + else + goto again; + } check_buffer_tree_ref(eb); set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags); @@ -6319,22 +6323,25 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, } if (uptodate) set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); - - do { - ret = xa_insert(&fs_info->extent_buffers, - start >> fs_info->sectorsize_bits, - eb, GFP_NOFS); - if (ret == -ENOMEM) { - exists = ERR_PTR(ret); +again: + ret = radix_tree_preload(GFP_NOFS); + if (ret) { + exists = ERR_PTR(ret); + goto free_eb; + } + + spin_lock(&fs_info->buffer_lock); + ret = radix_tree_insert(&fs_info->buffer_radix, + start >> fs_info->sectorsize_bits, eb); + spin_unlock(&fs_info->buffer_lock); + radix_tree_preload_end(); + if (ret == -EEXIST) { + exists = find_extent_buffer(fs_info, start); + if (exists) goto free_eb; - } - if (ret == -EBUSY) { - exists = find_extent_buffer(fs_info, start); - if (exists) - goto free_eb; - } - } while (ret); - + else + goto again; + } /* add one reference for the tree */ check_buffer_tree_ref(eb); set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags); @@ -6379,8 +6386,10 @@ static int release_extent_buffer(struct extent_buffer *eb) spin_unlock(&eb->refs_lock); - xa_erase(&fs_info->extent_buffers, - eb->start >> fs_info->sectorsize_bits); + spin_lock(&fs_info->buffer_lock); + radix_tree_delete(&fs_info->buffer_radix, + eb->start >> fs_info->sectorsize_bits); + spin_unlock(&fs_info->buffer_lock); } else { spin_unlock(&eb->refs_lock); } @@ -7325,25 +7334,42 @@ void memmove_extent_buffer(const struct extent_buffer *dst, } } +#define GANG_LOOKUP_SIZE 16 static struct extent_buffer *get_next_extent_buffer( struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr) { - struct extent_buffer *eb; - unsigned long index; + struct extent_buffer *gang[GANG_LOOKUP_SIZE]; + struct extent_buffer *found = NULL; u64 page_start = page_offset(page); + u64 cur = page_start; ASSERT(in_range(bytenr, page_start, PAGE_SIZE)); lockdep_assert_held(&fs_info->buffer_lock); - xa_for_each_start(&fs_info->extent_buffers, index, eb, - page_start >> fs_info->sectorsize_bits) { - if (in_range(eb->start, page_start, PAGE_SIZE)) - return eb; - else if (eb->start >= page_start + PAGE_SIZE) - /* Already beyond page end */ - return NULL; + while (cur < page_start + PAGE_SIZE) { + int ret; + int i; + + ret = radix_tree_gang_lookup(&fs_info->buffer_radix, + (void **)gang, cur >> fs_info->sectorsize_bits, + min_t(unsigned int, GANG_LOOKUP_SIZE, + PAGE_SIZE / fs_info->nodesize)); + if (ret == 0) + goto out; + for (i = 0; i < ret; i++) { + /* Already beyond page end */ + if (gang[i]->start >= page_start + PAGE_SIZE) + goto out; + /* Found one */ + if (gang[i]->start >= bytenr) { + found = gang[i]; + goto out; + } + } + cur = gang[ret - 1]->start + gang[ret - 1]->len; } - return NULL; +out: + return found; } static int try_release_subpage_extent_buffer(struct page *page) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 0172f75e051a..d50448bf8eed 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3578,7 +3578,6 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) u64 last_objectid = 0; int ret = 0, nr_unlink = 0; - /* Bail out if the cleanup is already running. */ if (test_and_set_bit(BTRFS_ROOT_ORPHAN_CLEANUP, &root->state)) return 0; @@ -3661,17 +3660,17 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) * * btrfs_find_orphan_roots() ran before us, which has * found all deleted roots and loaded them into - * fs_info->fs_roots. So here we can find if an + * fs_info->fs_roots_radix. So here we can find if an * orphan item corresponds to a deleted root by looking - * up the root from that xarray. + * up the root from that radix tree. */ - spin_lock(&fs_info->fs_roots_lock); - dead_root = xa_load(&fs_info->fs_roots, - (unsigned long)found_key.objectid); + spin_lock(&fs_info->fs_roots_radix_lock); + dead_root = radix_tree_lookup(&fs_info->fs_roots_radix, + (unsigned long)found_key.objectid); if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0) is_dead_root = 1; - spin_unlock(&fs_info->fs_roots_lock); + spin_unlock(&fs_info->fs_roots_radix_lock); if (is_dead_root) { /* prevent this orphan from being found again */ @@ -3911,7 +3910,7 @@ cache_index: * cache. * * This is required for both inode re-read from disk and delayed inode - * in the delayed_nodes xarray. + * in delayed_nodes_tree. */ if (BTRFS_I(inode)->last_trans == fs_info->generation) set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index fa56890ff81f..c7dea639a56f 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -10,6 +10,7 @@ #include <linux/mount.h> #include <linux/xattr.h> #include <linux/posix_acl_xattr.h> +#include <linux/radix-tree.h> #include <linux/vmalloc.h> #include <linux/string.h> #include <linux/compat.h> @@ -127,7 +128,7 @@ struct send_ctx { struct list_head new_refs; struct list_head deleted_refs; - struct xarray name_cache; + struct radix_tree_root name_cache; struct list_head name_cache_list; int name_cache_size; @@ -268,13 +269,14 @@ struct orphan_dir_info { struct name_cache_entry { struct list_head list; /* - * On 32bit kernels, xarray has only 32bit indices, but we need to - * handle 64bit inums. We use the lower 32bit of the 64bit inum to store - * it in the tree. If more than one inum would fall into the same entry, - * we use inum_aliases to store the additional entries. inum_aliases is - * also used to store entries with the same inum but different generations. + * radix_tree has only 32bit entries but we need to handle 64bit inums. + * We use the lower 32bit of the 64bit inum to store it in the tree. If + * more then one inum would fall into the same entry, we use radix_list + * to store the additional entries. radix_list is also used to store + * entries where two entries have the same inum but different + * generations. */ - struct list_head inum_aliases; + struct list_head radix_list; u64 ino; u64 gen; u64 parent_ino; @@ -2024,9 +2026,9 @@ out: } /* - * Insert a name cache entry. On 32bit kernels the xarray index is 32bit, + * Insert a name cache entry. On 32bit kernels the radix tree index is 32bit, * so we need to do some special handling in case we have clashes. This function - * takes care of this with the help of name_cache_entry::inum_aliases. + * takes care of this with the help of name_cache_entry::radix_list. * In case of error, nce is kfreed. */ static int name_cache_insert(struct send_ctx *sctx, @@ -2035,7 +2037,8 @@ static int name_cache_insert(struct send_ctx *sctx, int ret = 0; struct list_head *nce_head; - nce_head = xa_load(&sctx->name_cache, (unsigned long)nce->ino); + nce_head = radix_tree_lookup(&sctx->name_cache, + (unsigned long)nce->ino); if (!nce_head) { nce_head = kmalloc(sizeof(*nce_head), GFP_KERNEL); if (!nce_head) { @@ -2044,14 +2047,14 @@ static int name_cache_insert(struct send_ctx *sctx, } INIT_LIST_HEAD(nce_head); - ret = xa_insert(&sctx->name_cache, nce->ino, nce_head, GFP_KERNEL); + ret = radix_tree_insert(&sctx->name_cache, nce->ino, nce_head); if (ret < 0) { kfree(nce_head); kfree(nce); return ret; } } - list_add_tail(&nce->inum_aliases, nce_head); + list_add_tail(&nce->radix_list, nce_head); list_add_tail(&nce->list, &sctx->name_cache_list); sctx->name_cache_size++; @@ -2063,14 +2066,15 @@ static void name_cache_delete(struct send_ctx *sctx, { struct list_head *nce_head; - nce_head = xa_load(&sctx->name_cache, (unsigned long)nce->ino); + nce_head = radix_tree_lookup(&sctx->name_cache, + (unsigned long)nce->ino); if (!nce_head) { btrfs_err(sctx->send_root->fs_info, "name_cache_delete lookup failed ino %llu cache size %d, leaking memory", nce->ino, sctx->name_cache_size); } - list_del(&nce->inum_aliases); + list_del(&nce->radix_list); list_del(&nce->list); sctx->name_cache_size--; @@ -2078,7 +2082,7 @@ static void name_cache_delete(struct send_ctx *sctx, * We may not get to the final release of nce_head if the lookup fails */ if (nce_head && list_empty(nce_head)) { - xa_erase(&sctx->name_cache, (unsigned long)nce->ino); + radix_tree_delete(&sctx->name_cache, (unsigned long)nce->ino); kfree(nce_head); } } @@ -2089,11 +2093,11 @@ static struct name_cache_entry *name_cache_search(struct send_ctx *sctx, struct list_head *nce_head; struct name_cache_entry *cur; - nce_head = xa_load(&sctx->name_cache, (unsigned long)ino); + nce_head = radix_tree_lookup(&sctx->name_cache, (unsigned long)ino); if (!nce_head) return NULL; - list_for_each_entry(cur, nce_head, inum_aliases) { + list_for_each_entry(cur, nce_head, radix_list) { if (cur->ino == ino && cur->gen == gen) return cur; } @@ -7518,7 +7522,7 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) INIT_LIST_HEAD(&sctx->new_refs); INIT_LIST_HEAD(&sctx->deleted_refs); - xa_init_flags(&sctx->name_cache, GFP_KERNEL); + INIT_RADIX_TREE(&sctx->name_cache, GFP_KERNEL); INIT_LIST_HEAD(&sctx->name_cache_list); sctx->flags = arg->flags; diff --git a/fs/btrfs/tests/btrfs-tests.c b/fs/btrfs/tests/btrfs-tests.c index 1591bfa55bcc..d8e56edd6991 100644 --- a/fs/btrfs/tests/btrfs-tests.c +++ b/fs/btrfs/tests/btrfs-tests.c @@ -150,8 +150,8 @@ struct btrfs_fs_info *btrfs_alloc_dummy_fs_info(u32 nodesize, u32 sectorsize) void btrfs_free_dummy_fs_info(struct btrfs_fs_info *fs_info) { - unsigned long index; - struct extent_buffer *eb; + struct radix_tree_iter iter; + void **slot; struct btrfs_device *dev, *tmp; if (!fs_info) @@ -163,9 +163,25 @@ void btrfs_free_dummy_fs_info(struct btrfs_fs_info *fs_info) test_mnt->mnt_sb->s_fs_info = NULL; - xa_for_each(&fs_info->extent_buffers, index, eb) { + spin_lock(&fs_info->buffer_lock); + radix_tree_for_each_slot(slot, &fs_info->buffer_radix, &iter, 0) { + struct extent_buffer *eb; + + eb = radix_tree_deref_slot_protected(slot, &fs_info->buffer_lock); + if (!eb) + continue; + /* Shouldn't happen but that kind of thinking creates CVE's */ + if (radix_tree_exception(eb)) { + if (radix_tree_deref_retry(eb)) + slot = radix_tree_iter_retry(&iter); + continue; + } + slot = radix_tree_iter_resume(slot, &iter); + spin_unlock(&fs_info->buffer_lock); free_extent_buffer_stale(eb); + spin_lock(&fs_info->buffer_lock); } + spin_unlock(&fs_info->buffer_lock); btrfs_mapping_tree_free(&fs_info->mapping_tree); list_for_each_entry_safe(dev, tmp, &fs_info->fs_devices->devices, @@ -186,7 +202,7 @@ void btrfs_free_dummy_root(struct btrfs_root *root) if (!root) return; /* Will be freed by btrfs_free_fs_roots */ - if (WARN_ON(test_bit(BTRFS_ROOT_REGISTERED, &root->state))) + if (WARN_ON(test_bit(BTRFS_ROOT_IN_RADIX, &root->state))) return; btrfs_global_root_delete(root); btrfs_put_root(root); diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 06c0a958d114..875b801ab3d7 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -23,7 +23,7 @@ #include "space-info.h" #include "zoned.h" -#define BTRFS_ROOT_TRANS_TAG XA_MARK_0 +#define BTRFS_ROOT_TRANS_TAG 0 /* * Transaction states and transitions @@ -437,15 +437,15 @@ static int record_root_in_trans(struct btrfs_trans_handle *trans, */ smp_wmb(); - spin_lock(&fs_info->fs_roots_lock); + spin_lock(&fs_info->fs_roots_radix_lock); if (root->last_trans == trans->transid && !force) { - spin_unlock(&fs_info->fs_roots_lock); + spin_unlock(&fs_info->fs_roots_radix_lock); return 0; } - xa_set_mark(&fs_info->fs_roots, - (unsigned long)root->root_key.objectid, - BTRFS_ROOT_TRANS_TAG); - spin_unlock(&fs_info->fs_roots_lock); + radix_tree_tag_set(&fs_info->fs_roots_radix, + (unsigned long)root->root_key.objectid, + BTRFS_ROOT_TRANS_TAG); + spin_unlock(&fs_info->fs_roots_radix_lock); root->last_trans = trans->transid; /* this is pretty tricky. We don't want to @@ -487,9 +487,11 @@ void btrfs_add_dropped_root(struct btrfs_trans_handle *trans, spin_unlock(&cur_trans->dropped_roots_lock); /* Make sure we don't try to update the root at commit time */ - xa_clear_mark(&fs_info->fs_roots, - (unsigned long)root->root_key.objectid, - BTRFS_ROOT_TRANS_TAG); + spin_lock(&fs_info->fs_roots_radix_lock); + radix_tree_tag_clear(&fs_info->fs_roots_radix, + (unsigned long)root->root_key.objectid, + BTRFS_ROOT_TRANS_TAG); + spin_unlock(&fs_info->fs_roots_radix_lock); } int btrfs_record_root_in_trans(struct btrfs_trans_handle *trans, @@ -1402,8 +1404,9 @@ void btrfs_add_dead_root(struct btrfs_root *root) static noinline int commit_fs_roots(struct btrfs_trans_handle *trans) { struct btrfs_fs_info *fs_info = trans->fs_info; - struct btrfs_root *root; - unsigned long index; + struct btrfs_root *gang[8]; + int i; + int ret; /* * At this point no one can be using this transaction to modify any tree @@ -1411,46 +1414,57 @@ static noinline int commit_fs_roots(struct btrfs_trans_handle *trans) */ ASSERT(trans->transaction->state == TRANS_STATE_COMMIT_DOING); - spin_lock(&fs_info->fs_roots_lock); - xa_for_each_marked(&fs_info->fs_roots, index, root, BTRFS_ROOT_TRANS_TAG) { - int ret; - - /* - * At this point we can neither have tasks logging inodes - * from a root nor trying to commit a log tree. - */ - ASSERT(atomic_read(&root->log_writers) == 0); - ASSERT(atomic_read(&root->log_commit[0]) == 0); - ASSERT(atomic_read(&root->log_commit[1]) == 0); - - xa_clear_mark(&fs_info->fs_roots, - (unsigned long)root->root_key.objectid, - BTRFS_ROOT_TRANS_TAG); - spin_unlock(&fs_info->fs_roots_lock); - - btrfs_free_log(trans, root); - ret = btrfs_update_reloc_root(trans, root); - if (ret) - return ret; - - /* See comments in should_cow_block() */ - clear_bit(BTRFS_ROOT_FORCE_COW, &root->state); - smp_mb__after_atomic(); + spin_lock(&fs_info->fs_roots_radix_lock); + while (1) { + ret = radix_tree_gang_lookup_tag(&fs_info->fs_roots_radix, + (void **)gang, 0, + ARRAY_SIZE(gang), + BTRFS_ROOT_TRANS_TAG); + if (ret == 0) + break; + for (i = 0; i < ret; i++) { + struct btrfs_root *root = gang[i]; + int ret2; + + /* + * At this point we can neither have tasks logging inodes + * from a root nor trying to commit a log tree. + */ + ASSERT(atomic_read(&root->log_writers) == 0); + ASSERT(atomic_read(&root->log_commit[0]) == 0); + ASSERT(atomic_read(&root->log_commit[1]) == 0); + + radix_tree_tag_clear(&fs_info->fs_roots_radix, + (unsigned long)root->root_key.objectid, + BTRFS_ROOT_TRANS_TAG); + spin_unlock(&fs_info->fs_roots_radix_lock); + + btrfs_free_log(trans, root); + ret2 = btrfs_update_reloc_root(trans, root); + if (ret2) + return ret2; + + /* see comments in should_cow_block() */ + clear_bit(BTRFS_ROOT_FORCE_COW, &root->state); + smp_mb__after_atomic(); + + if (root->commit_root != root->node) { + list_add_tail(&root->dirty_list, + &trans->transaction->switch_commits); + btrfs_set_root_node(&root->root_item, + root->node); + } - if (root->commit_root != root->node) { - list_add_tail(&root->dirty_list, - &trans->transaction->switch_commits); - btrfs_set_root_node(&root->root_item, root->node); + ret2 = btrfs_update_root(trans, fs_info->tree_root, + &root->root_key, + &root->root_item); + if (ret2) + return ret2; + spin_lock(&fs_info->fs_roots_radix_lock); + btrfs_qgroup_free_meta_all_pertrans(root); } - - ret = btrfs_update_root(trans, fs_info->tree_root, - &root->root_key, &root->root_item); - if (ret) - return ret; - spin_lock(&fs_info->fs_roots_lock); - btrfs_qgroup_free_meta_all_pertrans(root); } - spin_unlock(&fs_info->fs_roots_lock); + spin_unlock(&fs_info->fs_roots_radix_lock); return 0; } diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 6dee88815491..d6e5916138e4 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -63,7 +63,7 @@ (CONGESTION_ON_THRESH(congestion_kb) >> 2)) static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len, - struct folio *folio, void **_fsdata); + struct folio **foliop, void **_fsdata); static inline struct ceph_snap_context *page_snap_context(struct page *page) { @@ -1288,18 +1288,19 @@ ceph_find_incompatible(struct page *page) } static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len, - struct folio *folio, void **_fsdata) + struct folio **foliop, void **_fsdata) { struct inode *inode = file_inode(file); struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_snap_context *snapc; - snapc = ceph_find_incompatible(folio_page(folio, 0)); + snapc = ceph_find_incompatible(folio_page(*foliop, 0)); if (snapc) { int r; - folio_unlock(folio); - folio_put(folio); + folio_unlock(*foliop); + folio_put(*foliop); + *foliop = NULL; if (IS_ERR(snapc)) return PTR_ERR(snapc); diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index 42f892c5712e..0ce535852151 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -319,8 +319,9 @@ zero_out: * conflicting writes once the folio is grabbed and locked. It is passed a * pointer to the fsdata cookie that gets returned to the VM to be passed to * write_end. It is permitted to sleep. It should return 0 if the request - * should go ahead; unlock the folio and return -EAGAIN to cause the folio to - * be regot; or return an error. + * should go ahead or it may return an error. It may also unlock and put the + * folio, provided it sets ``*foliop`` to NULL, in which case a return of 0 + * will cause the folio to be re-got and the process to be retried. * * The calling netfs must initialise a netfs context contiguous to the vfs * inode before calling this. @@ -348,13 +349,13 @@ retry: if (ctx->ops->check_write_begin) { /* Allow the netfs (eg. ceph) to flush conflicts. */ - ret = ctx->ops->check_write_begin(file, pos, len, folio, _fsdata); + ret = ctx->ops->check_write_begin(file, pos, len, &folio, _fsdata); if (ret < 0) { trace_netfs_failure(NULL, NULL, ret, netfs_fail_check_write_begin); - if (ret == -EAGAIN) - goto retry; goto error; } + if (!folio) + goto retry; } if (folio_test_uptodate(folio)) @@ -416,8 +417,10 @@ have_folio_no_wait: error_put: netfs_put_request(rreq, false, netfs_rreq_trace_put_failed); error: - folio_unlock(folio); - folio_put(folio); + if (folio) { + folio_unlock(folio); + folio_put(folio); + } _leave(" = %d", ret); return ret; } diff --git a/fs/remap_range.c b/fs/remap_range.c index 5e0d97e02f96..881a306ee247 100644 --- a/fs/remap_range.c +++ b/fs/remap_range.c @@ -547,7 +547,7 @@ int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same) else if (deduped < 0) info->status = deduped; else - info->bytes_deduped = deduped; + info->bytes_deduped = len; next_fdput: fdput(dst_fd); diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index ff3e82553a76..cb2167c89eee 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -158,9 +158,24 @@ * Useful if your architecture doesn't use IPIs for remote TLB invalidates * and therefore doesn't naturally serialize with software page-table walkers. * + * MMU_GATHER_NO_FLUSH_CACHE + * + * Indicates the architecture has flush_cache_range() but it needs *NOT* be called + * before unmapping a VMA. + * + * NOTE: strictly speaking we shouldn't have this knob and instead rely on + * flush_cache_range() being a NOP, except Sparc64 seems to be + * different here. + * + * MMU_GATHER_MERGE_VMAS + * + * Indicates the architecture wants to merge ranges over VMAs; typical when + * multiple range invalidates are more expensive than a full invalidate. + * * MMU_GATHER_NO_RANGE * - * Use this if your architecture lacks an efficient flush_tlb_range(). + * Use this if your architecture lacks an efficient flush_tlb_range(). This + * option implies MMU_GATHER_MERGE_VMAS above. * * MMU_GATHER_NO_GATHER * @@ -288,6 +303,7 @@ struct mmu_gather { */ unsigned int vma_exec : 1; unsigned int vma_huge : 1; + unsigned int vma_pfn : 1; unsigned int batch_count; @@ -334,8 +350,8 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) #ifdef CONFIG_MMU_GATHER_NO_RANGE -#if defined(tlb_flush) || defined(tlb_start_vma) || defined(tlb_end_vma) -#error MMU_GATHER_NO_RANGE relies on default tlb_flush(), tlb_start_vma() and tlb_end_vma() +#if defined(tlb_flush) +#error MMU_GATHER_NO_RANGE relies on default tlb_flush() #endif /* @@ -355,17 +371,9 @@ static inline void tlb_flush(struct mmu_gather *tlb) static inline void tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { } -#define tlb_end_vma tlb_end_vma -static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { } - #else /* CONFIG_MMU_GATHER_NO_RANGE */ #ifndef tlb_flush - -#if defined(tlb_start_vma) || defined(tlb_end_vma) -#error Default tlb_flush() relies on default tlb_start_vma() and tlb_end_vma() -#endif - /* * When an architecture does not provide its own tlb_flush() implementation * but does have a reasonably efficient flush_vma_range() implementation @@ -385,6 +393,9 @@ static inline void tlb_flush(struct mmu_gather *tlb) flush_tlb_range(&vma, tlb->start, tlb->end); } } +#endif + +#endif /* CONFIG_MMU_GATHER_NO_RANGE */ static inline void tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) @@ -402,17 +413,9 @@ tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) */ tlb->vma_huge = is_vm_hugetlb_page(vma); tlb->vma_exec = !!(vma->vm_flags & VM_EXEC); + tlb->vma_pfn = !!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)); } -#else - -static inline void -tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { } - -#endif - -#endif /* CONFIG_MMU_GATHER_NO_RANGE */ - static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) { /* @@ -486,32 +489,36 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) * case where we're doing a full MM flush. When we're doing a munmap, * the vmas are adjusted to only cover the region to be torn down. */ -#ifndef tlb_start_vma static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { if (tlb->fullmm) return; tlb_update_vma_flags(tlb, vma); +#ifndef CONFIG_MMU_GATHER_NO_FLUSH_CACHE flush_cache_range(vma, vma->vm_start, vma->vm_end); -} #endif +} -#ifndef tlb_end_vma static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { if (tlb->fullmm) return; /* - * Do a TLB flush and reset the range at VMA boundaries; this avoids - * the ranges growing with the unused space between consecutive VMAs, - * but also the mmu_gather::vma_* flags from tlb_start_vma() rely on - * this. + * VM_PFNMAP is more fragile because the core mm will not track the + * page mapcount -- there might not be page-frames for these PFNs after + * all. Force flush TLBs for such ranges to avoid munmap() vs + * unmap_mapping_range() races. */ - tlb_flush_mmu_tlbonly(tlb); + if (tlb->vma_pfn || !IS_ENABLED(CONFIG_MMU_GATHER_MERGE_VMAS)) { + /* + * Do a TLB flush and reset the range at VMA boundaries; this avoids + * the ranges growing with the unused space between consecutive VMAs. + */ + tlb_flush_mmu_tlbonly(tlb); + } } -#endif /* * tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end, diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 2b21f2a3452f..20c26aed7896 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -47,6 +47,7 @@ struct kobject; struct mem_cgroup; struct module; struct bpf_func_state; +struct ftrace_ops; extern struct idr btf_idr; extern spinlock_t btf_idr_lock; @@ -221,7 +222,7 @@ struct bpf_map { u32 btf_vmlinux_value_type_id; struct btf *btf; #ifdef CONFIG_MEMCG_KMEM - struct mem_cgroup *memcg; + struct obj_cgroup *objcg; #endif char name[BPF_OBJ_NAME_LEN]; struct bpf_map_off_arr *off_arr; @@ -751,6 +752,16 @@ struct btf_func_model { /* Return the return value of fentry prog. Only used by bpf_struct_ops. */ #define BPF_TRAMP_F_RET_FENTRY_RET BIT(4) +/* Get original function from stack instead of from provided direct address. + * Makes sense for trampolines with fexit or fmod_ret programs. + */ +#define BPF_TRAMP_F_ORIG_STACK BIT(5) + +/* This trampoline is on a function with another ftrace_ops with IPMODIFY, + * e.g., a live patch. This flag is set and cleared by ftrace call backs, + */ +#define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6) + /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 * bytes on x86. */ @@ -833,9 +844,11 @@ struct bpf_tramp_image { struct bpf_trampoline { /* hlist for trampoline_table */ struct hlist_node hlist; + struct ftrace_ops *fops; /* serializes access to fields of this trampoline */ struct mutex mutex; refcount_t refcnt; + u32 flags; u64 key; struct { struct btf_func_model model; @@ -1044,7 +1057,6 @@ struct bpf_prog_aux { bool sleepable; bool tail_call_reachable; bool xdp_has_frags; - bool use_bpf_prog_pack; /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ const struct btf_type *attach_func_proto; /* function name for valid attach_btf_id */ @@ -1255,9 +1267,6 @@ struct bpf_dummy_ops { int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); #endif -int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, - int cgroup_atype); -void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog); #else static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) { @@ -1281,6 +1290,13 @@ static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, { return -EINVAL; } +#endif + +#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM) +int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, + int cgroup_atype); +void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog); +#else static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, int cgroup_atype) { @@ -1921,7 +1937,8 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs); int btf_check_kfunc_arg_match(struct bpf_verifier_env *env, const struct btf *btf, u32 func_id, - struct bpf_reg_state *regs); + struct bpf_reg_state *regs, + u32 kfunc_flags); int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *reg); int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 81b19669efba..2e3bad8640dc 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -345,10 +345,10 @@ struct bpf_verifier_state_list { }; struct bpf_loop_inline_state { - int initialized:1; /* set to true upon first entry */ - int fit_for_inline:1; /* true if callback function is the same - * at each call and flags are always zero - */ + unsigned int initialized:1; /* set to true upon first entry */ + unsigned int fit_for_inline:1; /* true if callback function is the same + * at each call and flags are always zero + */ u32 callback_subprogno; /* valid when fit_for_inline is true */ }; diff --git a/include/linux/btf.h b/include/linux/btf.h index 1bfed7fa0428..cdb376d53238 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -12,14 +12,43 @@ #define BTF_TYPE_EMIT(type) ((void)(type *)0) #define BTF_TYPE_EMIT_ENUM(enum_val) ((void)enum_val) -enum btf_kfunc_type { - BTF_KFUNC_TYPE_CHECK, - BTF_KFUNC_TYPE_ACQUIRE, - BTF_KFUNC_TYPE_RELEASE, - BTF_KFUNC_TYPE_RET_NULL, - BTF_KFUNC_TYPE_KPTR_ACQUIRE, - BTF_KFUNC_TYPE_MAX, -}; +/* These need to be macros, as the expressions are used in assembler input */ +#define KF_ACQUIRE (1 << 0) /* kfunc is an acquire function */ +#define KF_RELEASE (1 << 1) /* kfunc is a release function */ +#define KF_RET_NULL (1 << 2) /* kfunc returns a pointer that may be NULL */ +#define KF_KPTR_GET (1 << 3) /* kfunc returns reference to a kptr */ +/* Trusted arguments are those which are meant to be referenced arguments with + * unchanged offset. It is used to enforce that pointers obtained from acquire + * kfuncs remain unmodified when being passed to helpers taking trusted args. + * + * Consider + * struct foo { + * int data; + * struct foo *next; + * }; + * + * struct bar { + * int data; + * struct foo f; + * }; + * + * struct foo *f = alloc_foo(); // Acquire kfunc + * struct bar *b = alloc_bar(); // Acquire kfunc + * + * If a kfunc set_foo_data() wants to operate only on the allocated object, it + * will set the KF_TRUSTED_ARGS flag, which will prevent unsafe usage like: + * + * set_foo_data(f, 42); // Allowed + * set_foo_data(f->next, 42); // Rejected, non-referenced pointer + * set_foo_data(&f->next, 42);// Rejected, referenced, but wrong type + * set_foo_data(&b->f, 42); // Rejected, referenced, but bad offset + * + * In the final case, usually for the purposes of type matching, it is deduced + * by looking at the type of the member at the offset, but due to the + * requirement of trusted argument, this deduction will be strict and not done + * for this case. + */ +#define KF_TRUSTED_ARGS (1 << 4) /* kfunc only takes trusted pointer arguments */ struct btf; struct btf_member; @@ -30,16 +59,7 @@ struct btf_id_set; struct btf_kfunc_id_set { struct module *owner; - union { - struct { - struct btf_id_set *check_set; - struct btf_id_set *acquire_set; - struct btf_id_set *release_set; - struct btf_id_set *ret_null_set; - struct btf_id_set *kptr_acquire_set; - }; - struct btf_id_set *sets[BTF_KFUNC_TYPE_MAX]; - }; + struct btf_id_set8 *set; }; struct btf_id_dtor_kfunc { @@ -378,9 +398,9 @@ const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); const char *btf_name_by_offset(const struct btf *btf, u32 offset); struct btf *btf_parse_vmlinux(void); struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog); -bool btf_kfunc_id_set_contains(const struct btf *btf, +u32 *btf_kfunc_id_set_contains(const struct btf *btf, enum bpf_prog_type prog_type, - enum btf_kfunc_type type, u32 kfunc_btf_id); + u32 kfunc_btf_id); int register_btf_kfunc_id_set(enum bpf_prog_type prog_type, const struct btf_kfunc_id_set *s); s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id); @@ -397,12 +417,11 @@ static inline const char *btf_name_by_offset(const struct btf *btf, { return NULL; } -static inline bool btf_kfunc_id_set_contains(const struct btf *btf, +static inline u32 *btf_kfunc_id_set_contains(const struct btf *btf, enum bpf_prog_type prog_type, - enum btf_kfunc_type type, u32 kfunc_btf_id) { - return false; + return NULL; } static inline int register_btf_kfunc_id_set(enum bpf_prog_type prog_type, const struct btf_kfunc_id_set *s) diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index 252a4befeab1..2aea877d644f 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -8,6 +8,15 @@ struct btf_id_set { u32 ids[]; }; +struct btf_id_set8 { + u32 cnt; + u32 flags; + struct { + u32 id; + u32 flags; + } pairs[]; +}; + #ifdef CONFIG_DEBUG_INFO_BTF #include <linux/compiler.h> /* for __PASTE */ @@ -25,7 +34,7 @@ struct btf_id_set { #define BTF_IDS_SECTION ".BTF_ids" -#define ____BTF_ID(symbol) \ +#define ____BTF_ID(symbol, word) \ asm( \ ".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ ".local " #symbol " ; \n" \ @@ -33,10 +42,11 @@ asm( \ ".size " #symbol ", 4; \n" \ #symbol ": \n" \ ".zero 4 \n" \ +word \ ".popsection; \n"); -#define __BTF_ID(symbol) \ - ____BTF_ID(symbol) +#define __BTF_ID(symbol, word) \ + ____BTF_ID(symbol, word) #define __ID(prefix) \ __PASTE(prefix, __COUNTER__) @@ -46,7 +56,14 @@ asm( \ * to 4 zero bytes. */ #define BTF_ID(prefix, name) \ - __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__)) + __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), "") + +#define ____BTF_ID_FLAGS(prefix, name, flags) \ + __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), ".long " #flags "\n") +#define __BTF_ID_FLAGS(prefix, name, flags, ...) \ + ____BTF_ID_FLAGS(prefix, name, flags) +#define BTF_ID_FLAGS(prefix, name, ...) \ + __BTF_ID_FLAGS(prefix, name, ##__VA_ARGS__, 0) /* * The BTF_ID_LIST macro defines pure (unsorted) list @@ -145,10 +162,51 @@ asm( \ ".popsection; \n"); \ extern struct btf_id_set name; +/* + * The BTF_SET8_START/END macros pair defines sorted list of + * BTF IDs and their flags plus its members count, with the + * following layout: + * + * BTF_SET8_START(list) + * BTF_ID_FLAGS(type1, name1, flags) + * BTF_ID_FLAGS(type2, name2, flags) + * BTF_SET8_END(list) + * + * __BTF_ID__set8__list: + * .zero 8 + * list: + * __BTF_ID__type1__name1__3: + * .zero 4 + * .word (1 << 0) | (1 << 2) + * __BTF_ID__type2__name2__5: + * .zero 4 + * .word (1 << 3) | (1 << 1) | (1 << 2) + * + */ +#define __BTF_SET8_START(name, scope) \ +asm( \ +".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ +"." #scope " __BTF_ID__set8__" #name "; \n" \ +"__BTF_ID__set8__" #name ":; \n" \ +".zero 8 \n" \ +".popsection; \n"); + +#define BTF_SET8_START(name) \ +__BTF_ID_LIST(name, local) \ +__BTF_SET8_START(name, local) + +#define BTF_SET8_END(name) \ +asm( \ +".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ +".size __BTF_ID__set8__" #name ", .-" #name " \n" \ +".popsection; \n"); \ +extern struct btf_id_set8 name; + #else #define BTF_ID_LIST(name) static u32 __maybe_unused name[5]; #define BTF_ID(prefix, name) +#define BTF_ID_FLAGS(prefix, name, ...) #define BTF_ID_UNUSED #define BTF_ID_LIST_GLOBAL(name, n) u32 __maybe_unused name[n]; #define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 __maybe_unused name[1]; @@ -156,6 +214,8 @@ extern struct btf_id_set name; #define BTF_SET_START(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_START_GLOBAL(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_END(name) +#define BTF_SET8_START(name) static struct btf_id_set8 __maybe_unused name = { 0 }; +#define BTF_SET8_END(name) #endif /* CONFIG_DEBUG_INFO_BTF */ diff --git a/include/linux/filter.h b/include/linux/filter.h index 4c1a8b247545..a5f21dc3c432 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1027,6 +1027,14 @@ u64 bpf_jit_alloc_exec_limit(void); void *bpf_jit_alloc_exec(unsigned long size); void bpf_jit_free_exec(void *addr); void bpf_jit_free(struct bpf_prog *fp); +struct bpf_binary_header * +bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); + +static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) +{ + return list_empty(&fp->aux->ksym.lnode) || + fp->aux->ksym.lnode.prev == LIST_POISON2; +} struct bpf_binary_header * bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image, diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 979f6bfa2c25..0b61371e287b 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -208,6 +208,43 @@ enum { FTRACE_OPS_FL_DIRECT = BIT(17), }; +/* + * FTRACE_OPS_CMD_* commands allow the ftrace core logic to request changes + * to a ftrace_ops. Note, the requests may fail. + * + * ENABLE_SHARE_IPMODIFY_SELF - enable a DIRECT ops to work on the same + * function as an ops with IPMODIFY. Called + * when the DIRECT ops is being registered. + * This is called with both direct_mutex and + * ftrace_lock are locked. + * + * ENABLE_SHARE_IPMODIFY_PEER - enable a DIRECT ops to work on the same + * function as an ops with IPMODIFY. Called + * when the other ops (the one with IPMODIFY) + * is being registered. + * This is called with direct_mutex locked. + * + * DISABLE_SHARE_IPMODIFY_PEER - disable a DIRECT ops to work on the same + * function as an ops with IPMODIFY. Called + * when the other ops (the one with IPMODIFY) + * is being unregistered. + * This is called with direct_mutex locked. + */ +enum ftrace_ops_cmd { + FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF, + FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER, + FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER, +}; + +/* + * For most ftrace_ops_cmd, + * Returns: + * 0 - Success. + * Negative on failure. The return value is dependent on the + * callback. + */ +typedef int (*ftrace_ops_func_t)(struct ftrace_ops *op, enum ftrace_ops_cmd cmd); + #ifdef CONFIG_DYNAMIC_FTRACE /* The hash used to know what functions callbacks trace */ struct ftrace_ops_hash { @@ -250,6 +287,7 @@ struct ftrace_ops { unsigned long trampoline; unsigned long trampoline_size; struct list_head list; + ftrace_ops_func_t ops_func; #endif }; @@ -340,6 +378,7 @@ unsigned long ftrace_find_rec_direct(unsigned long ip); int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr); int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr); int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr); +int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr); #else struct ftrace_ops; @@ -384,6 +423,10 @@ static inline int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned lo { return -ENODEV; } +static inline int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr) +{ + return -ENODEV; +} #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS diff --git a/include/linux/inetdevice.h b/include/linux/inetdevice.h index ead323243e7b..ddb27fc0ee8c 100644 --- a/include/linux/inetdevice.h +++ b/include/linux/inetdevice.h @@ -131,7 +131,7 @@ static inline void ipv4_devconf_setall(struct in_device *in_dev) IN_DEV_ORCONF((in_dev), IGNORE_ROUTES_WITH_LINKDOWN) #define IN_DEV_ARPFILTER(in_dev) IN_DEV_ORCONF((in_dev), ARPFILTER) -#define IN_DEV_ARP_ACCEPT(in_dev) IN_DEV_ORCONF((in_dev), ARP_ACCEPT) +#define IN_DEV_ARP_ACCEPT(in_dev) IN_DEV_MAXCONF((in_dev), ARP_ACCEPT) #define IN_DEV_ARP_ANNOUNCE(in_dev) IN_DEV_MAXCONF((in_dev), ARP_ANNOUNCE) #define IN_DEV_ARP_IGNORE(in_dev) IN_DEV_MAXCONF((in_dev), ARP_IGNORE) #define IN_DEV_ARP_NOTIFY(in_dev) IN_DEV_MAXCONF((in_dev), ARP_NOTIFY) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 83cf7fd842e0..90a45ef7203b 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1822,6 +1822,15 @@ struct _kvm_stats_desc { STATS_DESC_PEAK(SCOPE, name, KVM_STATS_UNIT_NONE, \ KVM_STATS_BASE_POW10, 0) +/* Instantaneous boolean value, read only */ +#define STATS_DESC_IBOOLEAN(SCOPE, name) \ + STATS_DESC_INSTANT(SCOPE, name, KVM_STATS_UNIT_BOOLEAN, \ + KVM_STATS_BASE_POW10, 0) +/* Peak (sticky) boolean value, read/write */ +#define STATS_DESC_PBOOLEAN(SCOPE, name) \ + STATS_DESC_PEAK(SCOPE, name, KVM_STATS_UNIT_BOOLEAN, \ + KVM_STATS_BASE_POW10, 0) + /* Cumulative time in nanosecond */ #define STATS_DESC_TIME_NSEC(SCOPE, name) \ STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_SECONDS, \ @@ -1853,7 +1862,7 @@ struct _kvm_stats_desc { HALT_POLL_HIST_COUNT), \ STATS_DESC_LOGHIST_TIME_NSEC(VCPU_GENERIC, halt_wait_hist, \ HALT_POLL_HIST_COUNT), \ - STATS_DESC_ICOUNTER(VCPU_GENERIC, blocking) + STATS_DESC_IBOOLEAN(VCPU_GENERIC, blocking) extern struct dentry *kvm_debugfs_dir; diff --git a/include/linux/lapb.h b/include/linux/lapb.h index eb56472f23b2..b5333f9413dc 100644 --- a/include/linux/lapb.h +++ b/include/linux/lapb.h @@ -6,6 +6,11 @@ #ifndef LAPB_KERNEL_H #define LAPB_KERNEL_H +#include <linux/skbuff.h> +#include <linux/timer.h> + +struct net_device; + #define LAPB_OK 0 #define LAPB_BADTOKEN 1 #define LAPB_INVALUE 2 diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 254cc22f5eec..51b4e71017ee 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1833,7 +1833,11 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { u8 sw_vhca_id[0xe]; u8 reserved_at_230[0x10]; - u8 reserved_at_240[0x5c0]; + u8 reserved_at_240[0xb]; + u8 ts_cqe_metadata_size2wqe_counter[0x5]; + u8 reserved_at_250[0x10]; + + u8 reserved_at_260[0x5a0]; }; enum mlx5_ifc_flow_destination_type { diff --git a/include/linux/netfilter/nf_conntrack_h323.h b/include/linux/netfilter/nf_conntrack_h323.h index 4561ec0fcea4..9e937f64a1ad 100644 --- a/include/linux/netfilter/nf_conntrack_h323.h +++ b/include/linux/netfilter/nf_conntrack_h323.h @@ -38,60 +38,63 @@ void nf_conntrack_h245_expect(struct nf_conn *new, struct nf_conntrack_expect *this); void nf_conntrack_q931_expect(struct nf_conn *new, struct nf_conntrack_expect *this); -extern int (*set_h245_addr_hook) (struct sk_buff *skb, unsigned int protoff, - unsigned char **data, int dataoff, - H245_TransportAddress *taddr, - union nf_inet_addr *addr, - __be16 port); -extern int (*set_h225_addr_hook) (struct sk_buff *skb, unsigned int protoff, - unsigned char **data, int dataoff, - TransportAddress *taddr, - union nf_inet_addr *addr, - __be16 port); -extern int (*set_sig_addr_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, unsigned char **data, - TransportAddress *taddr, int count); -extern int (*set_ras_addr_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, unsigned char **data, - TransportAddress *taddr, int count); -extern int (*nat_rtp_rtcp_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, unsigned char **data, - int dataoff, - H245_TransportAddress *taddr, - __be16 port, __be16 rtp_port, - struct nf_conntrack_expect *rtp_exp, - struct nf_conntrack_expect *rtcp_exp); -extern int (*nat_t120_hook) (struct sk_buff *skb, struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, + +struct nfct_h323_nat_hooks { + int (*set_h245_addr)(struct sk_buff *skb, unsigned int protoff, unsigned char **data, int dataoff, - H245_TransportAddress *taddr, __be16 port, - struct nf_conntrack_expect *exp); -extern int (*nat_h245_hook) (struct sk_buff *skb, struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, + H245_TransportAddress *taddr, + union nf_inet_addr *addr, __be16 port); + int (*set_h225_addr)(struct sk_buff *skb, unsigned int protoff, unsigned char **data, int dataoff, - TransportAddress *taddr, __be16 port, - struct nf_conntrack_expect *exp); -extern int (*nat_callforwarding_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, int dataoff, - TransportAddress *taddr, - __be16 port, - struct nf_conntrack_expect *exp); -extern int (*nat_q931_hook) (struct sk_buff *skb, struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, TransportAddress *taddr, - int idx, __be16 port, - struct nf_conntrack_expect *exp); + TransportAddress *taddr, + union nf_inet_addr *addr, __be16 port); + int (*set_sig_addr)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, unsigned char **data, + TransportAddress *taddr, int count); + int (*set_ras_addr)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, unsigned char **data, + TransportAddress *taddr, int count); + int (*nat_rtp_rtcp)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned char **data, int dataoff, + H245_TransportAddress *taddr, + __be16 port, __be16 rtp_port, + struct nf_conntrack_expect *rtp_exp, + struct nf_conntrack_expect *rtcp_exp); + int (*nat_t120)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned char **data, int dataoff, + H245_TransportAddress *taddr, __be16 port, + struct nf_conntrack_expect *exp); + int (*nat_h245)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned char **data, int dataoff, + TransportAddress *taddr, __be16 port, + struct nf_conntrack_expect *exp); + int (*nat_callforwarding)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned char **data, int dataoff, + TransportAddress *taddr, __be16 port, + struct nf_conntrack_expect *exp); + int (*nat_q931)(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned char **data, TransportAddress *taddr, int idx, + __be16 port, struct nf_conntrack_expect *exp); +}; +extern const struct nfct_h323_nat_hooks __rcu *nfct_h323_nat_hook; #endif diff --git a/include/linux/netfilter/nf_conntrack_sip.h b/include/linux/netfilter/nf_conntrack_sip.h index c620521c42bc..dbc614dfe0d5 100644 --- a/include/linux/netfilter/nf_conntrack_sip.h +++ b/include/linux/netfilter/nf_conntrack_sip.h @@ -164,7 +164,7 @@ struct nf_nat_sip_hooks { unsigned int medialen, union nf_inet_addr *rtp_addr); }; -extern const struct nf_nat_sip_hooks *nf_nat_sip_hooks; +extern const struct nf_nat_sip_hooks __rcu *nf_nat_sip_hooks; int ct_sip_parse_request(const struct nf_conn *ct, const char *dptr, unsigned int datalen, unsigned int *matchoff, diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 1773e5df8e65..1b18dfa52e48 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -214,7 +214,7 @@ struct netfs_request_ops { void (*issue_read)(struct netfs_io_subrequest *subreq); bool (*is_still_valid)(struct netfs_io_request *rreq); int (*check_write_begin)(struct file *file, loff_t pos, unsigned len, - struct folio *folio, void **_fsdata); + struct folio **foliop, void **_fsdata); void (*done)(struct netfs_io_request *rreq); }; diff --git a/include/linux/reset.h b/include/linux/reset.h index 8a21b5756c3e..514ddf003efc 100644 --- a/include/linux/reset.h +++ b/include/linux/reset.h @@ -731,7 +731,7 @@ static inline int __must_check devm_reset_control_bulk_get_optional_exclusive(struct device *dev, int num_rstcs, struct reset_control_bulk_data *rstcs) { - return __devm_reset_control_bulk_get(dev, num_rstcs, rstcs, true, false, true); + return __devm_reset_control_bulk_get(dev, num_rstcs, rstcs, false, true, true); } /** diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 657a0fc68a3f..fde258b3decd 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -390,6 +390,11 @@ static const bool earlycon_acpi_spcr_enable EARLYCON_USED_OR_UNUSED; static inline int setup_earlycon(char *buf) { return 0; } #endif +static inline bool uart_console_enabled(struct uart_port *port) +{ + return uart_console(port) && (port->cons->flags & CON_ENABLED); +} + struct uart_port *uart_get_console(struct uart_port *ports, int nr, struct console *c); int uart_parse_earlycon(char *p, unsigned char *iotype, resource_size_t *addr, diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index f6a27ab19202..ca8afa382bf2 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -509,10 +509,18 @@ enum { * charged to the kernel memory. */ SKBFL_PURE_ZEROCOPY = BIT(2), + + SKBFL_DONT_ORPHAN = BIT(3), + + /* page references are managed by the ubuf_info, so it's safe to + * use frags only up until ubuf_info is released + */ + SKBFL_MANAGED_FRAG_REFS = BIT(4), }; #define SKBFL_ZEROCOPY_FRAG (SKBFL_ZEROCOPY_ENABLE | SKBFL_SHARED_FRAG) -#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY) +#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \ + SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS) /* * The callback notifies userspace to release buffers when skb DMA is done in @@ -1596,13 +1604,14 @@ void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref); void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, bool success); -int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb, - struct iov_iter *from, size_t length); +int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, + struct sk_buff *skb, struct iov_iter *from, + size_t length); static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len) { - return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len); + return __zerocopy_sg_from_iter(msg, skb->sk, skb, &msg->msg_iter, len); } int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, @@ -1629,6 +1638,11 @@ static inline bool skb_zcopy_pure(const struct sk_buff *skb) return skb_shinfo(skb)->flags & SKBFL_PURE_ZEROCOPY; } +static inline bool skb_zcopy_managed(const struct sk_buff *skb) +{ + return skb_shinfo(skb)->flags & SKBFL_MANAGED_FRAG_REFS; +} + static inline bool skb_pure_zcopy_same(const struct sk_buff *skb1, const struct sk_buff *skb2) { @@ -1703,6 +1717,14 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success) } } +void __skb_zcopy_downgrade_managed(struct sk_buff *skb); + +static inline void skb_zcopy_downgrade_managed(struct sk_buff *skb) +{ + if (unlikely(skb_zcopy_managed(skb))) + __skb_zcopy_downgrade_managed(skb); +} + static inline void skb_mark_not_on_list(struct sk_buff *skb) { skb->next = NULL; @@ -2351,6 +2373,22 @@ static inline unsigned int skb_pagelen(const struct sk_buff *skb) return skb_headlen(skb) + __skb_pagelen(skb); } +static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo, + int i, struct page *page, + int off, int size) +{ + skb_frag_t *frag = &shinfo->frags[i]; + + /* + * Propagate page pfmemalloc to the skb if we can. The problem is + * that not all callers have unique ownership of the page but rely + * on page_is_pfmemalloc doing the right thing(tm). + */ + frag->bv_page = page; + frag->bv_offset = off; + skb_frag_size_set(frag, size); +} + /** * skb_len_add - adds a number to len fields of skb * @skb: buffer to add len to @@ -2379,17 +2417,7 @@ static inline void skb_len_add(struct sk_buff *skb, int delta) static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, struct page *page, int off, int size) { - skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; - - /* - * Propagate page pfmemalloc to the skb if we can. The problem is - * that not all callers have unique ownership of the page but rely - * on page_is_pfmemalloc doing the right thing(tm). - */ - frag->bv_page = page; - frag->bv_offset = off; - skb_frag_size_set(frag, size); - + __skb_fill_page_desc_noacc(skb_shinfo(skb), i, page, off, size); page = compound_head(page); if (page_is_pfmemalloc(page)) skb->pfmemalloc = true; @@ -2459,6 +2487,14 @@ static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) #endif /* NET_SKBUFF_DATA_USES_OFFSET */ +static inline void skb_assert_len(struct sk_buff *skb) +{ +#ifdef CONFIG_DEBUG_NET + if (WARN_ONCE(!skb->len, "%s\n", __func__)) + DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false); +#endif /* CONFIG_DEBUG_NET */ +} + /* * Add data to an sk_buff */ @@ -3019,8 +3055,7 @@ static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask) { if (likely(!skb_zcopy(skb))) return 0; - if (!skb_zcopy_is_nouarg(skb) && - skb_uarg(skb)->callback == msg_zerocopy_callback) + if (skb_shinfo(skb)->flags & SKBFL_DONT_ORPHAN) return 0; return skb_copy_ubufs(skb, gfp_mask); } @@ -3333,7 +3368,10 @@ static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle) */ static inline void skb_frag_unref(struct sk_buff *skb, int f) { - __skb_frag_unref(&skb_shinfo(skb)->frags[f], skb->pp_recycle); + struct skb_shared_info *shinfo = skb_shinfo(skb); + + if (!skb_zcopy_managed(skb)) + __skb_frag_unref(&shinfo->frags[f], skb->pp_recycle); } /** diff --git a/include/linux/socket.h b/include/linux/socket.h index 414b8c7bb8f7..d2f5c4d5a906 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -14,6 +14,8 @@ struct file; struct pid; struct cred; struct socket; +struct sock; +struct sk_buff; #define __sockaddr_check_size(size) \ BUILD_BUG_ON(((size) > sizeof(struct __kernel_sockaddr_storage))) @@ -69,6 +71,9 @@ struct msghdr { unsigned int msg_flags; /* flags on received message */ __kernel_size_t msg_controllen; /* ancillary data buffer length */ struct kiocb *msg_iocb; /* ptr to iocb for async requests */ + struct ubuf_info *msg_ubuf; + int (*sg_from_iter)(struct sock *sk, struct sk_buff *skb, + struct iov_iter *from, size_t length); }; struct user_msghdr { diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index 29917850f079..8df475db88c0 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -260,6 +260,7 @@ struct plat_stmmacenet_data { bool has_crossts; int int_snapshot_num; int ext_snapshot_num; + bool int_snapshot_en; bool ext_snapshot_en; bool multi_msi_en; int msi_mac_vec; diff --git a/include/linux/usb/cdc_ncm.h b/include/linux/usb/cdc_ncm.h index f7cb3ddce7fb..2d207cb4837d 100644 --- a/include/linux/usb/cdc_ncm.h +++ b/include/linux/usb/cdc_ncm.h @@ -53,8 +53,8 @@ #define USB_CDC_NCM_NDP32_LENGTH_MIN 0x20 /* Maximum NTB length */ -#define CDC_NCM_NTB_MAX_SIZE_TX 32768 /* bytes */ -#define CDC_NCM_NTB_MAX_SIZE_RX 32768 /* bytes */ +#define CDC_NCM_NTB_MAX_SIZE_TX 65536 /* bytes */ +#define CDC_NCM_NTB_MAX_SIZE_RX 65536 /* bytes */ /* Initial NTB length */ #define CDC_NCM_NTB_DEF_SIZE_TX 16384 /* bytes */ diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h index f742e50207fb..1c53c4c4d88f 100644 --- a/include/net/af_vsock.h +++ b/include/net/af_vsock.h @@ -10,6 +10,7 @@ #include <linux/kernel.h> #include <linux/workqueue.h> +#include <net/sock.h> #include <uapi/linux/vm_sockets.h> #include "vsock_addr.h" diff --git a/include/net/amt.h b/include/net/amt.h index 0e40c3d64fcf..c881bc8b673b 100644 --- a/include/net/amt.h +++ b/include/net/amt.h @@ -7,6 +7,9 @@ #include <linux/siphash.h> #include <linux/jhash.h> +#include <linux/netdevice.h> +#include <net/gro_cells.h> +#include <net/rtnetlink.h> enum amt_msg_type { AMT_MSG_DISCOVERY = 1, @@ -78,6 +81,15 @@ enum amt_status { #define AMT_STATUS_MAX (__AMT_STATUS_MAX - 1) +/* Gateway events only */ +enum amt_event { + AMT_EVENT_NONE, + AMT_EVENT_RECEIVE, + AMT_EVENT_SEND_DISCOVERY, + AMT_EVENT_SEND_REQUEST, + __AMT_EVENT_MAX, +}; + struct amt_header { #if defined(__LITTLE_ENDIAN_BITFIELD) u8 type:4, @@ -292,6 +304,12 @@ struct amt_group_node { struct hlist_head sources[]; }; +#define AMT_MAX_EVENTS 16 +struct amt_events { + enum amt_event event; + struct sk_buff *skb; +}; + struct amt_dev { struct net_device *dev; struct net_device *stream_dev; @@ -308,6 +326,7 @@ struct amt_dev { struct delayed_work req_wq; /* Protected by RTNL */ struct delayed_work secret_wq; + struct work_struct event_wq; /* AMT status */ enum amt_status status; /* Generated key */ @@ -345,6 +364,10 @@ struct amt_dev { /* Used only in gateway mode */ u64 mac:48, reserved:16; + /* AMT gateway side message handler queue */ + struct amt_events events[AMT_MAX_EVENTS]; + u8 event_idx; + u8 nr_events; }; #define AMT_TOS 0xc0 diff --git a/include/net/ax88796.h b/include/net/ax88796.h index 2ed23a368602..b658471f97f0 100644 --- a/include/net/ax88796.h +++ b/include/net/ax88796.h @@ -8,6 +8,8 @@ #ifndef __NET_AX88796_PLAT_H #define __NET_AX88796_PLAT_H +#include <linux/types.h> + struct sk_buff; struct net_device; struct platform_device; diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h index 6b48d9e2aab9..e72f3b247b5e 100644 --- a/include/net/bluetooth/bluetooth.h +++ b/include/net/bluetooth/bluetooth.h @@ -55,6 +55,8 @@ #define BTPROTO_CMTP 5 #define BTPROTO_HIDP 6 #define BTPROTO_AVDTP 7 +#define BTPROTO_ISO 8 +#define BTPROTO_LAST BTPROTO_ISO #define SOL_HCI 0 #define SOL_L2CAP 6 @@ -149,10 +151,51 @@ struct bt_voice { #define BT_MODE_LE_FLOWCTL 0x03 #define BT_MODE_EXT_FLOWCTL 0x04 -#define BT_PKT_STATUS 16 +#define BT_PKT_STATUS 16 #define BT_SCM_PKT_STATUS 0x03 +#define BT_ISO_QOS 17 + +#define BT_ISO_QOS_CIG_UNSET 0xff +#define BT_ISO_QOS_CIS_UNSET 0xff + +#define BT_ISO_QOS_BIG_UNSET 0xff +#define BT_ISO_QOS_BIS_UNSET 0xff + +struct bt_iso_io_qos { + __u32 interval; + __u16 latency; + __u16 sdu; + __u8 phy; + __u8 rtn; +}; + +struct bt_iso_qos { + union { + __u8 cig; + __u8 big; + }; + union { + __u8 cis; + __u8 bis; + }; + union { + __u8 sca; + __u8 sync_interval; + }; + __u8 packing; + __u8 framing; + struct bt_iso_io_qos in; + struct bt_iso_io_qos out; +}; + +#define BT_ISO_PHY_1M 0x01 +#define BT_ISO_PHY_2M 0x02 +#define BT_ISO_PHY_CODED 0x04 +#define BT_ISO_PHY_ANY (BT_ISO_PHY_1M | BT_ISO_PHY_2M | \ + BT_ISO_PHY_CODED) + #define BT_CODEC 19 struct bt_codec_caps { @@ -177,6 +220,8 @@ struct bt_codecs { #define BT_CODEC_TRANSPARENT 0x03 #define BT_CODEC_MSBC 0x05 +#define BT_ISO_BASE 20 + __printf(1, 2) void bt_info(const char *fmt, ...); __printf(1, 2) @@ -494,7 +539,7 @@ static inline struct sk_buff *bt_skb_sendmmsg(struct sock *sk, struct sk_buff *skb, **frag; skb = bt_skb_sendmsg(sk, msg, len, mtu, headroom, tailroom); - if (IS_ERR_OR_NULL(skb)) + if (IS_ERR(skb)) return skb; len -= skb->len; @@ -521,6 +566,7 @@ static inline struct sk_buff *bt_skb_sendmmsg(struct sock *sk, } int bt_to_errno(u16 code); +__u8 bt_status(int err); void hci_sock_set_flag(struct sock *sk, int nr); void hci_sock_clear_flag(struct sock *sk, int nr); @@ -558,6 +604,27 @@ static inline void sco_exit(void) } #endif +#if IS_ENABLED(CONFIG_BT_LE) +int iso_init(void); +int iso_exit(void); +bool iso_enabled(void); +#else +static inline int iso_init(void) +{ + return 0; +} + +static inline int iso_exit(void) +{ + return 0; +} + +static inline bool iso_enabled(void) +{ + return false; +} +#endif + int mgmt_init(void); void mgmt_exit(void); diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index fe7935be7dc4..cf29511b25a8 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -228,17 +228,6 @@ enum { */ HCI_QUIRK_VALID_LE_STATES, - /* When this quirk is set, then erroneous data reporting - * is ignored. This is mainly due to the fact that the HCI - * Read Default Erroneous Data Reporting command is advertised, - * but not supported; these controllers often reply with unknown - * command and tend to lock up randomly. Needing a hard reset. - * - * This quirk can be set before hci_register_dev is called or - * during the hdev->setup vendor callback. - */ - HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, - /* * When this quirk is set, then the hci_suspend_notifier is not * registered. This is intended for devices which drop completely @@ -327,6 +316,7 @@ enum { HCI_USER_CHANNEL, HCI_EXT_CONFIGURED, HCI_LE_ADV, + HCI_LE_PER_ADV, HCI_LE_SCAN, HCI_SSP_ENABLED, HCI_SC_ENABLED, @@ -349,6 +339,7 @@ enum { HCI_LE_SCAN_INTERRUPTED, HCI_WIDEBAND_SPEECH_ENABLED, HCI_EVENT_FILTER_CONFIGURED, + HCI_PA_SYNC, HCI_DUT_MODE, HCI_VENDOR_DIAG, @@ -361,6 +352,7 @@ enum { HCI_QUALITY_REPORT, HCI_OFFLOAD_CODECS_ENABLED, HCI_LE_SIMULTANEOUS_ROLES, + HCI_CMD_DRAIN_WORKQUEUE, __HCI_NUM_FLAGS, }; @@ -496,6 +488,7 @@ enum { #define LMP_EXT_INQ 0x01 #define LMP_SIMUL_LE_BR 0x02 #define LMP_SIMPLE_PAIR 0x08 +#define LMP_ERR_DATA_REPORTING 0x20 #define LMP_NO_FLUSH 0x40 #define LMP_LSTO 0x01 @@ -528,9 +521,11 @@ enum { #define HCI_LE_PHY_2M 0x01 #define HCI_LE_PHY_CODED 0x08 #define HCI_LE_EXT_ADV 0x10 +#define HCI_LE_PERIODIC_ADV 0x20 #define HCI_LE_CHAN_SEL_ALG2 0x40 #define HCI_LE_CIS_CENTRAL 0x10 #define HCI_LE_CIS_PERIPHERAL 0x20 +#define HCI_LE_ISO_BROADCASTER 0x40 /* Connection modes */ #define HCI_CM_ACTIVE 0x0000 @@ -1874,6 +1869,22 @@ struct hci_cp_le_ext_conn_param { __le16 max_ce_len; } __packed; +#define HCI_OP_LE_PA_CREATE_SYNC 0x2044 +struct hci_cp_le_pa_create_sync { + __u8 options; + __u8 sid; + __u8 addr_type; + bdaddr_t addr; + __le16 skip; + __le16 sync_timeout; + __u8 sync_cte_type; +} __packed; + +#define HCI_OP_LE_PA_TERM_SYNC 0x2046 +struct hci_cp_le_pa_term_sync { + __le16 handle; +} __packed; + #define HCI_OP_LE_READ_NUM_SUPPORTED_ADV_SETS 0x203b struct hci_rp_le_read_num_supported_adv_sets { __u8 status; @@ -1908,13 +1919,6 @@ struct hci_rp_le_set_ext_adv_params { __u8 tx_power; } __packed; -#define HCI_OP_LE_SET_EXT_ADV_ENABLE 0x2039 -struct hci_cp_le_set_ext_adv_enable { - __u8 enable; - __u8 num_of_sets; - __u8 data[]; -} __packed; - struct hci_cp_ext_adv_set { __u8 handle; __le16 duration; @@ -1941,6 +1945,37 @@ struct hci_cp_le_set_ext_scan_rsp_data { __u8 data[]; } __packed; +#define HCI_OP_LE_SET_EXT_ADV_ENABLE 0x2039 +struct hci_cp_le_set_ext_adv_enable { + __u8 enable; + __u8 num_of_sets; + __u8 data[]; +} __packed; + +#define HCI_OP_LE_SET_PER_ADV_PARAMS 0x203e +struct hci_cp_le_set_per_adv_params { + __u8 handle; + __le16 min_interval; + __le16 max_interval; + __le16 periodic_properties; +} __packed; + +#define HCI_MAX_PER_AD_LENGTH 252 + +#define HCI_OP_LE_SET_PER_ADV_DATA 0x203f +struct hci_cp_le_set_per_adv_data { + __u8 handle; + __u8 operation; + __u8 length; + __u8 data[]; +} __packed; + +#define HCI_OP_LE_SET_PER_ADV_ENABLE 0x2040 +struct hci_cp_le_set_per_adv_enable { + __u8 enable; + __u8 handle; +} __packed; + #define LE_SET_ADV_DATA_OP_COMPLETE 0x03 #define LE_SET_ADV_DATA_NO_FRAG 0x01 @@ -1998,7 +2033,7 @@ struct hci_rp_le_read_iso_tx_sync { struct hci_cis_params { __u8 cis_id; __le16 c_sdu; - __le16 p_pdu; + __le16 p_sdu; __u8 c_phy; __u8 p_phy; __u8 c_rtn; @@ -2009,7 +2044,7 @@ struct hci_cp_le_set_cig_params { __u8 cig_id; __u8 c_interval[3]; __u8 p_interval[3]; - __u8 wc_sca; + __u8 sca; __u8 packing; __u8 framing; __le16 c_latency; @@ -2052,6 +2087,73 @@ struct hci_cp_le_reject_cis { __u8 reason; } __packed; +#define HCI_OP_LE_CREATE_BIG 0x2068 +struct hci_bis { + __u8 sdu_interval[3]; + __le16 sdu; + __le16 latency; + __u8 rtn; + __u8 phy; + __u8 packing; + __u8 framing; + __u8 encryption; + __u8 bcode[16]; +} __packed; + +struct hci_cp_le_create_big { + __u8 handle; + __u8 adv_handle; + __u8 num_bis; + struct hci_bis bis; +} __packed; + +#define HCI_OP_LE_TERM_BIG 0x206a +struct hci_cp_le_term_big { + __u8 handle; + __u8 reason; +} __packed; + +#define HCI_OP_LE_BIG_CREATE_SYNC 0x206b +struct hci_cp_le_big_create_sync { + __u8 handle; + __le16 sync_handle; + __u8 encryption; + __u8 bcode[16]; + __u8 mse; + __le16 timeout; + __u8 num_bis; + __u8 bis[0]; +} __packed; + +#define HCI_OP_LE_BIG_TERM_SYNC 0x206c +struct hci_cp_le_big_term_sync { + __u8 handle; +} __packed; + +#define HCI_OP_LE_SETUP_ISO_PATH 0x206e +struct hci_cp_le_setup_iso_path { + __le16 handle; + __u8 direction; + __u8 path; + __u8 codec; + __le16 codec_cid; + __le16 codec_vid; + __u8 delay[3]; + __u8 codec_cfg_len; + __u8 codec_cfg[0]; +} __packed; + +struct hci_rp_le_setup_iso_path { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_SET_HOST_FEATURE 0x2074 +struct hci_cp_le_set_host_feature { + __u8 bit_number; + __u8 bit_value; +} __packed; + /* ---- HCI Events ---- */ struct hci_ev_status { __u8 status; @@ -2580,6 +2682,18 @@ struct hci_ev_le_ext_adv_report { struct hci_ev_le_ext_adv_info info[]; } __packed; +#define HCI_EV_LE_PA_SYNC_ESTABLISHED 0x0e +struct hci_ev_le_pa_sync_established { + __u8 status; + __le16 handle; + __u8 sid; + __u8 bdaddr_type; + bdaddr_t bdaddr; + __u8 phy; + __le16 interval; + __u8 clock_accuracy; +} __packed; + #define HCI_EV_LE_ENHANCED_CONN_COMPLETE 0x0a struct hci_ev_le_enh_conn_complete { __u8 status; @@ -2631,6 +2745,55 @@ struct hci_evt_le_cis_req { __u8 cis_id; } __packed; +#define HCI_EVT_LE_CREATE_BIG_COMPLETE 0x1b +struct hci_evt_le_create_big_complete { + __u8 status; + __u8 handle; + __u8 sync_delay[3]; + __u8 transport_delay[3]; + __u8 phy; + __u8 nse; + __u8 bn; + __u8 pto; + __u8 irc; + __le16 max_pdu; + __le16 interval; + __u8 num_bis; + __le16 bis_handle[]; +} __packed; + +#define HCI_EVT_LE_BIG_SYNC_ESTABILISHED 0x1d +struct hci_evt_le_big_sync_estabilished { + __u8 status; + __u8 handle; + __u8 latency[3]; + __u8 nse; + __u8 bn; + __u8 pto; + __u8 irc; + __le16 max_pdu; + __le16 interval; + __u8 num_bis; + __le16 bis[]; +} __packed; + +#define HCI_EVT_LE_BIG_INFO_ADV_REPORT 0x22 +struct hci_evt_le_big_info_adv_report { + __le16 sync_handle; + __u8 num_bis; + __u8 nse; + __le16 iso_interval; + __u8 bn; + __u8 pto; + __u8 irc; + __le16 max_pdu; + __u8 sdu_interval[3]; + __le16 max_sdu; + __u8 phy; + __u8 framing; + __u8 encryption; +} __packed; + #define HCI_EV_VENDOR 0xff /* Internal events generated by Bluetooth stack */ diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index c0ea2a4892b1..e7862903187d 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -126,6 +126,7 @@ struct hci_conn_hash { unsigned int acl_num; unsigned int amp_num; unsigned int sco_num; + unsigned int iso_num; unsigned int le_num; unsigned int le_num_peripheral; }; @@ -234,8 +235,9 @@ struct oob_data { struct adv_info { struct list_head list; - bool enabled; - bool pending; + bool enabled; + bool pending; + bool periodic; __u8 instance; __u32 flags; __u16 timeout; @@ -243,8 +245,12 @@ struct adv_info { __u16 duration; __u16 adv_data_len; __u8 adv_data[HCI_MAX_EXT_AD_LENGTH]; + bool adv_data_changed; __u16 scan_rsp_len; __u8 scan_rsp_data[HCI_MAX_EXT_AD_LENGTH]; + bool scan_rsp_changed; + __u16 per_adv_data_len; + __u8 per_adv_data[HCI_MAX_PER_AD_LENGTH]; __s8 tx_power; __u32 min_interval; __u32 max_interval; @@ -258,6 +264,15 @@ struct adv_info { #define HCI_ADV_TX_POWER_NO_PREFERENCE 0x7F +#define DATA_CMP(_d1, _l1, _d2, _l2) \ + (_l1 == _l2 ? memcmp(_d1, _d2, _l1) : _l1 - _l2) + +#define ADV_DATA_CMP(_adv, _data, _len) \ + DATA_CMP((_adv)->adv_data, (_adv)->adv_data_len, _data, _len) + +#define SCAN_RSP_CMP(_adv, _data, _len) \ + DATA_CMP((_adv)->scan_rsp_data, (_adv)->scan_rsp_len, _data, _len) + struct monitored_device { struct list_head list; @@ -463,13 +478,16 @@ struct hci_dev { unsigned int acl_cnt; unsigned int sco_cnt; unsigned int le_cnt; + unsigned int iso_cnt; unsigned int acl_mtu; unsigned int sco_mtu; unsigned int le_mtu; + unsigned int iso_mtu; unsigned int acl_pkts; unsigned int sco_pkts; unsigned int le_pkts; + unsigned int iso_pkts; __u16 block_len; __u16 block_mtu; @@ -506,8 +524,6 @@ struct hci_dev { struct work_struct cmd_work; struct work_struct tx_work; - struct work_struct discov_update; - struct work_struct scan_update; struct delayed_work le_scan_disable; struct delayed_work le_scan_restart; @@ -516,6 +532,7 @@ struct hci_dev { struct sk_buff_head cmd_q; struct sk_buff *sent_cmd; + struct sk_buff *recv_event; struct mutex req_lock; wait_queue_head_t req_wait_q; @@ -580,6 +597,8 @@ struct hci_dev { __u8 adv_data_len; __u8 scan_rsp_data[HCI_MAX_EXT_AD_LENGTH]; __u8 scan_rsp_data_len; + __u8 per_adv_data[HCI_MAX_PER_AD_LENGTH]; + __u8 per_adv_data_len; struct list_head adv_instances; unsigned int adv_instance_cnt; @@ -647,6 +666,7 @@ enum conn_reasons { CONN_REASON_PAIR_DEVICE, CONN_REASON_L2CAP_CHAN, CONN_REASON_SCO_CONNECT, + CONN_REASON_ISO_CONNECT, }; struct hci_conn { @@ -664,6 +684,7 @@ struct hci_conn { __u8 resp_addr_type; __u8 adv_instance; __u16 handle; + __u16 sync_handle; __u16 state; __u8 mode; __u8 type; @@ -694,11 +715,14 @@ struct hci_conn { __u16 le_supv_timeout; __u8 le_adv_data[HCI_MAX_AD_LENGTH]; __u8 le_adv_data_len; + __u8 le_per_adv_data[HCI_MAX_PER_AD_LENGTH]; + __u8 le_per_adv_data_len; __u8 le_tx_phy; __u8 le_rx_phy; __s8 rssi; __s8 tx_power; __s8 max_tx_power; + struct bt_iso_qos iso_qos; unsigned long flags; enum conn_reasons conn_reason; @@ -729,6 +753,7 @@ struct hci_conn { struct hci_dev *hdev; void *l2cap_data; void *sco_data; + void *iso_data; struct amp_mgr *amp_mgr; struct hci_conn *link; @@ -737,6 +762,8 @@ struct hci_conn { void (*connect_cfm_cb) (struct hci_conn *conn, u8 status); void (*security_cfm_cb) (struct hci_conn *conn, u8 status); void (*disconn_cfm_cb) (struct hci_conn *conn, u8 reason); + + void (*cleanup)(struct hci_conn *conn); }; struct hci_chan { @@ -824,6 +851,21 @@ static inline void sco_recv_scodata(struct hci_conn *hcon, struct sk_buff *skb) } #endif +#if IS_ENABLED(CONFIG_BT_LE) +int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags); +void iso_recv(struct hci_conn *hcon, struct sk_buff *skb, u16 flags); +#else +static inline int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, + __u8 *flags) +{ + return 0; +} +static inline void iso_recv(struct hci_conn *hcon, struct sk_buff *skb, + u16 flags) +{ +} +#endif + /* ----- Inquiry cache ----- */ #define INQUIRY_CACHE_AGE_MAX (HZ*30) /* 30 seconds */ #define INQUIRY_ENTRY_AGE_MAX (HZ*60) /* 60 seconds */ @@ -908,6 +950,7 @@ enum { HCI_CONN_NEW_LINK_KEY, HCI_CONN_SCANNING, HCI_CONN_AUTH_FAILURE, + HCI_CONN_PER_ADV, }; static inline bool hci_conn_ssp_enabled(struct hci_conn *conn) @@ -944,6 +987,9 @@ static inline void hci_conn_hash_add(struct hci_dev *hdev, struct hci_conn *c) case ESCO_LINK: h->sco_num++; break; + case ISO_LINK: + h->iso_num++; + break; } } @@ -970,6 +1016,9 @@ static inline void hci_conn_hash_del(struct hci_dev *hdev, struct hci_conn *c) case ESCO_LINK: h->sco_num--; break; + case ISO_LINK: + h->iso_num--; + break; } } @@ -986,6 +1035,8 @@ static inline unsigned int hci_conn_num(struct hci_dev *hdev, __u8 type) case SCO_LINK: case ESCO_LINK: return h->sco_num; + case ISO_LINK: + return h->iso_num; default: return 0; } @@ -995,7 +1046,7 @@ static inline unsigned int hci_conn_count(struct hci_dev *hdev) { struct hci_conn_hash *c = &hdev->conn_hash; - return c->acl_num + c->amp_num + c->sco_num + c->le_num; + return c->acl_num + c->amp_num + c->sco_num + c->le_num + c->iso_num; } static inline __u8 hci_conn_lookup_type(struct hci_dev *hdev, __u16 handle) @@ -1018,6 +1069,29 @@ static inline __u8 hci_conn_lookup_type(struct hci_dev *hdev, __u16 handle) return type; } +static inline struct hci_conn *hci_conn_hash_lookup_bis(struct hci_dev *hdev, + bdaddr_t *ba, + __u8 big, __u8 bis) +{ + struct hci_conn_hash *h = &hdev->conn_hash; + struct hci_conn *c; + + rcu_read_lock(); + + list_for_each_entry_rcu(c, &h->list, list) { + if (bacmp(&c->dst, ba) || c->type != ISO_LINK) + continue; + + if (c->iso_qos.big == big && c->iso_qos.bis == bis) { + rcu_read_unlock(); + return c; + } + } + rcu_read_unlock(); + + return NULL; +} + static inline struct hci_conn *hci_conn_hash_lookup_handle(struct hci_dev *hdev, __u16 handle) { @@ -1081,6 +1155,76 @@ static inline struct hci_conn *hci_conn_hash_lookup_le(struct hci_dev *hdev, return NULL; } +static inline struct hci_conn *hci_conn_hash_lookup_cis(struct hci_dev *hdev, + bdaddr_t *ba, + __u8 ba_type) +{ + struct hci_conn_hash *h = &hdev->conn_hash; + struct hci_conn *c; + + rcu_read_lock(); + + list_for_each_entry_rcu(c, &h->list, list) { + if (c->type != ISO_LINK) + continue; + + if (ba_type == c->dst_type && !bacmp(&c->dst, ba)) { + rcu_read_unlock(); + return c; + } + } + + rcu_read_unlock(); + + return NULL; +} + +static inline struct hci_conn *hci_conn_hash_lookup_cig(struct hci_dev *hdev, + __u8 handle) +{ + struct hci_conn_hash *h = &hdev->conn_hash; + struct hci_conn *c; + + rcu_read_lock(); + + list_for_each_entry_rcu(c, &h->list, list) { + if (c->type != ISO_LINK) + continue; + + if (handle == c->iso_qos.cig) { + rcu_read_unlock(); + return c; + } + } + + rcu_read_unlock(); + + return NULL; +} + +static inline struct hci_conn *hci_conn_hash_lookup_big(struct hci_dev *hdev, + __u8 handle) +{ + struct hci_conn_hash *h = &hdev->conn_hash; + struct hci_conn *c; + + rcu_read_lock(); + + list_for_each_entry_rcu(c, &h->list, list) { + if (bacmp(&c->dst, BDADDR_ANY) || c->type != ISO_LINK) + continue; + + if (handle == c->iso_qos.big) { + rcu_read_unlock(); + return c; + } + } + + rcu_read_unlock(); + + return NULL; +} + static inline struct hci_conn *hci_conn_hash_lookup_state(struct hci_dev *hdev, __u8 type, __u16 state) { @@ -1101,6 +1245,27 @@ static inline struct hci_conn *hci_conn_hash_lookup_state(struct hci_dev *hdev, return NULL; } +typedef void (*hci_conn_func_t)(struct hci_conn *conn, void *data); +static inline void hci_conn_hash_list_state(struct hci_dev *hdev, + hci_conn_func_t func, __u8 type, + __u16 state, void *data) +{ + struct hci_conn_hash *h = &hdev->conn_hash; + struct hci_conn *c; + + if (!func) + return; + + rcu_read_lock(); + + list_for_each_entry_rcu(c, &h->list, list) { + if (c->type == type && c->state == state) + func(c, data); + } + + rcu_read_unlock(); +} + static inline struct hci_conn *hci_lookup_le_connect(struct hci_dev *hdev) { struct hci_conn_hash *h = &hdev->conn_hash; @@ -1124,6 +1289,8 @@ static inline struct hci_conn *hci_lookup_le_connect(struct hci_dev *hdev) int hci_disconnect(struct hci_conn *conn, __u8 reason); bool hci_setup_sync(struct hci_conn *conn, __u16 handle); void hci_sco_setup(struct hci_conn *conn, __u8 status); +bool hci_iso_setup_path(struct hci_conn *conn); +int hci_le_create_cis(struct hci_conn *conn); struct hci_conn *hci_conn_add(struct hci_dev *hdev, int type, bdaddr_t *dst, u8 role); @@ -1148,6 +1315,17 @@ struct hci_conn *hci_connect_acl(struct hci_dev *hdev, bdaddr_t *dst, enum conn_reasons conn_reason); struct hci_conn *hci_connect_sco(struct hci_dev *hdev, int type, bdaddr_t *dst, __u16 setting, struct bt_codec *codec); +struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst, + __u8 dst_type, struct bt_iso_qos *qos); +struct hci_conn *hci_connect_cis(struct hci_dev *hdev, bdaddr_t *dst, + __u8 dst_type, struct bt_iso_qos *qos); +struct hci_conn *hci_connect_bis(struct hci_dev *hdev, bdaddr_t *dst, + __u8 dst_type, struct bt_iso_qos *qos, + __u8 data_len, __u8 *data); +int hci_pa_create_sync(struct hci_dev *hdev, bdaddr_t *dst, __u8 dst_type, + __u8 sid); +int hci_le_big_create_sync(struct hci_dev *hdev, struct bt_iso_qos *qos, + __u16 sync_handle, __u8 num_bis, __u8 bis[]); int hci_conn_check_link_mode(struct hci_conn *conn); int hci_conn_check_secure(struct hci_conn *conn, __u8 sec_level); int hci_conn_security(struct hci_conn *conn, __u8 sec_level, __u8 auth_type, @@ -1286,6 +1464,8 @@ void hci_free_dev(struct hci_dev *hdev); int hci_register_dev(struct hci_dev *hdev); void hci_unregister_dev(struct hci_dev *hdev); void hci_release_dev(struct hci_dev *hdev); +int hci_register_suspend_notifier(struct hci_dev *hdev); +int hci_unregister_suspend_notifier(struct hci_dev *hdev); int hci_suspend_dev(struct hci_dev *hdev); int hci_resume_dev(struct hci_dev *hdev); int hci_reset_dev(struct hci_dev *hdev); @@ -1392,11 +1572,14 @@ int hci_remove_remote_oob_data(struct hci_dev *hdev, bdaddr_t *bdaddr, void hci_adv_instances_clear(struct hci_dev *hdev); struct adv_info *hci_find_adv_instance(struct hci_dev *hdev, u8 instance); struct adv_info *hci_get_next_instance(struct hci_dev *hdev, u8 instance); -int hci_add_adv_instance(struct hci_dev *hdev, u8 instance, u32 flags, - u16 adv_data_len, u8 *adv_data, - u16 scan_rsp_len, u8 *scan_rsp_data, - u16 timeout, u16 duration, s8 tx_power, - u32 min_interval, u32 max_interval); +struct adv_info *hci_add_adv_instance(struct hci_dev *hdev, u8 instance, + u32 flags, u16 adv_data_len, u8 *adv_data, + u16 scan_rsp_len, u8 *scan_rsp_data, + u16 timeout, u16 duration, s8 tx_power, + u32 min_interval, u32 max_interval); +struct adv_info *hci_add_per_instance(struct hci_dev *hdev, u8 instance, + u32 flags, u8 data_len, u8 *data, + u32 min_interval, u32 max_interval); int hci_set_adv_instance_data(struct hci_dev *hdev, u8 instance, u16 adv_data_len, u8 *adv_data, u16 scan_rsp_len, u8 *scan_rsp_data); @@ -1407,12 +1590,9 @@ bool hci_adv_instance_is_scannable(struct hci_dev *hdev, u8 instance); void hci_adv_monitors_clear(struct hci_dev *hdev); void hci_free_adv_monitor(struct hci_dev *hdev, struct adv_monitor *monitor); -int hci_add_adv_patterns_monitor_complete(struct hci_dev *hdev, u8 status); -int hci_remove_adv_monitor_complete(struct hci_dev *hdev, u8 status); -bool hci_add_adv_monitor(struct hci_dev *hdev, struct adv_monitor *monitor, - int *err); -bool hci_remove_single_adv_monitor(struct hci_dev *hdev, u16 handle, int *err); -bool hci_remove_all_adv_monitor(struct hci_dev *hdev, int *err); +int hci_add_adv_monitor(struct hci_dev *hdev, struct adv_monitor *monitor); +int hci_remove_single_adv_monitor(struct hci_dev *hdev, u16 handle); +int hci_remove_all_adv_monitor(struct hci_dev *hdev); bool hci_is_adv_monitoring(struct hci_dev *hdev); int hci_get_adv_monitor_offload_ext(struct hci_dev *hdev); @@ -1516,6 +1696,19 @@ void hci_conn_del_sysfs(struct hci_conn *conn); #define use_enhanced_conn_complete(dev) (ll_privacy_capable(dev) || \ ext_adv_capable(dev)) +/* Periodic advertising support */ +#define per_adv_capable(dev) (((dev)->le_features[1] & HCI_LE_PERIODIC_ADV)) + +/* CIS Master/Slave and BIS support */ +#define iso_capable(dev) (cis_capable(dev) || bis_capable(dev)) +#define cis_capable(dev) \ + (cis_central_capable(dev) || cis_peripheral_capable(dev)) +#define cis_central_capable(dev) \ + ((dev)->le_features[3] & HCI_LE_CIS_CENTRAL) +#define cis_peripheral_capable(dev) \ + ((dev)->le_features[3] & HCI_LE_CIS_PERIPHERAL) +#define bis_capable(dev) ((dev)->le_features[3] & HCI_LE_ISO_BROADCASTER) + /* ----- HCI protocols ----- */ #define HCI_PROTO_DEFER 0x01 @@ -1530,6 +1723,9 @@ static inline int hci_proto_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, case ESCO_LINK: return sco_connect_ind(hdev, bdaddr, flags); + case ISO_LINK: + return iso_connect_ind(hdev, bdaddr, flags); + default: BT_ERR("unknown link type %d", type); return -EINVAL; @@ -1737,8 +1933,10 @@ int hci_send_cmd(struct hci_dev *hdev, __u16 opcode, __u32 plen, const void *param); void hci_send_acl(struct hci_chan *chan, struct sk_buff *skb, __u16 flags); void hci_send_sco(struct hci_conn *conn, struct sk_buff *skb); +void hci_send_iso(struct hci_conn *conn, struct sk_buff *skb); void *hci_sent_cmd_data(struct hci_dev *hdev, __u16 opcode); +void *hci_recv_event_data(struct hci_dev *hdev, __u8 event); u32 hci_conn_get_phy(struct hci_conn *conn); @@ -1797,6 +1995,8 @@ void hci_mgmt_chan_unregister(struct hci_mgmt_chan *c); #define DISCOV_LE_RESTART_DELAY msecs_to_jiffies(200) /* msec */ #define DISCOV_LE_FAST_ADV_INT_MIN 0x00A0 /* 100 msec */ #define DISCOV_LE_FAST_ADV_INT_MAX 0x00F0 /* 150 msec */ +#define DISCOV_LE_PER_ADV_INT_MIN 0x00A0 /* 200 msec */ +#define DISCOV_LE_PER_ADV_INT_MAX 0x00A0 /* 200 msec */ #define NAME_RESOLVE_DURATION msecs_to_jiffies(10240) /* 10.24 sec */ @@ -1872,8 +2072,6 @@ void mgmt_advertising_removed(struct sock *sk, struct hci_dev *hdev, u8 instance); void mgmt_adv_monitor_removed(struct hci_dev *hdev, u16 handle); int mgmt_phy_configuration_changed(struct hci_dev *hdev, struct sock *skip); -int mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, u8 status); -int mgmt_remove_adv_monitor_complete(struct hci_dev *hdev, u8 status); void mgmt_adv_monitor_device_lost(struct hci_dev *hdev, u16 handle, bdaddr_t *bdaddr, u8 addr_type); diff --git a/include/net/bluetooth/hci_sock.h b/include/net/bluetooth/hci_sock.h index 9949870f7d78..0520e21ab698 100644 --- a/include/net/bluetooth/hci_sock.h +++ b/include/net/bluetooth/hci_sock.h @@ -124,6 +124,8 @@ struct hci_dev_info { __u16 acl_pkts; __u16 sco_mtu; __u16 sco_pkts; + __u16 iso_mtu; + __u16 iso_pkts; struct hci_dev_stats stat; }; diff --git a/include/net/bluetooth/hci_sync.h b/include/net/bluetooth/hci_sync.h index 2492e3b46a8f..3843f5060c73 100644 --- a/include/net/bluetooth/hci_sync.h +++ b/include/net/bluetooth/hci_sync.h @@ -65,6 +65,10 @@ int hci_enable_ext_advertising_sync(struct hci_dev *hdev, u8 instance); int hci_enable_advertising_sync(struct hci_dev *hdev); int hci_enable_advertising(struct hci_dev *hdev); +int hci_start_per_adv_sync(struct hci_dev *hdev, u8 instance, u8 data_len, + u8 *data, u32 flags, u16 min_interval, + u16 max_interval, u16 sync_interval); + int hci_remove_advertising_sync(struct hci_dev *hdev, struct sock *sk, u8 instance, bool force); int hci_disable_advertising_sync(struct hci_dev *hdev); @@ -78,10 +82,12 @@ int hci_read_clock_sync(struct hci_dev *hdev, struct hci_cp_read_clock *cp); int hci_write_fast_connectable_sync(struct hci_dev *hdev, bool enable); int hci_update_scan_sync(struct hci_dev *hdev); +int hci_update_scan(struct hci_dev *hdev); int hci_write_le_host_supported_sync(struct hci_dev *hdev, u8 le, u8 simul); int hci_remove_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance, struct sock *sk); +int hci_remove_ext_adv_instance(struct hci_dev *hdev, u8 instance); struct sk_buff *hci_read_local_oob_data_sync(struct hci_dev *hdev, bool ext, struct sock *sk); @@ -105,4 +111,14 @@ int hci_resume_sync(struct hci_dev *hdev); struct hci_conn; +int hci_abort_conn_sync(struct hci_dev *hdev, struct hci_conn *conn, u8 reason); + int hci_le_create_conn_sync(struct hci_dev *hdev, struct hci_conn *conn); + +int hci_le_remove_cig_sync(struct hci_dev *hdev, u8 handle); + +int hci_le_terminate_big_sync(struct hci_dev *hdev, u8 handle, u8 reason); + +int hci_le_big_terminate_sync(struct hci_dev *hdev, u8 handle); + +int hci_le_pa_terminate_sync(struct hci_dev *hdev, u16 handle); diff --git a/include/net/bluetooth/iso.h b/include/net/bluetooth/iso.h new file mode 100644 index 000000000000..3f4fe8b78e1b --- /dev/null +++ b/include/net/bluetooth/iso.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BlueZ - Bluetooth protocol stack for Linux + * + * Copyright (C) 2022 Intel Corporation + */ + +#ifndef __ISO_H +#define __ISO_H + +/* ISO defaults */ +#define ISO_DEFAULT_MTU 251 +#define ISO_MAX_NUM_BIS 0x1f + +/* ISO socket broadcast address */ +struct sockaddr_iso_bc { + bdaddr_t bc_bdaddr; + __u8 bc_bdaddr_type; + __u8 bc_sid; + __u8 bc_num_bis; + __u8 bc_bis[ISO_MAX_NUM_BIS]; +}; + +/* ISO socket address */ +struct sockaddr_iso { + sa_family_t iso_family; + bdaddr_t iso_bdaddr; + __u8 iso_bdaddr_type; + struct sockaddr_iso_bc iso_bc[]; +}; + +#endif /* __ISO_H */ diff --git a/include/net/bond_options.h b/include/net/bond_options.h index d2aea5cf1e41..69292ecc0325 100644 --- a/include/net/bond_options.h +++ b/include/net/bond_options.h @@ -7,6 +7,14 @@ #ifndef _NET_BOND_OPTIONS_H #define _NET_BOND_OPTIONS_H +#include <linux/bits.h> +#include <linux/limits.h> +#include <linux/types.h> +#include <linux/string.h> + +struct netlink_ext_ack; +struct nlattr; + #define BOND_OPT_MAX_NAMELEN 32 #define BOND_OPT_VALID(opt) ((opt) < BOND_OPT_LAST) #define BOND_MODE_ALL_EX(x) (~(x)) diff --git a/include/net/codel_qdisc.h b/include/net/codel_qdisc.h index 58b6d0ebea10..7d3d9219f4fe 100644 --- a/include/net/codel_qdisc.h +++ b/include/net/codel_qdisc.h @@ -49,6 +49,7 @@ * Implemented on linux by Dave Taht and Eric Dumazet */ +#include <net/codel.h> #include <net/pkt_sched.h> /* Qdiscs using codel plugin must use codel_skb_cb in their own cb[] */ diff --git a/include/net/datalink.h b/include/net/datalink.h index d9b7faaa539f..c837ffc7ebf8 100644 --- a/include/net/datalink.h +++ b/include/net/datalink.h @@ -2,6 +2,13 @@ #ifndef _NET_INET_DATALINK_H_ #define _NET_INET_DATALINK_H_ +#include <linux/list.h> + +struct llc_sap; +struct net_device; +struct packet_type; +struct sk_buff; + struct datalink_proto { unsigned char type[8]; diff --git a/include/net/dcbevent.h b/include/net/dcbevent.h index 43e34131a53f..02700262f71a 100644 --- a/include/net/dcbevent.h +++ b/include/net/dcbevent.h @@ -8,6 +8,8 @@ #ifndef _DCB_EVENT_H #define _DCB_EVENT_H +struct notifier_block; + enum dcbevent_notif_type { DCB_APP_EVENT = 1, }; diff --git a/include/net/dcbnl.h b/include/net/dcbnl.h index e4ad58c4062c..2b2d86fb3131 100644 --- a/include/net/dcbnl.h +++ b/include/net/dcbnl.h @@ -10,6 +10,8 @@ #include <linux/dcbnl.h> +struct net_device; + struct dcb_app_type { int ifindex; struct dcb_app app; diff --git a/include/net/devlink.h b/include/net/devlink.h index 88c701b375a2..780744b550b8 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -1517,6 +1517,7 @@ struct device *devlink_to_dev(const struct devlink *devlink); /* Devlink instance explicit locking */ void devl_lock(struct devlink *devlink); +int devl_trylock(struct devlink *devlink); void devl_unlock(struct devlink *devlink); void devl_assert_locked(struct devlink *devlink); bool devl_lock_is_held(struct devlink *devlink); @@ -1579,20 +1580,25 @@ void devlink_linecard_provision_clear(struct devlink_linecard *linecard); void devlink_linecard_provision_fail(struct devlink_linecard *linecard); void devlink_linecard_activate(struct devlink_linecard *linecard); void devlink_linecard_deactivate(struct devlink_linecard *linecard); +int devl_sb_register(struct devlink *devlink, unsigned int sb_index, + u32 size, u16 ingress_pools_count, + u16 egress_pools_count, u16 ingress_tc_count, + u16 egress_tc_count); int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, u32 size, u16 ingress_pools_count, u16 egress_pools_count, u16 ingress_tc_count, u16 egress_tc_count); +void devl_sb_unregister(struct devlink *devlink, unsigned int sb_index); void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index); -int devlink_dpipe_table_register(struct devlink *devlink, - const char *table_name, - struct devlink_dpipe_table_ops *table_ops, - void *priv, bool counter_control_extern); -void devlink_dpipe_table_unregister(struct devlink *devlink, - const char *table_name); -void devlink_dpipe_headers_register(struct devlink *devlink, - struct devlink_dpipe_headers *dpipe_headers); -void devlink_dpipe_headers_unregister(struct devlink *devlink); +int devl_dpipe_table_register(struct devlink *devlink, + const char *table_name, + struct devlink_dpipe_table_ops *table_ops, + void *priv, bool counter_control_extern); +void devl_dpipe_table_unregister(struct devlink *devlink, + const char *table_name); +void devl_dpipe_headers_register(struct devlink *devlink, + struct devlink_dpipe_headers *dpipe_headers); +void devl_dpipe_headers_unregister(struct devlink *devlink); bool devlink_dpipe_table_counter_enabled(struct devlink *devlink, const char *table_name); int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx); @@ -1608,23 +1614,40 @@ extern struct devlink_dpipe_header devlink_dpipe_header_ethernet; extern struct devlink_dpipe_header devlink_dpipe_header_ipv4; extern struct devlink_dpipe_header devlink_dpipe_header_ipv6; +int devl_resource_register(struct devlink *devlink, + const char *resource_name, + u64 resource_size, + u64 resource_id, + u64 parent_resource_id, + const struct devlink_resource_size_params *size_params); int devlink_resource_register(struct devlink *devlink, const char *resource_name, u64 resource_size, u64 resource_id, u64 parent_resource_id, const struct devlink_resource_size_params *size_params); +void devl_resources_unregister(struct devlink *devlink); void devlink_resources_unregister(struct devlink *devlink); -int devlink_resource_size_get(struct devlink *devlink, - u64 resource_id, - u64 *p_resource_size); +int devl_resource_size_get(struct devlink *devlink, + u64 resource_id, + u64 *p_resource_size); +int devl_dpipe_table_resource_set(struct devlink *devlink, + const char *table_name, u64 resource_id, + u64 resource_units); int devlink_dpipe_table_resource_set(struct devlink *devlink, const char *table_name, u64 resource_id, u64 resource_units); +void devl_resource_occ_get_register(struct devlink *devlink, + u64 resource_id, + devlink_resource_occ_get_t *occ_get, + void *occ_get_priv); void devlink_resource_occ_get_register(struct devlink *devlink, u64 resource_id, devlink_resource_occ_get_t *occ_get, void *occ_get_priv); +void devl_resource_occ_get_unregister(struct devlink *devlink, + u64 resource_id); + void devlink_resource_occ_get_unregister(struct devlink *devlink, u64 resource_id); int devlink_params_register(struct devlink *devlink, @@ -1642,6 +1665,10 @@ int devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id, int devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id, union devlink_param_value init_val); void devlink_param_value_changed(struct devlink *devlink, u32 param_id); +struct devlink_region *devl_region_create(struct devlink *devlink, + const struct devlink_region_ops *ops, + u32 region_max_snapshots, + u64 region_size); struct devlink_region * devlink_region_create(struct devlink *devlink, const struct devlink_region_ops *ops, @@ -1650,6 +1677,7 @@ struct devlink_region * devlink_port_region_create(struct devlink_port *port, const struct devlink_port_region_ops *ops, u32 region_max_snapshots, u64 region_size); +void devl_region_destroy(struct devlink_region *region); void devlink_region_destroy(struct devlink_region *region); void devlink_port_region_destroy(struct devlink_region *region); @@ -1745,9 +1773,15 @@ void devlink_flash_update_timeout_notify(struct devlink *devlink, const char *component, unsigned long timeout); +int devl_traps_register(struct devlink *devlink, + const struct devlink_trap *traps, + size_t traps_count, void *priv); int devlink_traps_register(struct devlink *devlink, const struct devlink_trap *traps, size_t traps_count, void *priv); +void devl_traps_unregister(struct devlink *devlink, + const struct devlink_trap *traps, + size_t traps_count); void devlink_traps_unregister(struct devlink *devlink, const struct devlink_trap *traps, size_t traps_count); @@ -1755,20 +1789,26 @@ void devlink_trap_report(struct devlink *devlink, struct sk_buff *skb, void *trap_ctx, struct devlink_port *in_devlink_port, const struct flow_action_cookie *fa_cookie); void *devlink_trap_ctx_priv(void *trap_ctx); +int devl_trap_groups_register(struct devlink *devlink, + const struct devlink_trap_group *groups, + size_t groups_count); int devlink_trap_groups_register(struct devlink *devlink, const struct devlink_trap_group *groups, size_t groups_count); +void devl_trap_groups_unregister(struct devlink *devlink, + const struct devlink_trap_group *groups, + size_t groups_count); void devlink_trap_groups_unregister(struct devlink *devlink, const struct devlink_trap_group *groups, size_t groups_count); int -devlink_trap_policers_register(struct devlink *devlink, - const struct devlink_trap_policer *policers, - size_t policers_count); +devl_trap_policers_register(struct devlink *devlink, + const struct devlink_trap_policer *policers, + size_t policers_count); void -devlink_trap_policers_unregister(struct devlink *devlink, - const struct devlink_trap_policer *policers, - size_t policers_count); +devl_trap_policers_unregister(struct devlink *devlink, + const struct devlink_trap_policer *policers, + size_t policers_count); #if IS_ENABLED(CONFIG_NET_DEVLINK) diff --git a/include/net/dn_dev.h b/include/net/dn_dev.h index 595b4f6c1eb1..bec303ea8367 100644 --- a/include/net/dn_dev.h +++ b/include/net/dn_dev.h @@ -2,6 +2,7 @@ #ifndef _NET_DN_DEV_H #define _NET_DN_DEV_H +#include <linux/netdevice.h> struct dn_dev; diff --git a/include/net/dn_fib.h b/include/net/dn_fib.h index ddd6565957b3..1929a3cd5ebe 100644 --- a/include/net/dn_fib.h +++ b/include/net/dn_fib.h @@ -4,6 +4,8 @@ #include <linux/netlink.h> #include <linux/refcount.h> +#include <linux/rtnetlink.h> +#include <net/fib_rules.h> extern const struct nla_policy rtm_dn_policy[]; diff --git a/include/net/dn_neigh.h b/include/net/dn_neigh.h index 2e3e7793973a..1f7df98bfc33 100644 --- a/include/net/dn_neigh.h +++ b/include/net/dn_neigh.h @@ -2,6 +2,8 @@ #ifndef _NET_DN_NEIGH_H #define _NET_DN_NEIGH_H +#include <net/neighbour.h> + /* * The position of the first two fields of * this structure are critical - SJW diff --git a/include/net/dn_nsp.h b/include/net/dn_nsp.h index f83932b864a9..a4a18fee0b7c 100644 --- a/include/net/dn_nsp.h +++ b/include/net/dn_nsp.h @@ -6,6 +6,12 @@ *******************************************************************************/ /* dn_nsp.c functions prototyping */ +#include <linux/atomic.h> +#include <linux/types.h> +#include <net/sock.h> + +struct sk_buff; +struct sk_buff_head; void dn_nsp_send_data_ack(struct sock *sk); void dn_nsp_send_oth_ack(struct sock *sk); diff --git a/include/net/dn_route.h b/include/net/dn_route.h index 6f1e94ac0bdf..88c0300236cc 100644 --- a/include/net/dn_route.h +++ b/include/net/dn_route.h @@ -7,6 +7,9 @@ *******************************************************************************/ +#include <linux/types.h> +#include <net/dst.h> + struct sk_buff *dn_alloc_skb(struct sock *sk, int size, gfp_t pri); int dn_route_output_sock(struct dst_entry __rcu **pprt, struct flowidn *, struct sock *sk, int flags); diff --git a/include/net/erspan.h b/include/net/erspan.h index 0d9e86bd9893..6cb4cbd6a48f 100644 --- a/include/net/erspan.h +++ b/include/net/erspan.h @@ -58,6 +58,9 @@ * GRE proto ERSPAN type I/II = 0x88BE, type III = 0x22EB */ +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/skbuff.h> #include <uapi/linux/erspan.h> #define ERSPAN_VERSION 0x1 /* ERSPAN type II */ diff --git a/include/net/esp.h b/include/net/esp.h index 9c5637d41d95..322950727dd0 100644 --- a/include/net/esp.h +++ b/include/net/esp.h @@ -5,6 +5,7 @@ #include <linux/skbuff.h> struct ip_esp_hdr; +struct xfrm_state; static inline struct ip_esp_hdr *ip_esp_hdr(const struct sk_buff *skb) { diff --git a/include/net/ethoc.h b/include/net/ethoc.h index 78519ed42ab4..73810f3ca492 100644 --- a/include/net/ethoc.h +++ b/include/net/ethoc.h @@ -10,6 +10,9 @@ #ifndef LINUX_NET_ETHOC_H #define LINUX_NET_ETHOC_H 1 +#include <linux/if.h> +#include <linux/types.h> + struct ethoc_platform_data { u8 hwaddr[IFHWADDRLEN]; s8 phy_id; diff --git a/include/net/firewire.h b/include/net/firewire.h index 299e5df38552..2442d645e412 100644 --- a/include/net/firewire.h +++ b/include/net/firewire.h @@ -2,6 +2,8 @@ #ifndef _NET_FIREWIRE_H #define _NET_FIREWIRE_H +#include <linux/types.h> + /* Pseudo L2 address */ #define FWNET_ALEN 16 union fwnet_hwaddr { diff --git a/include/net/fq.h b/include/net/fq.h index 2eccbbd2b559..07b5aff6ec58 100644 --- a/include/net/fq.h +++ b/include/net/fq.h @@ -7,6 +7,10 @@ #ifndef __NET_SCHED_FQ_H #define __NET_SCHED_FQ_H +#include <linux/skbuff.h> +#include <linux/spinlock.h> +#include <linux/types.h> + struct fq_tin; /** diff --git a/include/net/garp.h b/include/net/garp.h index 4d9a0c6a2e5f..59a07b171def 100644 --- a/include/net/garp.h +++ b/include/net/garp.h @@ -2,6 +2,8 @@ #ifndef _NET_GARP_H #define _NET_GARP_H +#include <linux/if_ether.h> +#include <linux/types.h> #include <net/stp.h> #define GARP_PROTOCOL_ID 0x1 diff --git a/include/net/gtp.h b/include/net/gtp.h index c1d6169df331..2a503f035d18 100644 --- a/include/net/gtp.h +++ b/include/net/gtp.h @@ -2,6 +2,10 @@ #ifndef _GTP_H_ #define _GTP_H_ +#include <linux/netdevice.h> +#include <linux/types.h> +#include <net/rtnetlink.h> + /* General GTP protocol related definitions. */ #define GTP0_PORT 3386 diff --git a/include/net/gue.h b/include/net/gue.h index e42402f180b7..dfca298bec9c 100644 --- a/include/net/gue.h +++ b/include/net/gue.h @@ -30,6 +30,9 @@ * may refer to options placed after this field. */ +#include <asm/byteorder.h> +#include <linux/types.h> + struct guehdr { union { struct { diff --git a/include/net/hwbm.h b/include/net/hwbm.h index c81444611a22..aa495decec35 100644 --- a/include/net/hwbm.h +++ b/include/net/hwbm.h @@ -2,6 +2,8 @@ #ifndef _HWBM_H #define _HWBM_H +#include <linux/mutex.h> + struct hwbm_pool { /* Capacity of the pool */ int size; diff --git a/include/net/ila.h b/include/net/ila.h index f98dcd5791b0..73ebe5eab272 100644 --- a/include/net/ila.h +++ b/include/net/ila.h @@ -8,6 +8,8 @@ #ifndef _NET_ILA_H #define _NET_ILA_H +struct sk_buff; + int ila_xlat_outgoing(struct sk_buff *skb); int ila_xlat_incoming(struct sk_buff *skb); diff --git a/include/net/inet6_connection_sock.h b/include/net/inet6_connection_sock.h index 7392f959a405..025bd8d3c769 100644 --- a/include/net/inet6_connection_sock.h +++ b/include/net/inet6_connection_sock.h @@ -11,6 +11,8 @@ #include <linux/types.h> +struct flowi; +struct flowi6; struct request_sock; struct sk_buff; struct sock; diff --git a/include/net/inet_common.h b/include/net/inet_common.h index cad2a611efde..cec453c18f1d 100644 --- a/include/net/inet_common.h +++ b/include/net/inet_common.h @@ -3,6 +3,10 @@ #define _INET_COMMON_H #include <linux/indirect_call_wrapper.h> +#include <linux/net.h> +#include <linux/netdev_features.h> +#include <linux/types.h> +#include <net/sock.h> extern const struct proto_ops inet_stream_ops; extern const struct proto_ops inet_dgram_ops; @@ -12,6 +16,8 @@ extern const struct proto_ops inet_dgram_ops; */ struct msghdr; +struct net; +struct page; struct sock; struct sockaddr; struct socket; diff --git a/include/net/inet_frag.h b/include/net/inet_frag.h index 911ad930867d..0b0876610553 100644 --- a/include/net/inet_frag.h +++ b/include/net/inet_frag.h @@ -4,6 +4,9 @@ #include <linux/rhashtable-types.h> #include <linux/completion.h> +#include <linux/in6.h> +#include <linux/rbtree_types.h> +#include <linux/refcount.h> /* Per netns frag queues directory */ struct fqdir { diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h index ebfa3df6f8dc..fd6b510d114b 100644 --- a/include/net/inet_hashtables.h +++ b/include/net/inet_hashtables.h @@ -179,7 +179,7 @@ static inline bool inet_sk_bound_dev_eq(struct net *net, int bound_dev_if, int dif, int sdif) { #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) - return inet_bound_dev_eq(!!net->ipv4.sysctl_tcp_l3mdev_accept, + return inet_bound_dev_eq(!!READ_ONCE(net->ipv4.sysctl_tcp_l3mdev_accept), bound_dev_if, dif, sdif); #else return inet_bound_dev_eq(true, bound_dev_if, dif, sdif); diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index daead5fb389a..6395f6b9a5d2 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -107,7 +107,8 @@ static inline struct inet_request_sock *inet_rsk(const struct request_sock *sk) static inline u32 inet_request_mark(const struct sock *sk, struct sk_buff *skb) { - if (!sk->sk_mark && sock_net(sk)->ipv4.sysctl_tcp_fwmark_accept) + if (!sk->sk_mark && + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fwmark_accept)) return skb->mark; return sk->sk_mark; @@ -120,7 +121,7 @@ static inline int inet_request_bound_dev_if(const struct sock *sk, #ifdef CONFIG_NET_L3_MASTER_DEV struct net *net = sock_net(sk); - if (!bound_dev_if && net->ipv4.sysctl_tcp_l3mdev_accept) + if (!bound_dev_if && READ_ONCE(net->ipv4.sysctl_tcp_l3mdev_accept)) return l3mdev_master_ifindex_by_index(net, skb->skb_iif); #endif @@ -132,7 +133,7 @@ static inline int inet_sk_bound_l3mdev(const struct sock *sk) #ifdef CONFIG_NET_L3_MASTER_DEV struct net *net = sock_net(sk); - if (!net->ipv4.sysctl_tcp_l3mdev_accept) + if (!READ_ONCE(net->ipv4.sysctl_tcp_l3mdev_accept)) return l3mdev_master_ifindex_by_index(net, sk->sk_bound_dev_if); #endif @@ -374,7 +375,7 @@ static inline bool inet_get_convert_csum(struct sock *sk) static inline bool inet_can_nonlocal_bind(struct net *net, struct inet_sock *inet) { - return net->ipv4.sysctl_ip_nonlocal_bind || + return READ_ONCE(net->ipv4.sysctl_ip_nonlocal_bind) || inet->freebind || inet->transparent; } diff --git a/include/net/ip.h b/include/net/ip.h index 26fffda78cca..1c979fd1904c 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -357,7 +357,7 @@ static inline bool sysctl_dev_name_is_allowed(const char *name) static inline bool inet_port_requires_bind_service(struct net *net, unsigned short port) { - return port < net->ipv4.sysctl_ip_prot_sock; + return port < READ_ONCE(net->ipv4.sysctl_ip_prot_sock); } #else @@ -384,7 +384,7 @@ void ipfrag_init(void); void ip_static_sysctl_init(void); #define IP4_REPLY_MARK(net, mark) \ - ((net)->ipv4.sysctl_fwmark_reflect ? (mark) : 0) + (READ_ONCE((net)->ipv4.sysctl_fwmark_reflect) ? (mark) : 0) static inline bool ip_is_fragment(const struct iphdr *iph) { @@ -446,7 +446,7 @@ static inline unsigned int ip_dst_mtu_maybe_forward(const struct dst_entry *dst, struct net *net = dev_net(dst->dev); unsigned int mtu; - if (net->ipv4.sysctl_ip_fwd_use_pmtu || + if (READ_ONCE(net->ipv4.sysctl_ip_fwd_use_pmtu) || ip_mtu_locked(dst) || !forwarding) { mtu = rt->rt_pmtu; diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index ca2d6b60e1ec..035d61d50a98 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -2,6 +2,16 @@ #ifndef _NET_IP6_ROUTE_H #define _NET_IP6_ROUTE_H +#include <net/addrconf.h> +#include <net/flow.h> +#include <net/ip6_fib.h> +#include <net/sock.h> +#include <net/lwtunnel.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/route.h> +#include <net/nexthop.h> + struct route_info { __u8 type; __u8 length; @@ -19,16 +29,6 @@ struct route_info { __u8 prefix[]; /* 0,8 or 16 */ }; -#include <net/addrconf.h> -#include <net/flow.h> -#include <net/ip6_fib.h> -#include <net/sock.h> -#include <net/lwtunnel.h> -#include <linux/ip.h> -#include <linux/ipv6.h> -#include <linux/route.h> -#include <net/nexthop.h> - #define RT6_LOOKUP_F_IFACE 0x00000001 #define RT6_LOOKUP_F_REACHABLE 0x00000002 #define RT6_LOOKUP_F_HAS_SADDR 0x00000004 diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index 70cbc4a72669..20db95055db3 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -387,9 +387,11 @@ static inline int ip_tunnel_encap(struct sk_buff *skb, struct ip_tunnel *t, static inline u8 ip_tunnel_get_dsfield(const struct iphdr *iph, const struct sk_buff *skb) { - if (skb->protocol == htons(ETH_P_IP)) + __be16 payload_protocol = skb_protocol(skb, true); + + if (payload_protocol == htons(ETH_P_IP)) return iph->tos; - else if (skb->protocol == htons(ETH_P_IPV6)) + else if (payload_protocol == htons(ETH_P_IPV6)) return ipv6_get_dsfield((const struct ipv6hdr *)iph); else return 0; @@ -398,9 +400,11 @@ static inline u8 ip_tunnel_get_dsfield(const struct iphdr *iph, static inline u8 ip_tunnel_get_ttl(const struct iphdr *iph, const struct sk_buff *skb) { - if (skb->protocol == htons(ETH_P_IP)) + __be16 payload_protocol = skb_protocol(skb, true); + + if (payload_protocol == htons(ETH_P_IP)) return iph->ttl; - else if (skb->protocol == htons(ETH_P_IPV6)) + else if (payload_protocol == htons(ETH_P_IPV6)) return ((const struct ipv6hdr *)iph)->hop_limit; else return 0; diff --git a/include/net/ipcomp.h b/include/net/ipcomp.h index fee6fc451597..c31108295079 100644 --- a/include/net/ipcomp.h +++ b/include/net/ipcomp.h @@ -2,11 +2,13 @@ #ifndef _NET_IPCOMP_H #define _NET_IPCOMP_H +#include <linux/skbuff.h> #include <linux/types.h> #define IPCOMP_SCRATCH_SIZE 65400 struct crypto_comp; +struct ip_comp_hdr; struct ipcomp_data { u16 threshold; diff --git a/include/net/ipconfig.h b/include/net/ipconfig.h index e3534299bd2a..8276897d0c2e 100644 --- a/include/net/ipconfig.h +++ b/include/net/ipconfig.h @@ -7,6 +7,8 @@ /* The following are initdata: */ +#include <linux/types.h> + extern int ic_proto_enabled; /* Protocols enabled (see IC_xxx) */ extern int ic_set_manually; /* IPconfig parameters set manually */ diff --git a/include/net/llc_c_ac.h b/include/net/llc_c_ac.h index e766300b3e99..3e1f76786d7b 100644 --- a/include/net/llc_c_ac.h +++ b/include/net/llc_c_ac.h @@ -16,6 +16,13 @@ * Connection state transition actions * (Fb = F bit; Pb = P bit; Xb = X bit) */ + +#include <linux/types.h> + +struct sk_buff; +struct sock; +struct timer_list; + #define LLC_CONN_AC_CLR_REMOTE_BUSY 1 #define LLC_CONN_AC_CONN_IND 2 #define LLC_CONN_AC_CONN_CONFIRM 3 diff --git a/include/net/llc_c_st.h b/include/net/llc_c_st.h index 48f3f891b2f9..53823d61d8b6 100644 --- a/include/net/llc_c_st.h +++ b/include/net/llc_c_st.h @@ -11,6 +11,10 @@ * * See the GNU General Public License for more details. */ + +#include <net/llc_c_ac.h> +#include <net/llc_c_ev.h> + /* Connection component state management */ /* connection states */ #define LLC_CONN_OUT_OF_SVC 0 /* prior to allocation */ diff --git a/include/net/llc_s_ac.h b/include/net/llc_s_ac.h index a61b98c108ee..f71790305bc9 100644 --- a/include/net/llc_s_ac.h +++ b/include/net/llc_s_ac.h @@ -11,6 +11,10 @@ * * See the GNU General Public License for more details. */ + +struct llc_sap; +struct sk_buff; + /* SAP component actions */ #define SAP_ACT_UNITDATA_IND 1 #define SAP_ACT_SEND_UI 2 diff --git a/include/net/llc_s_ev.h b/include/net/llc_s_ev.h index 84db3a59ed28..fb7df1d70af3 100644 --- a/include/net/llc_s_ev.h +++ b/include/net/llc_s_ev.h @@ -13,6 +13,7 @@ */ #include <linux/skbuff.h> +#include <net/llc.h> /* Defines SAP component events */ /* Types of events (possible values in 'ev->type') */ diff --git a/include/net/mpls_iptunnel.h b/include/net/mpls_iptunnel.h index 9deb3a3735da..0c71c27979fb 100644 --- a/include/net/mpls_iptunnel.h +++ b/include/net/mpls_iptunnel.h @@ -6,6 +6,9 @@ #ifndef _NET_MPLS_IPTUNNEL_H #define _NET_MPLS_IPTUNNEL_H 1 +#include <linux/types.h> +#include <net/lwtunnel.h> + struct mpls_iptunnel_encap { u8 labels; u8 ttl_propagate; diff --git a/include/net/mrp.h b/include/net/mrp.h index 1c308c034e1a..92cd3fb6cf9d 100644 --- a/include/net/mrp.h +++ b/include/net/mrp.h @@ -2,6 +2,10 @@ #ifndef _NET_MRP_H #define _NET_MRP_H +#include <linux/netdevice.h> +#include <linux/skbuff.h> +#include <linux/types.h> + #define MRP_END_MARK 0x0 struct mrp_pdu_hdr { diff --git a/include/net/ncsi.h b/include/net/ncsi.h index fbefe80361ee..08a50d9acb0a 100644 --- a/include/net/ncsi.h +++ b/include/net/ncsi.h @@ -2,6 +2,8 @@ #ifndef __NET_NCSI_H #define __NET_NCSI_H +#include <linux/types.h> + /* * The NCSI device states seen from external. More NCSI device states are * only visible internally (in net/ncsi/internal.h). When the NCSI device diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h index 20a2992901c2..8c3587d5c308 100644 --- a/include/net/net_namespace.h +++ b/include/net/net_namespace.h @@ -26,6 +26,9 @@ #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) #include <net/netns/conntrack.h> #endif +#if IS_ENABLED(CONFIG_NF_FLOW_TABLE) +#include <net/netns/flow_table.h> +#endif #include <net/netns/nftables.h> #include <net/netns/xfrm.h> #include <net/netns/mpls.h> @@ -142,6 +145,9 @@ struct net { #if defined(CONFIG_NF_TABLES) || defined(CONFIG_NF_TABLES_MODULE) struct netns_nftables nft; #endif +#if IS_ENABLED(CONFIG_NF_FLOW_TABLE) + struct netns_ft ft; +#endif #endif #ifdef CONFIG_WEXT_CORE struct sk_buff_head wext_nlevents; diff --git a/include/net/netevent.h b/include/net/netevent.h index 4107016c3bb4..1be3757a8b7f 100644 --- a/include/net/netevent.h +++ b/include/net/netevent.h @@ -14,6 +14,7 @@ struct dst_entry; struct neighbour; +struct notifier_block ; struct netevent_redirect { struct dst_entry *old; diff --git a/include/net/netfilter/nf_conntrack_core.h b/include/net/netfilter/nf_conntrack_core.h index 37866c8386e2..3cd3a6e631aa 100644 --- a/include/net/netfilter/nf_conntrack_core.h +++ b/include/net/netfilter/nf_conntrack_core.h @@ -84,4 +84,23 @@ void nf_conntrack_lock(spinlock_t *lock); extern spinlock_t nf_conntrack_expect_lock; +/* ctnetlink code shared by both ctnetlink and nf_conntrack_bpf */ + +#if (IS_BUILTIN(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \ + (IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES) || \ + IS_ENABLED(CONFIG_NF_CT_NETLINK)) + +static inline void __nf_ct_set_timeout(struct nf_conn *ct, u64 timeout) +{ + if (timeout > INT_MAX) + timeout = INT_MAX; + WRITE_ONCE(ct->timeout, nfct_time_stamp + (u32)timeout); +} + +int __nf_ct_change_timeout(struct nf_conn *ct, u64 cta_timeout); +void __nf_ct_change_status(struct nf_conn *ct, unsigned long on, unsigned long off); +int nf_ct_change_status_common(struct nf_conn *ct, unsigned int status); + +#endif + #endif /* _NF_CONNTRACK_CORE_H */ diff --git a/include/net/netfilter/nf_conntrack_timeout.h b/include/net/netfilter/nf_conntrack_timeout.h index fea258983d23..9fdaba911de6 100644 --- a/include/net/netfilter/nf_conntrack_timeout.h +++ b/include/net/netfilter/nf_conntrack_timeout.h @@ -105,7 +105,7 @@ struct nf_ct_timeout_hooks { void (*timeout_put)(struct nf_ct_timeout *timeout); }; -extern const struct nf_ct_timeout_hooks *nf_ct_timeout_hook; +extern const struct nf_ct_timeout_hooks __rcu *nf_ct_timeout_hook; #endif #endif /* _NF_CONNTRACK_TIMEOUT_H */ diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h index 64daafd1fc41..d5326c44b453 100644 --- a/include/net/netfilter/nf_flow_table.h +++ b/include/net/netfilter/nf_flow_table.h @@ -335,4 +335,25 @@ static inline __be16 nf_flow_pppoe_proto(const struct sk_buff *skb) return 0; } +#define NF_FLOW_TABLE_STAT_INC(net, count) __this_cpu_inc((net)->ft.stat->count) +#define NF_FLOW_TABLE_STAT_DEC(net, count) __this_cpu_dec((net)->ft.stat->count) +#define NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count) \ + this_cpu_inc((net)->ft.stat->count) +#define NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count) \ + this_cpu_dec((net)->ft.stat->count) + +#ifdef CONFIG_NF_FLOW_TABLE_PROCFS +int nf_flow_table_init_proc(struct net *net); +void nf_flow_table_fini_proc(struct net *net); +#else +static inline int nf_flow_table_init_proc(struct net *net) +{ + return 0; +} + +static inline void nf_flow_table_fini_proc(struct net *net) +{ +} +#endif /* CONFIG_NF_FLOW_TABLE_PROCFS */ + #endif /* _NF_FLOW_TABLE_H */ diff --git a/include/net/netfilter/nf_nat.h b/include/net/netfilter/nf_nat.h index 987111ae5240..e9eb01e99d2f 100644 --- a/include/net/netfilter/nf_nat.h +++ b/include/net/netfilter/nf_nat.h @@ -104,7 +104,7 @@ unsigned int nf_nat_inet_fn(void *priv, struct sk_buff *skb, const struct nf_hook_state *state); -static inline int nf_nat_initialized(struct nf_conn *ct, +static inline int nf_nat_initialized(const struct nf_conn *ct, enum nf_nat_manip_type manip) { if (manip == NF_NAT_MANIP_SRC) diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index 64cf655c818c..8bfb9c74afbf 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -157,11 +157,26 @@ static inline void nft_reg_store16(u32 *dreg, u16 val) *(u16 *)dreg = val; } +static inline void nft_reg_store_be16(u32 *dreg, __be16 val) +{ + nft_reg_store16(dreg, (__force __u16)val); +} + static inline u16 nft_reg_load16(const u32 *sreg) { return *(u16 *)sreg; } +static inline __be16 nft_reg_load_be16(const u32 *sreg) +{ + return (__force __be16)nft_reg_load16(sreg); +} + +static inline __be32 nft_reg_load_be32(const u32 *sreg) +{ + return *(__force __be32 *)sreg; +} + static inline void nft_reg_store64(u32 *dreg, u64 val) { put_unaligned(val, (u64 *)dreg); diff --git a/include/net/netfilter/nf_tables_core.h b/include/net/netfilter/nf_tables_core.h index 0ea7c55cea4d..1223af68cd9a 100644 --- a/include/net/netfilter/nf_tables_core.h +++ b/include/net/netfilter/nf_tables_core.h @@ -56,16 +56,6 @@ struct nft_immediate_expr { u8 dlen; }; -/* Calculate the mask for the nft_cmp_fast expression. On big endian the - * mask needs to include the *upper* bytes when interpreting that data as - * something smaller than the full u32, therefore a cpu_to_le32 is done. - */ -static inline u32 nft_cmp_fast_mask(unsigned int len) -{ - return cpu_to_le32(~0U >> (sizeof_field(struct nft_cmp_fast_expr, - data) * BITS_PER_BYTE - len)); -} - extern const struct nft_expr_ops nft_cmp_fast_ops; extern const struct nft_expr_ops nft_cmp16_fast_ops; diff --git a/include/net/netns/can.h b/include/net/netns/can.h index 52fbd8291a96..48b79f7e6236 100644 --- a/include/net/netns/can.h +++ b/include/net/netns/can.h @@ -7,6 +7,7 @@ #define __NETNS_CAN_H__ #include <linux/spinlock.h> +#include <linux/timer.h> struct can_dev_rcv_lists; struct can_pkg_stats; diff --git a/include/net/netns/core.h b/include/net/netns/core.h index 388244e315e7..8249060cf5d0 100644 --- a/include/net/netns/core.h +++ b/include/net/netns/core.h @@ -2,6 +2,8 @@ #ifndef __NETNS_CORE_H__ #define __NETNS_CORE_H__ +#include <linux/types.h> + struct ctl_table_header; struct prot_inuse; diff --git a/include/net/netns/flow_table.h b/include/net/netns/flow_table.h new file mode 100644 index 000000000000..1c5fc657e267 --- /dev/null +++ b/include/net/netns/flow_table.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NETNS_FLOW_TABLE_H +#define __NETNS_FLOW_TABLE_H + +struct nf_flow_table_stat { + unsigned int count_wq_add; + unsigned int count_wq_del; + unsigned int count_wq_stats; +}; + +struct netns_ft { + struct nf_flow_table_stat __percpu *stat; +}; +#endif diff --git a/include/net/netns/generic.h b/include/net/netns/generic.h index 8a1ab47c3fb3..7ce68183f6e1 100644 --- a/include/net/netns/generic.h +++ b/include/net/netns/generic.h @@ -8,6 +8,7 @@ #include <linux/bug.h> #include <linux/rcupdate.h> +#include <net/net_namespace.h> /* * Generic net pointers are to be used by modules to put some private diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index ce0cc4e8d8c7..c7320ef356d9 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -9,6 +9,7 @@ #include <linux/uidgid.h> #include <net/inet_frag.h> #include <linux/rcupdate.h> +#include <linux/seqlock.h> #include <linux/siphash.h> struct ctl_table_header; diff --git a/include/net/netns/mctp.h b/include/net/netns/mctp.h index acedef12a35e..1db8f9aaddb4 100644 --- a/include/net/netns/mctp.h +++ b/include/net/netns/mctp.h @@ -6,6 +6,7 @@ #ifndef __NETNS_MCTP_H__ #define __NETNS_MCTP_H__ +#include <linux/mutex.h> #include <linux/types.h> struct netns_mctp { diff --git a/include/net/netns/mpls.h b/include/net/netns/mpls.h index a7bdcfbb0b28..19ad2574b267 100644 --- a/include/net/netns/mpls.h +++ b/include/net/netns/mpls.h @@ -6,6 +6,8 @@ #ifndef __NETNS_MPLS_H__ #define __NETNS_MPLS_H__ +#include <linux/types.h> + struct mpls_route; struct ctl_table_header; diff --git a/include/net/netns/nexthop.h b/include/net/netns/nexthop.h index 1849e77eb68a..434239b37014 100644 --- a/include/net/netns/nexthop.h +++ b/include/net/netns/nexthop.h @@ -6,6 +6,7 @@ #ifndef __NETNS_NEXTHOP_H__ #define __NETNS_NEXTHOP_H__ +#include <linux/notifier.h> #include <linux/rbtree.h> struct netns_nexthop { diff --git a/include/net/netns/sctp.h b/include/net/netns/sctp.h index 40240722cdca..a681147aecd8 100644 --- a/include/net/netns/sctp.h +++ b/include/net/netns/sctp.h @@ -2,6 +2,9 @@ #ifndef __NETNS_SCTP_H__ #define __NETNS_SCTP_H__ +#include <linux/timer.h> +#include <net/snmp.h> + struct sock; struct proc_dir_entry; struct sctp_mib; diff --git a/include/net/netns/smc.h b/include/net/netns/smc.h index e5389eeaf8bd..2adbe2b245df 100644 --- a/include/net/netns/smc.h +++ b/include/net/netns/smc.h @@ -18,5 +18,6 @@ struct netns_smc { struct ctl_table_header *smc_hdr; #endif unsigned int sysctl_autocorking_size; + unsigned int sysctl_smcr_buf_type; }; #endif diff --git a/include/net/netns/unix.h b/include/net/netns/unix.h index 6f1a33df061d..9859d134d5a8 100644 --- a/include/net/netns/unix.h +++ b/include/net/netns/unix.h @@ -5,6 +5,8 @@ #ifndef __NETNS_UNIX_H__ #define __NETNS_UNIX_H__ +#include <linux/spinlock.h> + struct unix_table { spinlock_t *locks; struct hlist_head *buckets; diff --git a/include/net/netrom.h b/include/net/netrom.h index 80f15b1c1a48..f0565a5987d1 100644 --- a/include/net/netrom.h +++ b/include/net/netrom.h @@ -14,6 +14,7 @@ #include <net/sock.h> #include <linux/refcount.h> #include <linux/seq_file.h> +#include <net/ax25.h> #define NR_NETWORK_LEN 15 #define NR_TRANSPORT_LEN 5 diff --git a/include/net/p8022.h b/include/net/p8022.h index c2bacc66bfbc..b690ffcad66b 100644 --- a/include/net/p8022.h +++ b/include/net/p8022.h @@ -1,6 +1,11 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _NET_P8022_H #define _NET_P8022_H + +struct net_device; +struct packet_type; +struct sk_buff; + struct datalink_proto * register_8022_client(unsigned char type, int (*func)(struct sk_buff *skb, diff --git a/include/net/phonet/pep.h b/include/net/phonet/pep.h index 27b1ab5e4e6d..645dddf5ce77 100644 --- a/include/net/phonet/pep.h +++ b/include/net/phonet/pep.h @@ -10,6 +10,9 @@ #ifndef NET_PHONET_PEP_H #define NET_PHONET_PEP_H +#include <linux/skbuff.h> +#include <net/phonet/phonet.h> + struct pep_sock { struct pn_sock pn_sk; diff --git a/include/net/phonet/phonet.h b/include/net/phonet/phonet.h index a27bdc6cfeab..862f1719b523 100644 --- a/include/net/phonet/phonet.h +++ b/include/net/phonet/phonet.h @@ -10,6 +10,10 @@ #ifndef AF_PHONET_H #define AF_PHONET_H +#include <linux/phonet.h> +#include <linux/skbuff.h> +#include <net/sock.h> + /* * The lower layers may not require more space, ever. Make sure it's * enough. diff --git a/include/net/phonet/pn_dev.h b/include/net/phonet/pn_dev.h index 05b49d4d2b11..e9dc8dca5817 100644 --- a/include/net/phonet/pn_dev.h +++ b/include/net/phonet/pn_dev.h @@ -10,6 +10,11 @@ #ifndef PN_DEV_H #define PN_DEV_H +#include <linux/list.h> +#include <linux/mutex.h> + +struct net; + struct phonet_device_list { struct list_head list; struct mutex lock; diff --git a/include/net/pptp.h b/include/net/pptp.h index 383e25ca53a7..e63176bdd4c8 100644 --- a/include/net/pptp.h +++ b/include/net/pptp.h @@ -2,6 +2,9 @@ #ifndef _NET_PPTP_H #define _NET_PPTP_H +#include <linux/types.h> +#include <net/gre.h> + #define PPP_LCP_ECHOREQ 0x09 #define PPP_LCP_ECHOREP 0x0A #define SC_RCV_BITS (SC_RCV_B7_1|SC_RCV_B7_0|SC_RCV_ODDP|SC_RCV_EVNP) diff --git a/include/net/protocol.h b/include/net/protocol.h index f51c06ae365f..6aef8cb11cc8 100644 --- a/include/net/protocol.h +++ b/include/net/protocol.h @@ -35,8 +35,6 @@ /* This is used to register protocols. */ struct net_protocol { - int (*early_demux)(struct sk_buff *skb); - int (*early_demux_handler)(struct sk_buff *skb); int (*handler)(struct sk_buff *skb); /* This returns an error if we weren't able to handle the error. */ @@ -52,8 +50,6 @@ struct net_protocol { #if IS_ENABLED(CONFIG_IPV6) struct inet6_protocol { - void (*early_demux)(struct sk_buff *skb); - void (*early_demux_handler)(struct sk_buff *skb); int (*handler)(struct sk_buff *skb); /* This returns an error if we weren't able to handle the error. */ diff --git a/include/net/psnap.h b/include/net/psnap.h index 7cb0c8ab4171..88802b0754ad 100644 --- a/include/net/psnap.h +++ b/include/net/psnap.h @@ -2,6 +2,11 @@ #ifndef _NET_PSNAP_H #define _NET_PSNAP_H +struct datalink_proto; +struct sk_buff; +struct packet_type; +struct net_device; + struct datalink_proto * register_snap_client(const unsigned char *desc, int (*rcvfunc)(struct sk_buff *, struct net_device *, diff --git a/include/net/regulatory.h b/include/net/regulatory.h index 47f06f6f5a67..896191f420d5 100644 --- a/include/net/regulatory.h +++ b/include/net/regulatory.h @@ -1,3 +1,4 @@ + #ifndef __NET_REGULATORY_H #define __NET_REGULATORY_H /* @@ -19,6 +20,8 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include <linux/ieee80211.h> +#include <linux/nl80211.h> #include <linux/rcupdate.h> /** diff --git a/include/net/rose.h b/include/net/rose.h index 0f0a4ce0fee7..f192a64ddef2 100644 --- a/include/net/rose.h +++ b/include/net/rose.h @@ -9,6 +9,7 @@ #define _ROSE_H #include <linux/rose.h> +#include <net/ax25.h> #include <net/sock.h> #define ROSE_ADDR_LEN 5 diff --git a/include/net/route.h b/include/net/route.h index 651424ef09c9..6e92dd5bcd61 100644 --- a/include/net/route.h +++ b/include/net/route.h @@ -240,8 +240,7 @@ void ip_rt_multicast_event(struct in_device *); int ip_rt_ioctl(struct net *, unsigned int cmd, struct rtentry *rt); void ip_rt_get_source(u8 *src, struct sk_buff *skb, struct rtable *rt); struct rtable *rt_dst_alloc(struct net_device *dev, - unsigned int flags, u16 type, - bool nopolicy, bool noxfrm); + unsigned int flags, u16 type, bool noxfrm); struct rtable *rt_dst_clone(struct net_device *dev, struct rtable *rt); struct in_ifaddr; @@ -369,7 +368,7 @@ static inline int ip4_dst_hoplimit(const struct dst_entry *dst) struct net *net = dev_net(dst->dev); if (hoplimit == 0) - hoplimit = net->ipv4.sysctl_ip_default_ttl; + hoplimit = READ_ONCE(net->ipv4.sysctl_ip_default_ttl); return hoplimit; } diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index d6cf5116b5f9..ec693fe7c553 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -551,25 +551,6 @@ static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) return qdisc->dev_queue->qdisc_sleeping; } -/* The qdisc root lock is a mechanism by which to top level - * of a qdisc tree can be locked from any qdisc node in the - * forest. This allows changing the configuration of some - * aspect of the qdisc tree while blocking out asynchronous - * qdisc access in the packet processing paths. - * - * It is only legal to do this when the root will not change - * on us. Otherwise we'll potentially lock the wrong qdisc - * root. This is enforced by holding the RTNL semaphore, which - * all users of this lock accessor must do. - */ -static inline spinlock_t *qdisc_root_lock(const struct Qdisc *qdisc) -{ - struct Qdisc *root = qdisc_root(qdisc); - - ASSERT_RTNL(); - return qdisc_lock(root); -} - static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) { struct Qdisc *root = qdisc_root_sleeping(qdisc); diff --git a/include/net/secure_seq.h b/include/net/secure_seq.h index dac91aa38c5a..21e7fa2a1813 100644 --- a/include/net/secure_seq.h +++ b/include/net/secure_seq.h @@ -4,6 +4,8 @@ #include <linux/types.h> +struct net; + u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport); diff --git a/include/net/smc.h b/include/net/smc.h index e441aa97ad61..37f829d9c6e5 100644 --- a/include/net/smc.h +++ b/include/net/smc.h @@ -11,6 +11,13 @@ #ifndef _SMC_H #define _SMC_H +#include <linux/device.h> +#include <linux/spinlock.h> +#include <linux/types.h> +#include <linux/wait.h> + +struct sock; + #define SMC_MAX_PNETID_LEN 16 /* Max. length of PNET id */ struct smc_hashinfo { diff --git a/include/net/stp.h b/include/net/stp.h index 2914e6d53490..528103fce2c0 100644 --- a/include/net/stp.h +++ b/include/net/stp.h @@ -2,6 +2,8 @@ #ifndef _NET_STP_H #define _NET_STP_H +#include <linux/if_ether.h> + struct stp_proto { unsigned char group_address[ETH_ALEN]; void (*rcv)(const struct stp_proto *, struct sk_buff *, diff --git a/include/net/strparser.h b/include/net/strparser.h index 88900b05443e..41e2ce9e9e10 100644 --- a/include/net/strparser.h +++ b/include/net/strparser.h @@ -72,7 +72,6 @@ struct sk_skb_cb { /* strp users' data follows */ struct tls_msg { u8 control; - u8 decrypted; } tls; /* temp_reg is a temporary register used for bpf_convert_data_end_access * when dst_reg == src_reg. diff --git a/include/net/tcp.h b/include/net/tcp.h index 8e48dc56837b..f9e7c85ea829 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -936,7 +936,7 @@ extern const struct inet_connection_sock_af_ops ipv6_specific; INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)); INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb)); -INDIRECT_CALLABLE_DECLARE(void tcp_v6_early_demux(struct sk_buff *skb)); +void tcp_v6_early_demux(struct sk_buff *skb); #endif @@ -1407,8 +1407,8 @@ static inline void tcp_slow_start_after_idle_check(struct sock *sk) struct tcp_sock *tp = tcp_sk(sk); s32 delta; - if (!sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle || tp->packets_out || - ca_ops->cong_control) + if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) || + tp->packets_out || ca_ops->cong_control) return; delta = tcp_jiffies32 - tp->lsndtime; if (delta > inet_csk(sk)->icsk_rto) @@ -1497,21 +1497,24 @@ static inline int keepalive_intvl_when(const struct tcp_sock *tp) { struct net *net = sock_net((struct sock *)tp); - return tp->keepalive_intvl ? : net->ipv4.sysctl_tcp_keepalive_intvl; + return tp->keepalive_intvl ? : + READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl); } static inline int keepalive_time_when(const struct tcp_sock *tp) { struct net *net = sock_net((struct sock *)tp); - return tp->keepalive_time ? : net->ipv4.sysctl_tcp_keepalive_time; + return tp->keepalive_time ? : + READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time); } static inline int keepalive_probes(const struct tcp_sock *tp) { struct net *net = sock_net((struct sock *)tp); - return tp->keepalive_probes ? : net->ipv4.sysctl_tcp_keepalive_probes; + return tp->keepalive_probes ? : + READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes); } static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) @@ -1524,7 +1527,8 @@ static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) static inline int tcp_fin_time(const struct sock *sk) { - int fin_timeout = tcp_sk(sk)->linger2 ? : sock_net(sk)->ipv4.sysctl_tcp_fin_timeout; + int fin_timeout = tcp_sk(sk)->linger2 ? : + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout); const int rto = inet_csk(sk)->icsk_rto; if (fin_timeout < (rto << 2) - (rto >> 1)) @@ -2027,7 +2031,7 @@ void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr); static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) { struct net *net = sock_net((struct sock *)tp); - return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat; + return tp->notsent_lowat ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat); } bool tcp_stream_memory_free(const struct sock *sk, int wake); diff --git a/include/net/tls.h b/include/net/tls.h index 8742e13bc362..181c496b01b8 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -116,11 +116,15 @@ struct tls_sw_context_rx { void (*saved_data_ready)(struct sock *sk); struct sk_buff *recv_pkt; + u8 reader_present; u8 async_capable:1; u8 zc_capable:1; + u8 reader_contended:1; atomic_t decrypt_pending; /* protect crypto_wait with decrypt_pending*/ spinlock_t decrypt_compl_lock; + struct sk_buff_head async_hold; + struct wait_queue_head wq; }; struct tls_record_info { diff --git a/include/net/transp_v6.h b/include/net/transp_v6.h index da06613c9603..b830463e3dff 100644 --- a/include/net/transp_v6.h +++ b/include/net/transp_v6.h @@ -3,6 +3,7 @@ #define _TRANSP_V6_H #include <net/checksum.h> +#include <net/sock.h> /* IPv6 transport protocols */ extern struct proto rawv6_prot; @@ -12,6 +13,7 @@ extern struct proto tcpv6_prot; extern struct proto pingv6_prot; struct flowi6; +struct ipcm6_cookie; /* extension headers */ int ipv6_exthdrs_init(void); diff --git a/include/net/tun_proto.h b/include/net/tun_proto.h index 2ea3deba4c99..7b0de7852908 100644 --- a/include/net/tun_proto.h +++ b/include/net/tun_proto.h @@ -1,7 +1,8 @@ #ifndef __NET_TUN_PROTO_H #define __NET_TUN_PROTO_H -#include <linux/kernel.h> +#include <linux/if_ether.h> +#include <linux/types.h> /* One byte protocol values as defined by VXLAN-GPE and NSH. These will * hopefully get a shared IANA registry. diff --git a/include/net/udp.h b/include/net/udp.h index 987f7fc7c0aa..5ee88ddf79c3 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -168,7 +168,7 @@ static inline void udp_csum_pull_header(struct sk_buff *skb) typedef struct sock *(*udp_lookup_t)(const struct sk_buff *skb, __be16 sport, __be16 dport); -INDIRECT_CALLABLE_DECLARE(void udp_v6_early_demux(struct sk_buff *)); +void udp_v6_early_demux(struct sk_buff *skb); INDIRECT_CALLABLE_DECLARE(int udpv6_rcv(struct sk_buff *)); struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, @@ -239,7 +239,7 @@ static inline bool udp_sk_bound_dev_eq(struct net *net, int bound_dev_if, int dif, int sdif) { #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) - return inet_bound_dev_eq(!!net->ipv4.sysctl_udp_l3mdev_accept, + return inet_bound_dev_eq(!!READ_ONCE(net->ipv4.sysctl_udp_l3mdev_accept), bound_dev_if, dif, sdif); #else return inet_bound_dev_eq(true, bound_dev_if, dif, sdif); diff --git a/include/net/udplite.h b/include/net/udplite.h index a3c53110d30b..0143b373602e 100644 --- a/include/net/udplite.h +++ b/include/net/udplite.h @@ -6,6 +6,7 @@ #define _UDPLITE_H #include <net/ip6_checksum.h> +#include <net/udp.h> /* UDP-Lite socket options */ #define UDPLITE_SEND_CSCOV 10 /* sender partial coverage (as sent) */ diff --git a/include/net/xdp_priv.h b/include/net/xdp_priv.h index a2d58b1a12e1..c9df68d5f258 100644 --- a/include/net/xdp_priv.h +++ b/include/net/xdp_priv.h @@ -3,6 +3,7 @@ #define __LINUX_NET_XDP_PRIV_H__ #include <linux/rhashtable.h> +#include <net/xdp.h> /* Private to net/core/xdp.c, but used by trace/events/xdp.h */ struct xdp_mem_allocator { diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 4aa031849668..4277b0dcee05 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -44,6 +44,15 @@ static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, xp_set_rxq_info(pool, rxq); } +static inline unsigned int xsk_pool_get_napi_id(struct xsk_buff_pool *pool) +{ +#ifdef CONFIG_NET_RX_BUSY_POLL + return pool->heads[0].xdp.rxq->napi_id; +#else + return 0; +#endif +} + static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs) { @@ -198,6 +207,11 @@ static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, { } +static inline unsigned int xsk_pool_get_napi_id(struct xsk_buff_pool *pool) +{ + return 0; +} + static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs) { diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 67b799ef5f67..6e8fa98f786f 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -583,8 +583,8 @@ struct xfrm_mgr { bool (*is_alive)(const struct km_event *c); }; -int xfrm_register_km(struct xfrm_mgr *km); -int xfrm_unregister_km(struct xfrm_mgr *km); +void xfrm_register_km(struct xfrm_mgr *km); +void xfrm_unregister_km(struct xfrm_mgr *km); struct xfrm_tunnel_skb_cb { union { diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 3dd13fe738b9..59a217ca2dfd 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -2361,7 +2361,8 @@ union bpf_attr { * Pull in non-linear data in case the *skb* is non-linear and not * all of *len* are part of the linear section. Make *len* bytes * from *skb* readable and writable. If a zero value is passed for - * *len*, then the whole length of the *skb* is pulled. + * *len*, then all bytes in the linear part of *skb* will be made + * readable and writable. * * This helper is only needed for reading and writing with direct * packet access. diff --git a/include/uapi/linux/can/error.h b/include/uapi/linux/can/error.h index 34633283de64..acc1ac393d2a 100644 --- a/include/uapi/linux/can/error.h +++ b/include/uapi/linux/can/error.h @@ -57,6 +57,8 @@ #define CAN_ERR_BUSOFF 0x00000040U /* bus off */ #define CAN_ERR_BUSERROR 0x00000080U /* bus error (may flood!) */ #define CAN_ERR_RESTARTED 0x00000100U /* controller restarted */ +#define CAN_ERR_CNT 0x00000200U /* TX error counter / data[6] */ + /* RX error counter / data[7] */ /* arbitration lost in bit ... / data[0] */ #define CAN_ERR_LOSTARB_UNSPEC 0x00 /* unspecified */ @@ -120,6 +122,22 @@ #define CAN_ERR_TRX_CANL_SHORT_TO_GND 0x70 /* 0111 0000 */ #define CAN_ERR_TRX_CANL_SHORT_TO_CANH 0x80 /* 1000 0000 */ -/* controller specific additional information / data[5..7] */ +/* data[5] is reserved (do not use) */ + +/* TX error counter / data[6] */ +/* RX error counter / data[7] */ + +/* CAN state thresholds + * + * Error counter Error state + * ----------------------------------- + * 0 - 95 Error-active + * 96 - 127 Error-warning + * 128 - 255 Error-passive + * 256 and greater Bus-off + */ +#define CAN_ERROR_WARNING_THRESHOLD 96 +#define CAN_ERROR_PASSIVE_THRESHOLD 128 +#define CAN_BUS_OFF_THRESHOLD 256 #endif /* _UAPI_CAN_ERROR_H */ diff --git a/include/uapi/linux/input.h b/include/uapi/linux/input.h index ef4257ab3026..2557eb7b0561 100644 --- a/include/uapi/linux/input.h +++ b/include/uapi/linux/input.h @@ -78,10 +78,13 @@ struct input_id { * Note that input core does not clamp reported values to the * [minimum, maximum] limits, such task is left to userspace. * - * The default resolution for main axes (ABS_X, ABS_Y, ABS_Z) - * is reported in units per millimeter (units/mm), resolution - * for rotational axes (ABS_RX, ABS_RY, ABS_RZ) is reported - * in units per radian. + * The default resolution for main axes (ABS_X, ABS_Y, ABS_Z, + * ABS_MT_POSITION_X, ABS_MT_POSITION_Y) is reported in units + * per millimeter (units/mm), resolution for rotational axes + * (ABS_RX, ABS_RY, ABS_RZ) is reported in units per radian. + * The resolution for the size axes (ABS_MT_TOUCH_MAJOR, + * ABS_MT_TOUCH_MINOR, ABS_MT_WIDTH_MAJOR, ABS_MT_WIDTH_MINOR) + * is reported in units per millimeter (units/mm). * When INPUT_PROP_ACCELEROMETER is set the resolution changes. * The main axes (ABS_X, ABS_Y, ABS_Z) are then reported in * units per g (units/g) and in units per degree per second diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 5088bd9f1922..811897dadcae 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -2083,6 +2083,7 @@ struct kvm_stats_header { #define KVM_STATS_UNIT_BYTES (0x1 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_SECONDS (0x2 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_CYCLES (0x3 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_BOOLEAN (0x4 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_MAX KVM_STATS_UNIT_CYCLES #define KVM_STATS_BASE_SHIFT 8 diff --git a/include/uapi/linux/smc.h b/include/uapi/linux/smc.h index 693f549f6966..bb4dacca31e7 100644 --- a/include/uapi/linux/smc.h +++ b/include/uapi/linux/smc.h @@ -124,6 +124,7 @@ enum { SMC_NLA_LGR_R_V2, /* nest */ SMC_NLA_LGR_R_NET_COOKIE, /* u64 */ SMC_NLA_LGR_R_PAD, /* flag */ + SMC_NLA_LGR_R_BUF_TYPE, /* u8 */ __SMC_NLA_LGR_R_MAX, SMC_NLA_LGR_R_MAX = __SMC_NLA_LGR_R_MAX - 1 }; diff --git a/include/uapi/linux/xfrm.h b/include/uapi/linux/xfrm.h index 65e13a099b1a..7929bf9cbee4 100644 --- a/include/uapi/linux/xfrm.h +++ b/include/uapi/linux/xfrm.h @@ -102,21 +102,21 @@ struct xfrm_replay_state_esn { struct xfrm_algo { char alg_name[64]; unsigned int alg_key_len; /* in bits */ - char alg_key[0]; + char alg_key[]; }; struct xfrm_algo_auth { char alg_name[64]; unsigned int alg_key_len; /* in bits */ unsigned int alg_trunc_len; /* in bits */ - char alg_key[0]; + char alg_key[]; }; struct xfrm_algo_aead { char alg_name[64]; unsigned int alg_key_len; /* in bits */ unsigned int alg_icv_len; /* in bits */ - char alg_key[0]; + char alg_key[]; }; struct xfrm_stats { @@ -511,9 +511,9 @@ struct xfrm_user_offload { int ifindex; __u8 flags; }; -/* This flag was exposed without any kernel code that supporting it. - * Unfortunately, strongswan has the code that uses sets this flag, - * which makes impossible to reuse this bit. +/* This flag was exposed without any kernel code that supports it. + * Unfortunately, strongswan has the code that sets this flag, + * which makes it impossible to reuse this bit. * * So leave it here to make sure that it won't be reused by mistake. */ diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index fe40d3b9458f..d3e734bf8056 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -70,10 +70,8 @@ int array_map_alloc_check(union bpf_attr *attr) attr->map_flags & BPF_F_PRESERVE_ELEMS) return -EINVAL; - if (attr->value_size > KMALLOC_MAX_SIZE) - /* if value_size is bigger, the user space won't be able to - * access the elements. - */ + /* avoid overflow on round_up(map->value_size) */ + if (attr->value_size > INT_MAX) return -E2BIG; return 0; @@ -156,6 +154,11 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) return &array->map; } +static void *array_map_elem_ptr(struct bpf_array* array, u32 index) +{ + return array->value + (u64)array->elem_size * index; +} + /* Called from syscall or from eBPF program */ static void *array_map_lookup_elem(struct bpf_map *map, void *key) { @@ -165,7 +168,7 @@ static void *array_map_lookup_elem(struct bpf_map *map, void *key) if (unlikely(index >= array->map.max_entries)) return NULL; - return array->value + array->elem_size * (index & array->index_mask); + return array->value + (u64)array->elem_size * (index & array->index_mask); } static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, @@ -203,7 +206,7 @@ static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) { struct bpf_array *array = container_of(map, struct bpf_array, map); struct bpf_insn *insn = insn_buf; - u32 elem_size = round_up(map->value_size, 8); + u32 elem_size = array->elem_size; const int ret = BPF_REG_0; const int map_ptr = BPF_REG_1; const int index = BPF_REG_2; @@ -272,7 +275,7 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) * access 'value_size' of them, so copying rounded areas * will not leak any kernel data */ - size = round_up(map->value_size, 8); + size = array->elem_size; rcu_read_lock(); pptr = array->pptrs[index & array->index_mask]; for_each_possible_cpu(cpu) { @@ -339,7 +342,7 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value, value, map->value_size); } else { val = array->value + - array->elem_size * (index & array->index_mask); + (u64)array->elem_size * (index & array->index_mask); if (map_flags & BPF_F_LOCK) copy_map_value_locked(map, val, value, false); else @@ -376,7 +379,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, * returned or zeros which were zero-filled by percpu_alloc, * so no kernel data leaks possible */ - size = round_up(map->value_size, 8); + size = array->elem_size; rcu_read_lock(); pptr = array->pptrs[index & array->index_mask]; for_each_possible_cpu(cpu) { @@ -408,8 +411,7 @@ static void array_map_free_timers(struct bpf_map *map) return; for (i = 0; i < array->map.max_entries; i++) - bpf_timer_cancel_and_free(array->value + array->elem_size * i + - map->timer_off); + bpf_timer_cancel_and_free(array_map_elem_ptr(array, i) + map->timer_off); } /* Called when map->refcnt goes to zero, either from workqueue or from syscall */ @@ -420,7 +422,7 @@ static void array_map_free(struct bpf_map *map) if (map_value_has_kptrs(map)) { for (i = 0; i < array->map.max_entries; i++) - bpf_map_free_kptrs(map, array->value + array->elem_size * i); + bpf_map_free_kptrs(map, array_map_elem_ptr(array, i)); bpf_map_free_kptr_off_tab(map); } @@ -556,7 +558,7 @@ static void *bpf_array_map_seq_start(struct seq_file *seq, loff_t *pos) index = info->index & array->index_mask; if (info->percpu_value_buf) return array->pptrs[index]; - return array->value + array->elem_size * index; + return array_map_elem_ptr(array, index); } static void *bpf_array_map_seq_next(struct seq_file *seq, void *v, loff_t *pos) @@ -575,7 +577,7 @@ static void *bpf_array_map_seq_next(struct seq_file *seq, void *v, loff_t *pos) index = info->index & array->index_mask; if (info->percpu_value_buf) return array->pptrs[index]; - return array->value + array->elem_size * index; + return array_map_elem_ptr(array, index); } static int __bpf_array_map_seq_show(struct seq_file *seq, void *v) @@ -583,6 +585,7 @@ static int __bpf_array_map_seq_show(struct seq_file *seq, void *v) struct bpf_iter_seq_array_map_info *info = seq->private; struct bpf_iter__bpf_map_elem ctx = {}; struct bpf_map *map = info->map; + struct bpf_array *array = container_of(map, struct bpf_array, map); struct bpf_iter_meta meta; struct bpf_prog *prog; int off = 0, cpu = 0; @@ -603,7 +606,7 @@ static int __bpf_array_map_seq_show(struct seq_file *seq, void *v) ctx.value = v; } else { pptr = v; - size = round_up(map->value_size, 8); + size = array->elem_size; for_each_possible_cpu(cpu) { bpf_long_memcpy(info->percpu_value_buf + off, per_cpu_ptr(pptr, cpu), @@ -633,11 +636,12 @@ static int bpf_iter_init_array_map(void *priv_data, { struct bpf_iter_seq_array_map_info *seq_info = priv_data; struct bpf_map *map = aux->map; + struct bpf_array *array = container_of(map, struct bpf_array, map); void *value_buf; u32 buf_size; if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { - buf_size = round_up(map->value_size, 8) * num_possible_cpus(); + buf_size = array->elem_size * num_possible_cpus(); value_buf = kmalloc(buf_size, GFP_USER | __GFP_NOWARN); if (!value_buf) return -ENOMEM; @@ -690,7 +694,7 @@ static int bpf_for_each_array_elem(struct bpf_map *map, bpf_callback_t callback_ if (is_percpu) val = this_cpu_ptr(array->pptrs[i]); else - val = array->value + array->elem_size * i; + val = array_map_elem_ptr(array, i); num_elems++; key = i; ret = callback_fn((u64)(long)map, (u64)(long)&key, @@ -1322,7 +1326,7 @@ static int array_of_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) { struct bpf_array *array = container_of(map, struct bpf_array, map); - u32 elem_size = round_up(map->value_size, 8); + u32 elem_size = array->elem_size; struct bpf_insn *insn = insn_buf; const int ret = BPF_REG_0; const int map_ptr = BPF_REG_1; diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index d469b7f3deef..fa71d58b7ded 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -63,10 +63,11 @@ BTF_ID(func, bpf_lsm_socket_post_create) BTF_ID(func, bpf_lsm_socket_socketpair) BTF_SET_END(bpf_lsm_unlocked_sockopt_hooks) +#ifdef CONFIG_CGROUP_BPF void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, bpf_func_t *bpf_func) { - const struct btf_param *args; + const struct btf_param *args __maybe_unused; if (btf_type_vlen(prog->aux->attach_func_proto) < 1 || btf_id_set_contains(&bpf_lsm_current_hooks, @@ -75,9 +76,9 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, return; } +#ifdef CONFIG_NET args = btf_params(prog->aux->attach_func_proto); -#ifdef CONFIG_NET if (args[0].type == btf_sock_ids[BTF_SOCK_TYPE_SOCKET]) *bpf_func = __cgroup_bpf_run_lsm_socket; else if (args[0].type == btf_sock_ids[BTF_SOCK_TYPE_SOCK]) @@ -86,6 +87,7 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, #endif *bpf_func = __cgroup_bpf_run_lsm_current; } +#endif int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, const struct bpf_prog *prog) @@ -219,6 +221,7 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_get_retval: return prog->expected_attach_type == BPF_LSM_CGROUP ? &bpf_get_retval_proto : NULL; +#ifdef CONFIG_NET case BPF_FUNC_setsockopt: if (prog->expected_attach_type != BPF_LSM_CGROUP) return NULL; @@ -239,6 +242,7 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) prog->aux->attach_btf_id)) return &bpf_unlocked_sk_getsockopt_proto; return NULL; +#endif default: return tracing_prog_func_proto(func_id, prog); } diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 7e0068c3399c..84b2d9dba79a 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -341,6 +341,9 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, tlinks[BPF_TRAMP_FENTRY].links[0] = link; tlinks[BPF_TRAMP_FENTRY].nr_links = 1; + /* BPF_TRAMP_F_RET_FENTRY_RET is only used by bpf_struct_ops, + * and it must be used alone. + */ flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0; return arch_prepare_bpf_trampoline(NULL, image, image_end, model, flags, tlinks, NULL); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 4423045b8ff3..7ac971ea98d1 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -213,7 +213,7 @@ enum { }; struct btf_kfunc_set_tab { - struct btf_id_set *sets[BTF_KFUNC_HOOK_MAX][BTF_KFUNC_TYPE_MAX]; + struct btf_id_set8 *sets[BTF_KFUNC_HOOK_MAX]; }; struct btf_id_dtor_kfunc_tab { @@ -1116,7 +1116,8 @@ __printf(2, 3) static void btf_show(struct btf_show *show, const char *fmt, ...) */ #define btf_show_type_value(show, fmt, value) \ do { \ - if ((value) != 0 || (show->flags & BTF_SHOW_ZERO) || \ + if ((value) != (__typeof__(value))0 || \ + (show->flags & BTF_SHOW_ZERO) || \ show->state.depth == 0) { \ btf_show(show, "%s%s" fmt "%s%s", \ btf_show_indent(show), \ @@ -1615,7 +1616,7 @@ static void btf_free_id(struct btf *btf) static void btf_free_kfunc_set_tab(struct btf *btf) { struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab; - int hook, type; + int hook; if (!tab) return; @@ -1624,10 +1625,8 @@ static void btf_free_kfunc_set_tab(struct btf *btf) */ if (btf_is_module(btf)) goto free_tab; - for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++) { - for (type = 0; type < ARRAY_SIZE(tab->sets[0]); type++) - kfree(tab->sets[hook][type]); - } + for (hook = 0; hook < ARRAY_SIZE(tab->sets); hook++) + kfree(tab->sets[hook]); free_tab: kfree(tab); btf->kfunc_set_tab = NULL; @@ -6171,13 +6170,14 @@ static bool is_kfunc_arg_mem_size(const struct btf *btf, static int btf_check_func_arg_match(struct bpf_verifier_env *env, const struct btf *btf, u32 func_id, struct bpf_reg_state *regs, - bool ptr_to_mem_ok) + bool ptr_to_mem_ok, + u32 kfunc_flags) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); + bool rel = false, kptr_get = false, trusted_arg = false; struct bpf_verifier_log *log = &env->log; u32 i, nargs, ref_id, ref_obj_id = 0; bool is_kfunc = btf_is_kernel(btf); - bool rel = false, kptr_get = false; const char *func_name, *ref_tname; const struct btf_type *t, *ref_t; const struct btf_param *args; @@ -6209,10 +6209,9 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, if (is_kfunc) { /* Only kfunc can be release func */ - rel = btf_kfunc_id_set_contains(btf, resolve_prog_type(env->prog), - BTF_KFUNC_TYPE_RELEASE, func_id); - kptr_get = btf_kfunc_id_set_contains(btf, resolve_prog_type(env->prog), - BTF_KFUNC_TYPE_KPTR_ACQUIRE, func_id); + rel = kfunc_flags & KF_RELEASE; + kptr_get = kfunc_flags & KF_KPTR_GET; + trusted_arg = kfunc_flags & KF_TRUSTED_ARGS; } /* check that BTF function arguments match actual types that the @@ -6237,10 +6236,19 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, return -EINVAL; } + /* Check if argument must be a referenced pointer, args + i has + * been verified to be a pointer (after skipping modifiers). + */ + if (is_kfunc && trusted_arg && !reg->ref_obj_id) { + bpf_log(log, "R%d must be referenced\n", regno); + return -EINVAL; + } + ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); - if (rel && reg->ref_obj_id) + /* Trusted args have the same offset checks as release arguments */ + if (trusted_arg || (rel && reg->ref_obj_id)) arg_type |= OBJ_RELEASE; ret = check_func_arg_reg_off(env, reg, regno, arg_type); if (ret < 0) @@ -6338,7 +6346,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); if (!btf_struct_ids_match(log, reg_btf, reg_ref_id, - reg->off, btf, ref_id, rel && reg->ref_obj_id)) { + reg->off, btf, ref_id, + trusted_arg || (rel && reg->ref_obj_id))) { bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n", func_name, i, btf_type_str(ref_t), ref_tname, @@ -6441,7 +6450,7 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, return -EINVAL; is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; - err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global); + err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, 0); /* Compiler optimizations can remove arguments from static functions * or mismatched type can be passed into a global function. @@ -6454,9 +6463,10 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, int btf_check_kfunc_arg_match(struct bpf_verifier_env *env, const struct btf *btf, u32 func_id, - struct bpf_reg_state *regs) + struct bpf_reg_state *regs, + u32 kfunc_flags) { - return btf_check_func_arg_match(env, btf, func_id, regs, true); + return btf_check_func_arg_match(env, btf, func_id, regs, true, kfunc_flags); } /* Convert BTF of a function into bpf_reg_state if possible @@ -6853,6 +6863,11 @@ bool btf_id_set_contains(const struct btf_id_set *set, u32 id) return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL; } +static void *btf_id_set8_contains(const struct btf_id_set8 *set, u32 id) +{ + return bsearch(&id, set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func); +} + enum { BTF_MODULE_F_LIVE = (1 << 0), }; @@ -7101,16 +7116,16 @@ BTF_TRACING_TYPE_xxx /* Kernel Function (kfunc) BTF ID set registration API */ -static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, - enum btf_kfunc_type type, - struct btf_id_set *add_set, bool vmlinux_set) +static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, + struct btf_id_set8 *add_set) { + bool vmlinux_set = !btf_is_module(btf); struct btf_kfunc_set_tab *tab; - struct btf_id_set *set; + struct btf_id_set8 *set; u32 set_cnt; int ret; - if (hook >= BTF_KFUNC_HOOK_MAX || type >= BTF_KFUNC_TYPE_MAX) { + if (hook >= BTF_KFUNC_HOOK_MAX) { ret = -EINVAL; goto end; } @@ -7126,7 +7141,7 @@ static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, btf->kfunc_set_tab = tab; } - set = tab->sets[hook][type]; + set = tab->sets[hook]; /* Warn when register_btf_kfunc_id_set is called twice for the same hook * for module sets. */ @@ -7140,7 +7155,7 @@ static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, * pointer and return. */ if (!vmlinux_set) { - tab->sets[hook][type] = add_set; + tab->sets[hook] = add_set; return 0; } @@ -7149,7 +7164,7 @@ static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, * and concatenate all individual sets being registered. While each set * is individually sorted, they may become unsorted when concatenated, * hence re-sorting the final set again is required to make binary - * searching the set using btf_id_set_contains function work. + * searching the set using btf_id_set8_contains function work. */ set_cnt = set ? set->cnt : 0; @@ -7164,8 +7179,8 @@ static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, } /* Grow set */ - set = krealloc(tab->sets[hook][type], - offsetof(struct btf_id_set, ids[set_cnt + add_set->cnt]), + set = krealloc(tab->sets[hook], + offsetof(struct btf_id_set8, pairs[set_cnt + add_set->cnt]), GFP_KERNEL | __GFP_NOWARN); if (!set) { ret = -ENOMEM; @@ -7173,15 +7188,15 @@ static int __btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, } /* For newly allocated set, initialize set->cnt to 0 */ - if (!tab->sets[hook][type]) + if (!tab->sets[hook]) set->cnt = 0; - tab->sets[hook][type] = set; + tab->sets[hook] = set; /* Concatenate the two sets */ - memcpy(set->ids + set->cnt, add_set->ids, add_set->cnt * sizeof(set->ids[0])); + memcpy(set->pairs + set->cnt, add_set->pairs, add_set->cnt * sizeof(set->pairs[0])); set->cnt += add_set->cnt; - sort(set->ids, set->cnt, sizeof(set->ids[0]), btf_id_cmp_func, NULL); + sort(set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func, NULL); return 0; end: @@ -7189,38 +7204,25 @@ end: return ret; } -static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, - const struct btf_kfunc_id_set *kset) -{ - bool vmlinux_set = !btf_is_module(btf); - int type, ret = 0; - - for (type = 0; type < ARRAY_SIZE(kset->sets); type++) { - if (!kset->sets[type]) - continue; - - ret = __btf_populate_kfunc_set(btf, hook, type, kset->sets[type], vmlinux_set); - if (ret) - break; - } - return ret; -} - -static bool __btf_kfunc_id_set_contains(const struct btf *btf, +static u32 *__btf_kfunc_id_set_contains(const struct btf *btf, enum btf_kfunc_hook hook, - enum btf_kfunc_type type, u32 kfunc_btf_id) { - struct btf_id_set *set; + struct btf_id_set8 *set; + u32 *id; - if (hook >= BTF_KFUNC_HOOK_MAX || type >= BTF_KFUNC_TYPE_MAX) - return false; + if (hook >= BTF_KFUNC_HOOK_MAX) + return NULL; if (!btf->kfunc_set_tab) - return false; - set = btf->kfunc_set_tab->sets[hook][type]; + return NULL; + set = btf->kfunc_set_tab->sets[hook]; if (!set) - return false; - return btf_id_set_contains(set, kfunc_btf_id); + return NULL; + id = btf_id_set8_contains(set, kfunc_btf_id); + if (!id) + return NULL; + /* The flags for BTF ID are located next to it */ + return id + 1; } static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type) @@ -7248,14 +7250,14 @@ static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type) * keeping the reference for the duration of the call provides the necessary * protection for looking up a well-formed btf->kfunc_set_tab. */ -bool btf_kfunc_id_set_contains(const struct btf *btf, +u32 *btf_kfunc_id_set_contains(const struct btf *btf, enum bpf_prog_type prog_type, - enum btf_kfunc_type type, u32 kfunc_btf_id) + u32 kfunc_btf_id) { enum btf_kfunc_hook hook; hook = bpf_prog_type_to_kfunc_hook(prog_type); - return __btf_kfunc_id_set_contains(btf, hook, type, kfunc_btf_id); + return __btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id); } /* This function must be invoked only from initcalls/module init functions */ @@ -7282,7 +7284,7 @@ int register_btf_kfunc_id_set(enum bpf_prog_type prog_type, return PTR_ERR(btf); hook = bpf_prog_type_to_kfunc_hook(prog_type); - ret = btf_populate_kfunc_set(btf, hook, kset); + ret = btf_populate_kfunc_set(btf, hook, kset->set); btf_put(btf); return ret; } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index bfeb9b937315..c1e10d088dbb 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -652,12 +652,6 @@ static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) return fp->jited && !bpf_prog_was_classic(fp); } -static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) -{ - return list_empty(&fp->aux->ksym.lnode) || - fp->aux->ksym.lnode.prev == LIST_POISON2; -} - void bpf_prog_kallsyms_add(struct bpf_prog *fp) { if (!bpf_prog_kallsyms_candidate(fp) || @@ -833,15 +827,6 @@ struct bpf_prog_pack { #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) -static size_t bpf_prog_pack_size = -1; -static size_t bpf_prog_pack_mask = -1; - -static int bpf_prog_chunk_count(void) -{ - WARN_ON_ONCE(bpf_prog_pack_size == -1); - return bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE; -} - static DEFINE_MUTEX(pack_mutex); static LIST_HEAD(pack_list); @@ -849,55 +834,33 @@ static LIST_HEAD(pack_list); * CONFIG_MMU=n. Use PAGE_SIZE in these cases. */ #ifdef PMD_SIZE -#define BPF_HPAGE_SIZE PMD_SIZE -#define BPF_HPAGE_MASK PMD_MASK +#define BPF_PROG_PACK_SIZE (PMD_SIZE * num_possible_nodes()) #else -#define BPF_HPAGE_SIZE PAGE_SIZE -#define BPF_HPAGE_MASK PAGE_MASK +#define BPF_PROG_PACK_SIZE PAGE_SIZE #endif -static size_t select_bpf_prog_pack_size(void) -{ - size_t size; - void *ptr; - - size = BPF_HPAGE_SIZE * num_online_nodes(); - ptr = module_alloc(size); - - /* Test whether we can get huge pages. If not just use PAGE_SIZE - * packs. - */ - if (!ptr || !is_vm_area_hugepages(ptr)) { - size = PAGE_SIZE; - bpf_prog_pack_mask = PAGE_MASK; - } else { - bpf_prog_pack_mask = BPF_HPAGE_MASK; - } - - vfree(ptr); - return size; -} +#define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE) static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns) { struct bpf_prog_pack *pack; - pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(bpf_prog_chunk_count())), + pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)), GFP_KERNEL); if (!pack) return NULL; - pack->ptr = module_alloc(bpf_prog_pack_size); + pack->ptr = module_alloc(BPF_PROG_PACK_SIZE); if (!pack->ptr) { kfree(pack); return NULL; } - bpf_fill_ill_insns(pack->ptr, bpf_prog_pack_size); - bitmap_zero(pack->bitmap, bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE); + bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); + bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); list_add_tail(&pack->list, &pack_list); set_vm_flush_reset_perms(pack->ptr); - set_memory_ro((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); - set_memory_x((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); + set_memory_ro((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); + set_memory_x((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); return pack; } @@ -909,10 +872,7 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn void *ptr = NULL; mutex_lock(&pack_mutex); - if (bpf_prog_pack_size == -1) - bpf_prog_pack_size = select_bpf_prog_pack_size(); - - if (size > bpf_prog_pack_size) { + if (size > BPF_PROG_PACK_SIZE) { size = round_up(size, PAGE_SIZE); ptr = module_alloc(size); if (ptr) { @@ -924,9 +884,9 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn goto out; } list_for_each_entry(pack, &pack_list, list) { - pos = bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0, + pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, nbits, 0); - if (pos < bpf_prog_chunk_count()) + if (pos < BPF_PROG_CHUNK_COUNT) goto found_free_area; } @@ -950,18 +910,15 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) struct bpf_prog_pack *pack = NULL, *tmp; unsigned int nbits; unsigned long pos; - void *pack_ptr; mutex_lock(&pack_mutex); - if (hdr->size > bpf_prog_pack_size) { + if (hdr->size > BPF_PROG_PACK_SIZE) { module_memfree(hdr); goto out; } - pack_ptr = (void *)((unsigned long)hdr & bpf_prog_pack_mask); - list_for_each_entry(tmp, &pack_list, list) { - if (tmp->ptr == pack_ptr) { + if ((void *)hdr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > (void *)hdr) { pack = tmp; break; } @@ -971,14 +928,14 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr) goto out; nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size); - pos = ((unsigned long)hdr - (unsigned long)pack_ptr) >> BPF_PROG_CHUNK_SHIFT; + pos = ((unsigned long)hdr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT; WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size), "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); bitmap_clear(pack->bitmap, pos, nbits); - if (bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0, - bpf_prog_chunk_count(), 0) == 0) { + if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, + BPF_PROG_CHUNK_COUNT, 0) == 0) { list_del(&pack->list); module_memfree(pack->ptr); kfree(pack); @@ -1155,7 +1112,6 @@ int bpf_jit_binary_pack_finalize(struct bpf_prog *prog, bpf_prog_pack_free(ro_header); return PTR_ERR(ptr); } - prog->aux->use_bpf_prog_pack = true; return 0; } @@ -1179,17 +1135,23 @@ void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, bpf_jit_uncharge_modmem(size); } +struct bpf_binary_header * +bpf_jit_binary_pack_hdr(const struct bpf_prog *fp) +{ + unsigned long real_start = (unsigned long)fp->bpf_func; + unsigned long addr; + + addr = real_start & BPF_PROG_CHUNK_MASK; + return (void *)addr; +} + static inline struct bpf_binary_header * bpf_jit_binary_hdr(const struct bpf_prog *fp) { unsigned long real_start = (unsigned long)fp->bpf_func; unsigned long addr; - if (fp->aux->use_bpf_prog_pack) - addr = real_start & BPF_PROG_CHUNK_MASK; - else - addr = real_start & PAGE_MASK; - + addr = real_start & PAGE_MASK; return (void *)addr; } @@ -1202,11 +1164,7 @@ void __weak bpf_jit_free(struct bpf_prog *fp) if (fp->jited) { struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp); - if (fp->aux->use_bpf_prog_pack) - bpf_jit_binary_pack_free(hdr, NULL /* rw_buffer */); - else - bpf_jit_binary_free(hdr); - + bpf_jit_binary_free(hdr); WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); } diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index c2867068e5bd..1400561efb15 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -845,7 +845,7 @@ static struct bpf_dtab_netdev *__dev_map_alloc_node(struct net *net, struct bpf_dtab_netdev *dev; dev = bpf_map_kmalloc_node(&dtab->map, sizeof(*dev), - GFP_ATOMIC | __GFP_NOWARN, + GFP_NOWAIT | __GFP_NOWARN, dtab->map.numa_node); if (!dev) return ERR_PTR(-ENOMEM); diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 17fb69c0e0dc..da7578426a46 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -61,7 +61,7 @@ * * As regular device interrupt handlers and soft interrupts are forced into * thread context, the existing code which does - * spin_lock*(); alloc(GPF_ATOMIC); spin_unlock*(); + * spin_lock*(); alloc(GFP_ATOMIC); spin_unlock*(); * just works. * * In theory the BPF locks could be converted to regular spinlocks as well, @@ -978,7 +978,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, goto dec_count; } l_new = bpf_map_kmalloc_node(&htab->map, htab->elem_size, - GFP_ATOMIC | __GFP_NOWARN, + GFP_NOWAIT | __GFP_NOWARN, htab->map.numa_node); if (!l_new) { l_new = ERR_PTR(-ENOMEM); @@ -996,7 +996,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, } else { /* alloc_percpu zero-fills */ pptr = bpf_map_alloc_percpu(&htab->map, size, 8, - GFP_ATOMIC | __GFP_NOWARN); + GFP_NOWAIT | __GFP_NOWARN); if (!pptr) { kfree(l_new); l_new = ERR_PTR(-ENOMEM); diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c index 8654fc97f5fe..49ef0ce040c7 100644 --- a/kernel/bpf/local_storage.c +++ b/kernel/bpf/local_storage.c @@ -165,7 +165,7 @@ static int cgroup_storage_update_elem(struct bpf_map *map, void *key, } new = bpf_map_kmalloc_node(map, struct_size(new, data, map->value_size), - __GFP_ZERO | GFP_ATOMIC | __GFP_NOWARN, + __GFP_ZERO | GFP_NOWAIT | __GFP_NOWARN, map->numa_node); if (!new) return -ENOMEM; diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c index f0d05a3cc4b9..d789e3b831ad 100644 --- a/kernel/bpf/lpm_trie.c +++ b/kernel/bpf/lpm_trie.c @@ -285,7 +285,7 @@ static struct lpm_trie_node *lpm_trie_node_alloc(const struct lpm_trie *trie, if (value) size += trie->map.value_size; - node = bpf_map_kmalloc_node(&trie->map, size, GFP_ATOMIC | __GFP_NOWARN, + node = bpf_map_kmalloc_node(&trie->map, size, GFP_NOWAIT | __GFP_NOWARN, trie->map.numa_node); if (!node) return NULL; diff --git a/kernel/bpf/preload/iterators/Makefile b/kernel/bpf/preload/iterators/Makefile index bfe24f8c5a20..6762b1260f2f 100644 --- a/kernel/bpf/preload/iterators/Makefile +++ b/kernel/bpf/preload/iterators/Makefile @@ -9,7 +9,7 @@ LLVM_STRIP ?= llvm-strip TOOLS_PATH := $(abspath ../../../../tools) BPFTOOL_SRC := $(TOOLS_PATH)/bpf/bpftool BPFTOOL_OUTPUT := $(abs_out)/bpftool -DEFAULT_BPFTOOL := $(OUTPUT)/sbin/bpftool +DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)/bootstrap/bpftool BPFTOOL ?= $(DEFAULT_BPFTOOL) LIBBPF_SRC := $(TOOLS_PATH)/lib/bpf @@ -61,9 +61,5 @@ $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OU OUTPUT=$(abspath $(dir $@))/ prefix= \ DESTDIR=$(LIBBPF_DESTDIR) $(abspath $@) install_headers -$(DEFAULT_BPFTOOL): $(BPFOBJ) | $(BPFTOOL_OUTPUT) - $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOL_SRC) \ - OUTPUT=$(BPFTOOL_OUTPUT)/ \ - LIBBPF_OUTPUT=$(LIBBPF_OUTPUT)/ \ - LIBBPF_DESTDIR=$(LIBBPF_DESTDIR)/ \ - prefix= DESTDIR=$(abs_out)/ install-bin +$(DEFAULT_BPFTOOL): | $(BPFTOOL_OUTPUT) + $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOL_SRC) OUTPUT=$(BPFTOOL_OUTPUT)/ bootstrap diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index ab688d85b2c6..83c7136c5788 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -419,35 +419,53 @@ void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) #ifdef CONFIG_MEMCG_KMEM static void bpf_map_save_memcg(struct bpf_map *map) { - map->memcg = get_mem_cgroup_from_mm(current->mm); + /* Currently if a map is created by a process belonging to the root + * memory cgroup, get_obj_cgroup_from_current() will return NULL. + * So we have to check map->objcg for being NULL each time it's + * being used. + */ + map->objcg = get_obj_cgroup_from_current(); } static void bpf_map_release_memcg(struct bpf_map *map) { - mem_cgroup_put(map->memcg); + if (map->objcg) + obj_cgroup_put(map->objcg); +} + +static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map) +{ + if (map->objcg) + return get_mem_cgroup_from_objcg(map->objcg); + + return root_mem_cgroup; } void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, int node) { - struct mem_cgroup *old_memcg; + struct mem_cgroup *memcg, *old_memcg; void *ptr; - old_memcg = set_active_memcg(map->memcg); + memcg = bpf_map_get_memcg(map); + old_memcg = set_active_memcg(memcg); ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node); set_active_memcg(old_memcg); + mem_cgroup_put(memcg); return ptr; } void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) { - struct mem_cgroup *old_memcg; + struct mem_cgroup *memcg, *old_memcg; void *ptr; - old_memcg = set_active_memcg(map->memcg); + memcg = bpf_map_get_memcg(map); + old_memcg = set_active_memcg(memcg); ptr = kzalloc(size, flags | __GFP_ACCOUNT); set_active_memcg(old_memcg); + mem_cgroup_put(memcg); return ptr; } @@ -455,12 +473,14 @@ void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, gfp_t flags) { - struct mem_cgroup *old_memcg; + struct mem_cgroup *memcg, *old_memcg; void __percpu *ptr; - old_memcg = set_active_memcg(map->memcg); + memcg = bpf_map_get_memcg(map); + old_memcg = set_active_memcg(memcg); ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT); set_active_memcg(old_memcg); + mem_cgroup_put(memcg); return ptr; } diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 6cd226584c33..42e387a12694 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -13,6 +13,7 @@ #include <linux/static_call.h> #include <linux/bpf_verifier.h> #include <linux/bpf_lsm.h> +#include <linux/delay.h> /* dummy _ops. The verifier will operate on target program's ops. */ const struct bpf_verifier_ops bpf_extension_verifier_ops = { @@ -29,6 +30,81 @@ static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; /* serializes access to trampoline_table */ static DEFINE_MUTEX(trampoline_mutex); +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex); + +static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd cmd) +{ + struct bpf_trampoline *tr = ops->private; + int ret = 0; + + if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) { + /* This is called inside register_ftrace_direct_multi(), so + * tr->mutex is already locked. + */ + lockdep_assert_held_once(&tr->mutex); + + /* Instead of updating the trampoline here, we propagate + * -EAGAIN to register_ftrace_direct_multi(). Then we can + * retry register_ftrace_direct_multi() after updating the + * trampoline. + */ + if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && + !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) { + if (WARN_ON_ONCE(tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY)) + return -EBUSY; + + tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY; + return -EAGAIN; + } + + return 0; + } + + /* The normal locking order is + * tr->mutex => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c) + * + * The following two commands are called from + * + * prepare_direct_functions_for_ipmodify + * cleanup_direct_functions_after_ipmodify + * + * In both cases, direct_mutex is already locked. Use + * mutex_trylock(&tr->mutex) to avoid deadlock in race condition + * (something else is making changes to this same trampoline). + */ + if (!mutex_trylock(&tr->mutex)) { + /* sleep 1 ms to make sure whatever holding tr->mutex makes + * some progress. + */ + msleep(1); + return -EAGAIN; + } + + switch (cmd) { + case FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER: + tr->flags |= BPF_TRAMP_F_SHARE_IPMODIFY; + + if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) && + !(tr->flags & BPF_TRAMP_F_ORIG_STACK)) + ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */); + break; + case FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER: + tr->flags &= ~BPF_TRAMP_F_SHARE_IPMODIFY; + + if (tr->flags & BPF_TRAMP_F_ORIG_STACK) + ret = bpf_trampoline_update(tr, false /* lock_direct_mutex */); + break; + default: + ret = -EINVAL; + break; + }; + + mutex_unlock(&tr->mutex); + return ret; +} +#endif + bool bpf_prog_has_trampoline(const struct bpf_prog *prog) { enum bpf_attach_type eatype = prog->expected_attach_type; @@ -89,6 +165,16 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) tr = kzalloc(sizeof(*tr), GFP_KERNEL); if (!tr) goto out; +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + tr->fops = kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL); + if (!tr->fops) { + kfree(tr); + tr = NULL; + goto out; + } + tr->fops->private = tr; + tr->fops->ops_func = bpf_tramp_ftrace_ops_func; +#endif tr->key = key; INIT_HLIST_NODE(&tr->hlist); @@ -128,7 +214,7 @@ static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr) int ret; if (tr->func.ftrace_managed) - ret = unregister_ftrace_direct((long)ip, (long)old_addr); + ret = unregister_ftrace_direct_multi(tr->fops, (long)old_addr); else ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL); @@ -137,15 +223,20 @@ static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr) return ret; } -static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr) +static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr, + bool lock_direct_mutex) { void *ip = tr->func.addr; int ret; - if (tr->func.ftrace_managed) - ret = modify_ftrace_direct((long)ip, (long)old_addr, (long)new_addr); - else + if (tr->func.ftrace_managed) { + if (lock_direct_mutex) + ret = modify_ftrace_direct_multi(tr->fops, (long)new_addr); + else + ret = modify_ftrace_direct_multi_nolock(tr->fops, (long)new_addr); + } else { ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr); + } return ret; } @@ -163,10 +254,12 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) if (bpf_trampoline_module_get(tr)) return -ENOENT; - if (tr->func.ftrace_managed) - ret = register_ftrace_direct((long)ip, (long)new_addr); - else + if (tr->func.ftrace_managed) { + ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 0); + ret = register_ftrace_direct_multi(tr->fops, (long)new_addr); + } else { ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr); + } if (ret) bpf_trampoline_module_put(tr); @@ -332,11 +425,11 @@ out: return ERR_PTR(err); } -static int bpf_trampoline_update(struct bpf_trampoline *tr) +static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex) { struct bpf_tramp_image *im; struct bpf_tramp_links *tlinks; - u32 flags = BPF_TRAMP_F_RESTORE_REGS; + u32 orig_flags = tr->flags; bool ip_arg = false; int err, total; @@ -358,15 +451,31 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) goto out; } + /* clear all bits except SHARE_IPMODIFY */ + tr->flags &= BPF_TRAMP_F_SHARE_IPMODIFY; + if (tlinks[BPF_TRAMP_FEXIT].nr_links || - tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links) - flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; + tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links) { + /* NOTE: BPF_TRAMP_F_RESTORE_REGS and BPF_TRAMP_F_SKIP_FRAME + * should not be set together. + */ + tr->flags |= BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME; + } else { + tr->flags |= BPF_TRAMP_F_RESTORE_REGS; + } if (ip_arg) - flags |= BPF_TRAMP_F_IP_ARG; + tr->flags |= BPF_TRAMP_F_IP_ARG; + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +again: + if ((tr->flags & BPF_TRAMP_F_SHARE_IPMODIFY) && + (tr->flags & BPF_TRAMP_F_CALL_ORIG)) + tr->flags |= BPF_TRAMP_F_ORIG_STACK; +#endif err = arch_prepare_bpf_trampoline(im, im->image, im->image + PAGE_SIZE, - &tr->func.model, flags, tlinks, + &tr->func.model, tr->flags, tlinks, tr->func.addr); if (err < 0) goto out; @@ -375,17 +484,34 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) WARN_ON(!tr->cur_image && tr->selector); if (tr->cur_image) /* progs already running at this address */ - err = modify_fentry(tr, tr->cur_image->image, im->image); + err = modify_fentry(tr, tr->cur_image->image, im->image, lock_direct_mutex); else /* first time registering */ err = register_fentry(tr, im->image); + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS + if (err == -EAGAIN) { + /* -EAGAIN from bpf_tramp_ftrace_ops_func. Now + * BPF_TRAMP_F_SHARE_IPMODIFY is set, we can generate the + * trampoline again, and retry register. + */ + /* reset fops->func and fops->trampoline for re-register */ + tr->fops->func = NULL; + tr->fops->trampoline = 0; + goto again; + } +#endif if (err) goto out; + if (tr->cur_image) bpf_tramp_image_put(tr->cur_image); tr->cur_image = im; tr->selector++; out: + /* If any error happens, restore previous flags */ + if (err) + tr->flags = orig_flags; kfree(tlinks); return err; } @@ -451,7 +577,7 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]); tr->progs_cnt[kind]++; - err = bpf_trampoline_update(tr); + err = bpf_trampoline_update(tr, true /* lock_direct_mutex */); if (err) { hlist_del_init(&link->tramp_hlist); tr->progs_cnt[kind]--; @@ -484,7 +610,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_ } hlist_del_init(&link->tramp_hlist); tr->progs_cnt[kind]--; - return bpf_trampoline_update(tr); + return bpf_trampoline_update(tr, true /* lock_direct_mutex */); } /* bpf_trampoline_unlink_prog() should never fail. */ @@ -498,7 +624,7 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampolin return err; } -#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) +#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM) static void bpf_shim_tramp_link_release(struct bpf_link *link) { struct bpf_shim_tramp_link *shim_link = @@ -712,6 +838,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) * multiple rcu callbacks. */ hlist_del(&tr->hlist); + kfree(tr->fops); kfree(tr); out: mutex_unlock(&trampoline_mutex); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 328cfab3af60..096fdac70165 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5533,17 +5533,6 @@ static bool arg_type_is_mem_size(enum bpf_arg_type type) type == ARG_CONST_SIZE_OR_ZERO; } -static bool arg_type_is_alloc_size(enum bpf_arg_type type) -{ - return type == ARG_CONST_ALLOC_SIZE_OR_ZERO; -} - -static bool arg_type_is_int_ptr(enum bpf_arg_type type) -{ - return type == ARG_PTR_TO_INT || - type == ARG_PTR_TO_LONG; -} - static bool arg_type_is_release(enum bpf_arg_type type) { return type & OBJ_RELEASE; @@ -5929,7 +5918,8 @@ skip_type_check: meta->ref_obj_id = reg->ref_obj_id; } - if (arg_type == ARG_CONST_MAP_PTR) { + switch (base_type(arg_type)) { + case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ if (meta->map_ptr) { /* Use map_uid (which is unique id of inner map) to reject: @@ -5954,7 +5944,8 @@ skip_type_check: } meta->map_ptr = reg->map_ptr; meta->map_uid = reg->map_uid; - } else if (arg_type == ARG_PTR_TO_MAP_KEY) { + break; + case ARG_PTR_TO_MAP_KEY: /* bpf_map_xxx(..., map_ptr, ..., key) call: * check that [key, key + map->key_size) are within * stack limits and initialized @@ -5971,7 +5962,8 @@ skip_type_check: err = check_helper_mem_access(env, regno, meta->map_ptr->key_size, false, NULL); - } else if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { + break; + case ARG_PTR_TO_MAP_VALUE: if (type_may_be_null(arg_type) && register_is_null(reg)) return 0; @@ -5987,14 +5979,16 @@ skip_type_check: err = check_helper_mem_access(env, regno, meta->map_ptr->value_size, false, meta); - } else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) { + break; + case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { verbose(env, "Helper has invalid btf_id in R%d\n", regno); return -EACCES; } meta->ret_btf = reg->btf; meta->ret_btf_id = reg->btf_id; - } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { + break; + case ARG_PTR_TO_SPIN_LOCK: if (meta->func_id == BPF_FUNC_spin_lock) { if (process_spin_lock(env, regno, true)) return -EACCES; @@ -6005,12 +5999,15 @@ skip_type_check: verbose(env, "verifier internal error\n"); return -EFAULT; } - } else if (arg_type == ARG_PTR_TO_TIMER) { + break; + case ARG_PTR_TO_TIMER: if (process_timer_func(env, regno, meta)) return -EACCES; - } else if (arg_type == ARG_PTR_TO_FUNC) { + break; + case ARG_PTR_TO_FUNC: meta->subprogno = reg->subprogno; - } else if (base_type(arg_type) == ARG_PTR_TO_MEM) { + break; + case ARG_PTR_TO_MEM: /* The access to this pointer is only checked when we hit the * next is_mem_size argument below. */ @@ -6020,11 +6017,14 @@ skip_type_check: fn->arg_size[arg], false, meta); } - } else if (arg_type_is_mem_size(arg_type)) { - bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); - - err = check_mem_size_reg(env, reg, regno, zero_size_allowed, meta); - } else if (arg_type_is_dynptr(arg_type)) { + break; + case ARG_CONST_SIZE: + err = check_mem_size_reg(env, reg, regno, false, meta); + break; + case ARG_CONST_SIZE_OR_ZERO: + err = check_mem_size_reg(env, reg, regno, true, meta); + break; + case ARG_PTR_TO_DYNPTR: if (arg_type & MEM_UNINIT) { if (!is_dynptr_reg_valid_uninit(env, reg)) { verbose(env, "Dynptr has to be an uninitialized dynptr\n"); @@ -6058,21 +6058,28 @@ skip_type_check: err_extra, arg + 1); return -EINVAL; } - } else if (arg_type_is_alloc_size(arg_type)) { + break; + case ARG_CONST_ALLOC_SIZE_OR_ZERO: if (!tnum_is_const(reg->var_off)) { verbose(env, "R%d is not a known constant'\n", regno); return -EACCES; } meta->mem_size = reg->var_off.value; - } else if (arg_type_is_int_ptr(arg_type)) { + break; + case ARG_PTR_TO_INT: + case ARG_PTR_TO_LONG: + { int size = int_ptr_type_to_size(arg_type); err = check_helper_mem_access(env, regno, size, false, meta); if (err) return err; err = check_ptr_alignment(env, reg, 0, size, true); - } else if (arg_type == ARG_PTR_TO_CONST_STR) { + break; + } + case ARG_PTR_TO_CONST_STR: + { struct bpf_map *map = reg->map_ptr; int map_off; u64 map_addr; @@ -6111,9 +6118,12 @@ skip_type_check: verbose(env, "string is not zero-terminated\n"); return -EINVAL; } - } else if (arg_type == ARG_PTR_TO_KPTR) { + break; + } + case ARG_PTR_TO_KPTR: if (process_kptr_func(env, regno, meta)) return -EACCES; + break; } return err; @@ -7160,6 +7170,7 @@ static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx_p) { + enum bpf_prog_type prog_type = resolve_prog_type(env->prog); const struct bpf_func_proto *fn = NULL; enum bpf_return_type ret_type; enum bpf_type_flag ret_flag; @@ -7321,7 +7332,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } break; case BPF_FUNC_set_retval: - if (env->prog->expected_attach_type == BPF_LSM_CGROUP) { + if (prog_type == BPF_PROG_TYPE_LSM && + env->prog->expected_attach_type == BPF_LSM_CGROUP) { if (!env->prog->aux->attach_func_proto->type) { /* Make sure programs that attach to void * hooks don't try to modify return value. @@ -7550,6 +7562,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int err, insn_idx = *insn_idx_p; const struct btf_param *args; struct btf *desc_btf; + u32 *kfunc_flags; bool acq; /* skip for now, but return error when we find this in fixup_kfunc_call */ @@ -7565,18 +7578,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, func_name = btf_name_by_offset(desc_btf, func->name_off); func_proto = btf_type_by_id(desc_btf, func->type); - if (!btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), - BTF_KFUNC_TYPE_CHECK, func_id)) { + kfunc_flags = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), func_id); + if (!kfunc_flags) { verbose(env, "calling kernel function %s is not allowed\n", func_name); return -EACCES; } - - acq = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), - BTF_KFUNC_TYPE_ACQUIRE, func_id); + acq = *kfunc_flags & KF_ACQUIRE; /* Check the arguments */ - err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs); + err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs, *kfunc_flags); if (err < 0) return err; /* In case of release function, we get register number of refcounted @@ -7620,8 +7631,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].type = PTR_TO_BTF_ID; regs[BPF_REG_0].btf_id = ptr_type_id; - if (btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), - BTF_KFUNC_TYPE_RET_NULL, func_id)) { + if (*kfunc_flags & KF_RET_NULL) { regs[BPF_REG_0].type |= PTR_MAYBE_NULL; /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ regs[BPF_REG_0].id = ++env->id_gen; @@ -12562,6 +12572,7 @@ static bool is_tracing_prog_type(enum bpf_prog_type type) case BPF_PROG_TYPE_TRACEPOINT: case BPF_PROG_TYPE_PERF_EVENT: case BPF_PROG_TYPE_RAW_TRACEPOINT: + case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: return true; default: return false; @@ -13620,6 +13631,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) /* Below members will be freed only at prog->aux */ func[i]->aux->btf = prog->aux->btf; func[i]->aux->func_info = prog->aux->func_info; + func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; func[i]->aux->poke_tab = prog->aux->poke_tab; func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; @@ -13632,9 +13644,6 @@ static int jit_subprogs(struct bpf_verifier_env *env) poke->aux = func[i]->aux; } - /* Use bpf_prog_F_tag to indicate functions in stack traces. - * Long term would need debug info to populate names - */ func[i]->aux->name[0] = 'F'; func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; func[i]->jit_requested = 1; diff --git a/kernel/events/core.c b/kernel/events/core.c index 48bae58d240e..07d51f95d6a6 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6253,10 +6253,10 @@ again: if (!atomic_inc_not_zero(&event->rb->mmap_count)) { /* - * Raced against perf_mmap_close() through - * perf_event_set_output(). Try again, hope for better - * luck. + * Raced against perf_mmap_close(); remove the + * event and try again. */ + ring_buffer_attach(event, NULL); mutex_unlock(&event->mmap_mutex); goto again; } @@ -11829,14 +11829,25 @@ err_size: goto out; } +static void mutex_lock_double(struct mutex *a, struct mutex *b) +{ + if (b < a) + swap(a, b); + + mutex_lock(a); + mutex_lock_nested(b, SINGLE_DEPTH_NESTING); +} + static int perf_event_set_output(struct perf_event *event, struct perf_event *output_event) { struct perf_buffer *rb = NULL; int ret = -EINVAL; - if (!output_event) + if (!output_event) { + mutex_lock(&event->mmap_mutex); goto set; + } /* don't allow circular references */ if (event == output_event) @@ -11874,8 +11885,15 @@ perf_event_set_output(struct perf_event *event, struct perf_event *output_event) event->pmu != output_event->pmu) goto out; + /* + * Hold both mmap_mutex to serialize against perf_mmap_close(). Since + * output_event is already on rb->event_list, and the list iteration + * restarts after every removal, it is guaranteed this new event is + * observed *OR* if output_event is already removed, it's guaranteed we + * observe !rb->mmap_count. + */ + mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex); set: - mutex_lock(&event->mmap_mutex); /* Can't redirect output if we've got an active mmap() */ if (atomic_read(&event->mmap_count)) goto unlock; @@ -11885,6 +11903,12 @@ set: rb = ring_buffer_get(output_event); if (!rb) goto unlock; + + /* did we race against perf_mmap_close() */ + if (!atomic_read(&rb->mmap_count)) { + ring_buffer_put(rb); + goto unlock; + } } ring_buffer_attach(event, rb); @@ -11892,20 +11916,13 @@ set: ret = 0; unlock: mutex_unlock(&event->mmap_mutex); + if (output_event) + mutex_unlock(&output_event->mmap_mutex); out: return ret; } -static void mutex_lock_double(struct mutex *a, struct mutex *b) -{ - if (b < a) - swap(a, b); - - mutex_lock(a); - mutex_lock_nested(b, SINGLE_DEPTH_NESTING); -} - static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id) { bool nmi_safe = false; diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index fbdf8d3279ac..79a85834ce9d 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -30,6 +30,7 @@ #include <linux/module.h> #include <linux/kernel.h> #include <linux/bsearch.h> +#include <linux/btf_ids.h> /* * These will be re-linked against their real values @@ -799,6 +800,96 @@ static const struct seq_operations kallsyms_op = { .show = s_show }; +#ifdef CONFIG_BPF_SYSCALL + +struct bpf_iter__ksym { + __bpf_md_ptr(struct bpf_iter_meta *, meta); + __bpf_md_ptr(struct kallsym_iter *, ksym); +}; + +static int ksym_prog_seq_show(struct seq_file *m, bool in_stop) +{ + struct bpf_iter__ksym ctx; + struct bpf_iter_meta meta; + struct bpf_prog *prog; + + meta.seq = m; + prog = bpf_iter_get_info(&meta, in_stop); + if (!prog) + return 0; + + ctx.meta = &meta; + ctx.ksym = m ? m->private : NULL; + return bpf_iter_run_prog(prog, &ctx); +} + +static int bpf_iter_ksym_seq_show(struct seq_file *m, void *p) +{ + return ksym_prog_seq_show(m, false); +} + +static void bpf_iter_ksym_seq_stop(struct seq_file *m, void *p) +{ + if (!p) + (void) ksym_prog_seq_show(m, true); + else + s_stop(m, p); +} + +static const struct seq_operations bpf_iter_ksym_ops = { + .start = s_start, + .next = s_next, + .stop = bpf_iter_ksym_seq_stop, + .show = bpf_iter_ksym_seq_show, +}; + +static int bpf_iter_ksym_init(void *priv_data, struct bpf_iter_aux_info *aux) +{ + struct kallsym_iter *iter = priv_data; + + reset_iter(iter, 0); + + /* cache here as in kallsyms_open() case; use current process + * credentials to tell BPF iterators if values should be shown. + */ + iter->show_value = kallsyms_show_value(current_cred()); + + return 0; +} + +DEFINE_BPF_ITER_FUNC(ksym, struct bpf_iter_meta *meta, struct kallsym_iter *ksym) + +static const struct bpf_iter_seq_info ksym_iter_seq_info = { + .seq_ops = &bpf_iter_ksym_ops, + .init_seq_private = bpf_iter_ksym_init, + .fini_seq_private = NULL, + .seq_priv_size = sizeof(struct kallsym_iter), +}; + +static struct bpf_iter_reg ksym_iter_reg_info = { + .target = "ksym", + .feature = BPF_ITER_RESCHED, + .ctx_arg_info_size = 1, + .ctx_arg_info = { + { offsetof(struct bpf_iter__ksym, ksym), + PTR_TO_BTF_ID_OR_NULL }, + }, + .seq_info = &ksym_iter_seq_info, +}; + +BTF_ID_LIST(btf_ksym_iter_id) +BTF_ID(struct, kallsym_iter) + +static int __init bpf_ksym_iter_register(void) +{ + ksym_iter_reg_info.ctx_arg_info[0].btf_id = *btf_ksym_iter_id; + return bpf_iter_reg_target(&ksym_iter_reg_info); +} + +late_initcall(bpf_ksym_iter_register); + +#endif /* CONFIG_BPF_SYSCALL */ + static inline int kallsyms_for_perf(void) { #ifdef CONFIG_PERF_EVENTS diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index b49c6ff6dca0..a1a81fd9889b 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3380,6 +3380,7 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre diff = 0; console_lock(); + for_each_console(c) { if (con && con != c) continue; @@ -3389,11 +3390,19 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre if (printk_seq < seq) diff += seq - printk_seq; } - console_unlock(); - if (diff != last_diff && reset_on_progress) + /* + * If consoles are suspended, it cannot be expected that they + * make forward progress, so timeout immediately. @diff is + * still used to return a valid flush status. + */ + if (console_suspended) + remaining = 0; + else if (diff != last_diff && reset_on_progress) remaining = timeout_ms; + console_unlock(); + if (diff == 0 || remaining == 0) break; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 619b02443522..b233714a1c78 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2135,6 +2135,17 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_TWO_HUNDRED, }, +#ifdef CONFIG_NUMA + { + .procname = "numa_stat", + .data = &sysctl_vm_numa_stat, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = sysctl_vm_numa_stat_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +#endif #ifdef CONFIG_HUGETLB_PAGE { .procname = "nr_hugepages", @@ -2151,15 +2162,6 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = &hugetlb_mempolicy_sysctl_handler, }, - { - .procname = "numa_stat", - .data = &sysctl_vm_numa_stat, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = sysctl_vm_numa_stat_handler, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, #endif { .procname = "hugetlb_shm_group", diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 601ccf1b2f09..bc921a3f7ea8 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1861,6 +1861,8 @@ static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, ftrace_hash_rec_update_modify(ops, filter_hash, 1); } +static bool ops_references_ip(struct ftrace_ops *ops, unsigned long ip); + /* * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK * or no-needed to update, -EBUSY if it detects a conflict of the flag @@ -1869,6 +1871,13 @@ static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected) * - If the hash is EMPTY_HASH, it hits nothing * - Anything else hits the recs which match the hash entries. + * + * DIRECT ops does not have IPMODIFY flag, but we still need to check it + * against functions with FTRACE_FL_IPMODIFY. If there is any overlap, call + * ops_func(SHARE_IPMODIFY_SELF) to make sure current ops can share with + * IPMODIFY. If ops_func(SHARE_IPMODIFY_SELF) returns non-zero, propagate + * the return value to the caller and eventually to the owner of the DIRECT + * ops. */ static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, struct ftrace_hash *old_hash, @@ -1877,17 +1886,26 @@ static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, struct ftrace_page *pg; struct dyn_ftrace *rec, *end = NULL; int in_old, in_new; + bool is_ipmodify, is_direct; /* Only update if the ops has been registered */ if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) return 0; - if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) + is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY; + is_direct = ops->flags & FTRACE_OPS_FL_DIRECT; + + /* neither IPMODIFY nor DIRECT, skip */ + if (!is_ipmodify && !is_direct) + return 0; + + if (WARN_ON_ONCE(is_ipmodify && is_direct)) return 0; /* - * Since the IPMODIFY is a very address sensitive action, we do not - * allow ftrace_ops to set all functions to new hash. + * Since the IPMODIFY and DIRECT are very address sensitive + * actions, we do not allow ftrace_ops to set all functions to new + * hash. */ if (!new_hash || !old_hash) return -EINVAL; @@ -1905,12 +1923,32 @@ static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, continue; if (in_new) { - /* New entries must ensure no others are using it */ - if (rec->flags & FTRACE_FL_IPMODIFY) - goto rollback; - rec->flags |= FTRACE_FL_IPMODIFY; - } else /* Removed entry */ + if (rec->flags & FTRACE_FL_IPMODIFY) { + int ret; + + /* Cannot have two ipmodify on same rec */ + if (is_ipmodify) + goto rollback; + + FTRACE_WARN_ON(rec->flags & FTRACE_FL_DIRECT); + + /* + * Another ops with IPMODIFY is already + * attached. We are now attaching a direct + * ops. Run SHARE_IPMODIFY_SELF, to check + * whether sharing is supported. + */ + if (!ops->ops_func) + return -EBUSY; + ret = ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF); + if (ret) + return ret; + } else if (is_ipmodify) { + rec->flags |= FTRACE_FL_IPMODIFY; + } + } else if (is_ipmodify) { rec->flags &= ~FTRACE_FL_IPMODIFY; + } } while_for_each_ftrace_rec(); return 0; @@ -2454,8 +2492,7 @@ static void call_direct_funcs(unsigned long ip, unsigned long pip, struct ftrace_ops direct_ops = { .func = call_direct_funcs, - .flags = FTRACE_OPS_FL_IPMODIFY - | FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS + .flags = FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_PERMANENT, /* * By declaring the main trampoline as this trampoline @@ -3072,14 +3109,14 @@ static inline int ops_traces_mod(struct ftrace_ops *ops) } /* - * Check if the current ops references the record. + * Check if the current ops references the given ip. * * If the ops traces all functions, then it was already accounted for. * If the ops does not trace the current record function, skip it. * If the ops ignores the function via notrace filter, skip it. */ -static inline bool -ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) +static bool +ops_references_ip(struct ftrace_ops *ops, unsigned long ip) { /* If ops isn't enabled, ignore it */ if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) @@ -3091,16 +3128,29 @@ ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) /* The function must be in the filter */ if (!ftrace_hash_empty(ops->func_hash->filter_hash) && - !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip)) + !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip)) return false; /* If in notrace hash, we ignore it too */ - if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) + if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip)) return false; return true; } +/* + * Check if the current ops references the record. + * + * If the ops traces all functions, then it was already accounted for. + * If the ops does not trace the current record function, skip it. + * If the ops ignores the function via notrace filter, skip it. + */ +static bool +ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) +{ + return ops_references_ip(ops, rec->ip); +} + static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) { bool init_nop = ftrace_need_init_nop(); @@ -5215,6 +5265,8 @@ static struct ftrace_direct_func *ftrace_alloc_direct_func(unsigned long addr) return direct; } +static int register_ftrace_function_nolock(struct ftrace_ops *ops); + /** * register_ftrace_direct - Call a custom trampoline directly * @ip: The address of the nop at the beginning of a function @@ -5286,7 +5338,7 @@ int register_ftrace_direct(unsigned long ip, unsigned long addr) ret = ftrace_set_filter_ip(&direct_ops, ip, 0, 0); if (!ret && !(direct_ops.flags & FTRACE_OPS_FL_ENABLED)) { - ret = register_ftrace_function(&direct_ops); + ret = register_ftrace_function_nolock(&direct_ops); if (ret) ftrace_set_filter_ip(&direct_ops, ip, 1, 0); } @@ -5545,8 +5597,7 @@ int modify_ftrace_direct(unsigned long ip, } EXPORT_SYMBOL_GPL(modify_ftrace_direct); -#define MULTI_FLAGS (FTRACE_OPS_FL_IPMODIFY | FTRACE_OPS_FL_DIRECT | \ - FTRACE_OPS_FL_SAVE_REGS) +#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS) static int check_direct_multi(struct ftrace_ops *ops) { @@ -5639,7 +5690,7 @@ int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) ops->flags = MULTI_FLAGS; ops->trampoline = FTRACE_REGS_ADDR; - err = register_ftrace_function(ops); + err = register_ftrace_function_nolock(ops); out_remove: if (err) @@ -5691,22 +5742,8 @@ int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) } EXPORT_SYMBOL_GPL(unregister_ftrace_direct_multi); -/** - * modify_ftrace_direct_multi - Modify an existing direct 'multi' call - * to call something else - * @ops: The address of the struct ftrace_ops object - * @addr: The address of the new trampoline to call at @ops functions - * - * This is used to unregister currently registered direct caller and - * register new one @addr on functions registered in @ops object. - * - * Note there's window between ftrace_shutdown and ftrace_startup calls - * where there will be no callbacks called. - * - * Returns: zero on success. Non zero on error, which includes: - * -EINVAL - The @ops object was not properly registered. - */ -int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) +static int +__modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) { struct ftrace_hash *hash; struct ftrace_func_entry *entry, *iter; @@ -5717,20 +5754,15 @@ int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) int i, size; int err; - if (check_direct_multi(ops)) - return -EINVAL; - if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) - return -EINVAL; - - mutex_lock(&direct_mutex); + lockdep_assert_held_once(&direct_mutex); /* Enable the tmp_ops to have the same functions as the direct ops */ ftrace_ops_init(&tmp_ops); tmp_ops.func_hash = ops->func_hash; - err = register_ftrace_function(&tmp_ops); + err = register_ftrace_function_nolock(&tmp_ops); if (err) - goto out_direct; + return err; /* * Now the ftrace_ops_list_func() is called to do the direct callers. @@ -5754,7 +5786,64 @@ int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) /* Removing the tmp_ops will add the updated direct callers to the functions */ unregister_ftrace_function(&tmp_ops); - out_direct: + return err; +} + +/** + * modify_ftrace_direct_multi_nolock - Modify an existing direct 'multi' call + * to call something else + * @ops: The address of the struct ftrace_ops object + * @addr: The address of the new trampoline to call at @ops functions + * + * This is used to unregister currently registered direct caller and + * register new one @addr on functions registered in @ops object. + * + * Note there's window between ftrace_shutdown and ftrace_startup calls + * where there will be no callbacks called. + * + * Caller should already have direct_mutex locked, so we don't lock + * direct_mutex here. + * + * Returns: zero on success. Non zero on error, which includes: + * -EINVAL - The @ops object was not properly registered. + */ +int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr) +{ + if (check_direct_multi(ops)) + return -EINVAL; + if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) + return -EINVAL; + + return __modify_ftrace_direct_multi(ops, addr); +} +EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi_nolock); + +/** + * modify_ftrace_direct_multi - Modify an existing direct 'multi' call + * to call something else + * @ops: The address of the struct ftrace_ops object + * @addr: The address of the new trampoline to call at @ops functions + * + * This is used to unregister currently registered direct caller and + * register new one @addr on functions registered in @ops object. + * + * Note there's window between ftrace_shutdown and ftrace_startup calls + * where there will be no callbacks called. + * + * Returns: zero on success. Non zero on error, which includes: + * -EINVAL - The @ops object was not properly registered. + */ +int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr) +{ + int err; + + if (check_direct_multi(ops)) + return -EINVAL; + if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) + return -EINVAL; + + mutex_lock(&direct_mutex); + err = __modify_ftrace_direct_multi(ops, addr); mutex_unlock(&direct_mutex); return err; } @@ -7965,6 +8054,143 @@ int ftrace_is_dead(void) return ftrace_disabled; } +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +/* + * When registering ftrace_ops with IPMODIFY, it is necessary to make sure + * it doesn't conflict with any direct ftrace_ops. If there is existing + * direct ftrace_ops on a kernel function being patched, call + * FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER on it to enable sharing. + * + * @ops: ftrace_ops being registered. + * + * Returns: + * 0 on success; + * Negative on failure. + */ +static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops) +{ + struct ftrace_func_entry *entry; + struct ftrace_hash *hash; + struct ftrace_ops *op; + int size, i, ret; + + lockdep_assert_held_once(&direct_mutex); + + if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) + return 0; + + hash = ops->func_hash->filter_hash; + size = 1 << hash->size_bits; + for (i = 0; i < size; i++) { + hlist_for_each_entry(entry, &hash->buckets[i], hlist) { + unsigned long ip = entry->ip; + bool found_op = false; + + mutex_lock(&ftrace_lock); + do_for_each_ftrace_op(op, ftrace_ops_list) { + if (!(op->flags & FTRACE_OPS_FL_DIRECT)) + continue; + if (ops_references_ip(op, ip)) { + found_op = true; + break; + } + } while_for_each_ftrace_op(op); + mutex_unlock(&ftrace_lock); + + if (found_op) { + if (!op->ops_func) + return -EBUSY; + + ret = op->ops_func(op, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER); + if (ret) + return ret; + } + } + } + + return 0; +} + +/* + * Similar to prepare_direct_functions_for_ipmodify, clean up after ops + * with IPMODIFY is unregistered. The cleanup is optional for most DIRECT + * ops. + */ +static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops) +{ + struct ftrace_func_entry *entry; + struct ftrace_hash *hash; + struct ftrace_ops *op; + int size, i; + + if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) + return; + + mutex_lock(&direct_mutex); + + hash = ops->func_hash->filter_hash; + size = 1 << hash->size_bits; + for (i = 0; i < size; i++) { + hlist_for_each_entry(entry, &hash->buckets[i], hlist) { + unsigned long ip = entry->ip; + bool found_op = false; + + mutex_lock(&ftrace_lock); + do_for_each_ftrace_op(op, ftrace_ops_list) { + if (!(op->flags & FTRACE_OPS_FL_DIRECT)) + continue; + if (ops_references_ip(op, ip)) { + found_op = true; + break; + } + } while_for_each_ftrace_op(op); + mutex_unlock(&ftrace_lock); + + /* The cleanup is optional, ignore any errors */ + if (found_op && op->ops_func) + op->ops_func(op, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER); + } + } + mutex_unlock(&direct_mutex); +} + +#define lock_direct_mutex() mutex_lock(&direct_mutex) +#define unlock_direct_mutex() mutex_unlock(&direct_mutex) + +#else /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ + +static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops) +{ + return 0; +} + +static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops) +{ +} + +#define lock_direct_mutex() do { } while (0) +#define unlock_direct_mutex() do { } while (0) + +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ + +/* + * Similar to register_ftrace_function, except we don't lock direct_mutex. + */ +static int register_ftrace_function_nolock(struct ftrace_ops *ops) +{ + int ret; + + ftrace_ops_init(ops); + + mutex_lock(&ftrace_lock); + + ret = ftrace_startup(ops, 0); + + mutex_unlock(&ftrace_lock); + + return ret; +} + /** * register_ftrace_function - register a function for profiling * @ops: ops structure that holds the function for profiling. @@ -7980,14 +8206,15 @@ int register_ftrace_function(struct ftrace_ops *ops) { int ret; - ftrace_ops_init(ops); - - mutex_lock(&ftrace_lock); - - ret = ftrace_startup(ops, 0); + lock_direct_mutex(); + ret = prepare_direct_functions_for_ipmodify(ops); + if (ret < 0) + goto out_unlock; - mutex_unlock(&ftrace_lock); + ret = register_ftrace_function_nolock(ops); +out_unlock: + unlock_direct_mutex(); return ret; } EXPORT_SYMBOL_GPL(register_ftrace_function); @@ -8006,6 +8233,7 @@ int unregister_ftrace_function(struct ftrace_ops *ops) ret = ftrace_shutdown(ops, 0); mutex_unlock(&ftrace_lock); + cleanup_direct_functions_after_ipmodify(ops); return ret; } EXPORT_SYMBOL_GPL(unregister_ftrace_function); diff --git a/kernel/watch_queue.c b/kernel/watch_queue.c index 230038d4f908..bb9962b33f95 100644 --- a/kernel/watch_queue.c +++ b/kernel/watch_queue.c @@ -34,6 +34,27 @@ MODULE_LICENSE("GPL"); #define WATCH_QUEUE_NOTE_SIZE 128 #define WATCH_QUEUE_NOTES_PER_PAGE (PAGE_SIZE / WATCH_QUEUE_NOTE_SIZE) +/* + * This must be called under the RCU read-lock, which makes + * sure that the wqueue still exists. It can then take the lock, + * and check that the wqueue hasn't been destroyed, which in + * turn makes sure that the notification pipe still exists. + */ +static inline bool lock_wqueue(struct watch_queue *wqueue) +{ + spin_lock_bh(&wqueue->lock); + if (unlikely(wqueue->defunct)) { + spin_unlock_bh(&wqueue->lock); + return false; + } + return true; +} + +static inline void unlock_wqueue(struct watch_queue *wqueue) +{ + spin_unlock_bh(&wqueue->lock); +} + static void watch_queue_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { @@ -69,6 +90,10 @@ static const struct pipe_buf_operations watch_queue_pipe_buf_ops = { /* * Post a notification to a watch queue. + * + * Must be called with the RCU lock for reading, and the + * watch_queue lock held, which guarantees that the pipe + * hasn't been released. */ static bool post_one_notification(struct watch_queue *wqueue, struct watch_notification *n) @@ -85,9 +110,6 @@ static bool post_one_notification(struct watch_queue *wqueue, spin_lock_irq(&pipe->rd_wait.lock); - if (wqueue->defunct) - goto out; - mask = pipe->ring_size - 1; head = pipe->head; tail = pipe->tail; @@ -203,7 +225,10 @@ void __post_watch_notification(struct watch_list *wlist, if (security_post_notification(watch->cred, cred, n) < 0) continue; - post_one_notification(wqueue, n); + if (lock_wqueue(wqueue)) { + post_one_notification(wqueue, n); + unlock_wqueue(wqueue); + } } rcu_read_unlock(); @@ -462,11 +487,12 @@ int add_watch_to_object(struct watch *watch, struct watch_list *wlist) return -EAGAIN; } - spin_lock_bh(&wqueue->lock); - kref_get(&wqueue->usage); - kref_get(&watch->usage); - hlist_add_head(&watch->queue_node, &wqueue->watches); - spin_unlock_bh(&wqueue->lock); + if (lock_wqueue(wqueue)) { + kref_get(&wqueue->usage); + kref_get(&watch->usage); + hlist_add_head(&watch->queue_node, &wqueue->watches); + unlock_wqueue(wqueue); + } hlist_add_head(&watch->list_node, &wlist->watchers); return 0; @@ -520,20 +546,15 @@ found: wqueue = rcu_dereference(watch->queue); - /* We don't need the watch list lock for the next bit as RCU is - * protecting *wqueue from deallocation. - */ - if (wqueue) { + if (lock_wqueue(wqueue)) { post_one_notification(wqueue, &n.watch); - spin_lock_bh(&wqueue->lock); - if (!hlist_unhashed(&watch->queue_node)) { hlist_del_init_rcu(&watch->queue_node); put_watch(watch); } - spin_unlock_bh(&wqueue->lock); + unlock_wqueue(wqueue); } if (wlist->release_watch) { diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index a9f7eb047768..fd15230a703b 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -84,6 +84,9 @@ config UBSAN_SHIFT config UBSAN_DIV_ZERO bool "Perform checking for integer divide-by-zero" depends on $(cc-option,-fsanitize=integer-divide-by-zero) + # https://github.com/ClangBuiltLinux/linux/issues/1657 + # https://github.com/llvm/llvm-project/issues/56289 + depends on !CC_IS_CLANG help This option enables -fsanitize=integer-divide-by-zero which checks for integer division by zero. This is effectively redundant with the diff --git a/net/bluetooth/Kconfig b/net/bluetooth/Kconfig index e0ab4cd7afc3..ae3bdc6dfc92 100644 --- a/net/bluetooth/Kconfig +++ b/net/bluetooth/Kconfig @@ -29,6 +29,7 @@ menuconfig BT SCO audio links L2CAP (Logical Link Control and Adaptation Protocol) SMP (Security Manager Protocol) on LE (Low Energy) links + ISO isochronous links HCI Device drivers (Interface to the hardware) RFCOMM Module (RFCOMM Protocol) BNEP Module (Bluetooth Network Encapsulation Protocol) diff --git a/net/bluetooth/Makefile b/net/bluetooth/Makefile index a52bba8500e1..0e7b7db42750 100644 --- a/net/bluetooth/Makefile +++ b/net/bluetooth/Makefile @@ -18,6 +18,7 @@ bluetooth-y := af_bluetooth.o hci_core.o hci_conn.o hci_event.o mgmt.o \ eir.o hci_sync.o bluetooth-$(CONFIG_BT_BREDR) += sco.o +bluetooth-$(CONFIG_BT_LE) += iso.o bluetooth-$(CONFIG_BT_HS) += a2mp.o amp.o bluetooth-$(CONFIG_BT_LEDS) += leds.o bluetooth-$(CONFIG_BT_MSFTEXT) += msft.o diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c index b506409bb498..dc65974f5adb 100644 --- a/net/bluetooth/af_bluetooth.c +++ b/net/bluetooth/af_bluetooth.c @@ -38,7 +38,7 @@ #include "selftest.h" /* Bluetooth sockets */ -#define BT_MAX_PROTO 8 +#define BT_MAX_PROTO (BTPROTO_LAST + 1) static const struct net_proto_family *bt_proto[BT_MAX_PROTO]; static DEFINE_RWLOCK(bt_proto_lock); @@ -52,6 +52,7 @@ static const char *const bt_key_strings[BT_MAX_PROTO] = { "sk_lock-AF_BLUETOOTH-BTPROTO_CMTP", "sk_lock-AF_BLUETOOTH-BTPROTO_HIDP", "sk_lock-AF_BLUETOOTH-BTPROTO_AVDTP", + "sk_lock-AF_BLUETOOTH-BTPROTO_ISO", }; static struct lock_class_key bt_slock_key[BT_MAX_PROTO]; @@ -64,6 +65,7 @@ static const char *const bt_slock_key_strings[BT_MAX_PROTO] = { "slock-AF_BLUETOOTH-BTPROTO_CMTP", "slock-AF_BLUETOOTH-BTPROTO_HIDP", "slock-AF_BLUETOOTH-BTPROTO_AVDTP", + "slock-AF_BLUETOOTH-BTPROTO_ISO", }; void bt_sock_reclassify_lock(struct sock *sk, int proto) diff --git a/net/bluetooth/eir.c b/net/bluetooth/eir.c index 7d77fb00c2bf..8a85f6cdfbc1 100644 --- a/net/bluetooth/eir.c +++ b/net/bluetooth/eir.c @@ -13,6 +13,20 @@ #define PNP_INFO_SVCLASS_ID 0x1200 +static u8 eir_append_name(u8 *eir, u16 eir_len, u8 type, u8 *data, u8 data_len) +{ + u8 name[HCI_MAX_SHORT_NAME_LENGTH + 1]; + + /* If data is already NULL terminated just pass it directly */ + if (data[data_len - 1] == '\0') + return eir_append_data(eir, eir_len, type, data, data_len); + + memcpy(name, data, HCI_MAX_SHORT_NAME_LENGTH); + name[HCI_MAX_SHORT_NAME_LENGTH] = '\0'; + + return eir_append_data(eir, eir_len, type, name, sizeof(name)); +} + u8 eir_append_local_name(struct hci_dev *hdev, u8 *ptr, u8 ad_len) { size_t short_len; @@ -23,29 +37,26 @@ u8 eir_append_local_name(struct hci_dev *hdev, u8 *ptr, u8 ad_len) return ad_len; /* use complete name if present and fits */ - complete_len = strlen(hdev->dev_name); + complete_len = strnlen(hdev->dev_name, sizeof(hdev->dev_name)); if (complete_len && complete_len <= HCI_MAX_SHORT_NAME_LENGTH) - return eir_append_data(ptr, ad_len, EIR_NAME_COMPLETE, + return eir_append_name(ptr, ad_len, EIR_NAME_COMPLETE, hdev->dev_name, complete_len + 1); /* use short name if present */ - short_len = strlen(hdev->short_name); + short_len = strnlen(hdev->short_name, sizeof(hdev->short_name)); if (short_len) - return eir_append_data(ptr, ad_len, EIR_NAME_SHORT, - hdev->short_name, short_len + 1); + return eir_append_name(ptr, ad_len, EIR_NAME_SHORT, + hdev->short_name, + short_len == HCI_MAX_SHORT_NAME_LENGTH ? + short_len : short_len + 1); /* use shortened full name if present, we already know that name * is longer then HCI_MAX_SHORT_NAME_LENGTH */ - if (complete_len) { - u8 name[HCI_MAX_SHORT_NAME_LENGTH + 1]; - - memcpy(name, hdev->dev_name, HCI_MAX_SHORT_NAME_LENGTH); - name[HCI_MAX_SHORT_NAME_LENGTH] = '\0'; - - return eir_append_data(ptr, ad_len, EIR_NAME_SHORT, name, - sizeof(name)); - } + if (complete_len) + return eir_append_name(ptr, ad_len, EIR_NAME_SHORT, + hdev->dev_name, + HCI_MAX_SHORT_NAME_LENGTH); return ad_len; } @@ -181,7 +192,7 @@ void eir_create(struct hci_dev *hdev, u8 *data) u8 *ptr = data; size_t name_len; - name_len = strlen(hdev->dev_name); + name_len = strnlen(hdev->dev_name, sizeof(hdev->dev_name)); if (name_len > 0) { /* EIR Data type */ @@ -225,6 +236,27 @@ void eir_create(struct hci_dev *hdev, u8 *data) ptr = create_uuid128_list(hdev, ptr, HCI_MAX_EIR_LENGTH - (ptr - data)); } +u8 eir_create_per_adv_data(struct hci_dev *hdev, u8 instance, u8 *ptr) +{ + struct adv_info *adv = NULL; + u8 ad_len = 0; + + /* Return 0 when the current instance identifier is invalid. */ + if (instance) { + adv = hci_find_adv_instance(hdev, instance); + if (!adv) + return 0; + } + + if (adv) { + memcpy(ptr, adv->per_adv_data, adv->per_adv_data_len); + ad_len += adv->per_adv_data_len; + ptr += adv->per_adv_data_len; + } + + return ad_len; +} + u8 eir_create_adv_data(struct hci_dev *hdev, u8 instance, u8 *ptr) { struct adv_info *adv = NULL; diff --git a/net/bluetooth/eir.h b/net/bluetooth/eir.h index 62f2374078f2..0df19f2f4af9 100644 --- a/net/bluetooth/eir.h +++ b/net/bluetooth/eir.h @@ -11,6 +11,7 @@ void eir_create(struct hci_dev *hdev, u8 *data); u8 eir_create_adv_data(struct hci_dev *hdev, u8 instance, u8 *ptr); u8 eir_create_scan_rsp(struct hci_dev *hdev, u8 instance, u8 *ptr); +u8 eir_create_per_adv_data(struct hci_dev *hdev, u8 instance, u8 *ptr); u8 eir_append_local_name(struct hci_dev *hdev, u8 *eir, u8 ad_len); u8 eir_append_appearance(struct hci_dev *hdev, u8 *ptr, u8 ad_len); diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index ac06c9724c7f..f54864e19866 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -30,10 +30,13 @@ #include <net/bluetooth/bluetooth.h> #include <net/bluetooth/hci_core.h> #include <net/bluetooth/l2cap.h> +#include <net/bluetooth/iso.h> +#include <net/bluetooth/mgmt.h> #include "hci_request.h" #include "smp.h" #include "a2mp.h" +#include "eir.h" struct sco_param { u16 pkt_type; @@ -118,10 +121,16 @@ static void hci_conn_cleanup(struct hci_conn *conn) if (test_bit(HCI_CONN_PARAM_REMOVAL_PEND, &conn->flags)) hci_conn_params_del(conn->hdev, &conn->dst, conn->dst_type); + if (test_and_clear_bit(HCI_CONN_FLUSH_KEY, &conn->flags)) + hci_remove_link_key(hdev, &conn->dst); + hci_chan_list_flush(conn); hci_conn_hash_del(hdev, conn); + if (conn->cleanup) + conn->cleanup(conn); + if (conn->type == SCO_LINK || conn->type == ESCO_LINK) { switch (conn->setting & SCO_AIRMODE_MASK) { case SCO_AIRMODE_CVSD: @@ -678,6 +687,199 @@ static void le_conn_timeout(struct work_struct *work) hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM); } +struct iso_list_data { + union { + u8 cig; + u8 big; + }; + union { + u8 cis; + u8 bis; + u16 sync_handle; + }; + int count; + struct { + struct hci_cp_le_set_cig_params cp; + struct hci_cis_params cis[0x11]; + } pdu; +}; + +static void bis_list(struct hci_conn *conn, void *data) +{ + struct iso_list_data *d = data; + + /* Skip if not broadcast/ANY address */ + if (bacmp(&conn->dst, BDADDR_ANY)) + return; + + if (d->big != conn->iso_qos.big || d->bis == BT_ISO_QOS_BIS_UNSET || + d->bis != conn->iso_qos.bis) + return; + + d->count++; +} + +static void find_bis(struct hci_conn *conn, void *data) +{ + struct iso_list_data *d = data; + + /* Ignore unicast */ + if (bacmp(&conn->dst, BDADDR_ANY)) + return; + + d->count++; +} + +static int terminate_big_sync(struct hci_dev *hdev, void *data) +{ + struct iso_list_data *d = data; + + bt_dev_dbg(hdev, "big 0x%2.2x bis 0x%2.2x", d->big, d->bis); + + hci_remove_ext_adv_instance_sync(hdev, d->bis, NULL); + + /* Check if ISO connection is a BIS and terminate BIG if there are + * no other connections using it. + */ + hci_conn_hash_list_state(hdev, find_bis, ISO_LINK, BT_CONNECTED, d); + if (d->count) + return 0; + + return hci_le_terminate_big_sync(hdev, d->big, + HCI_ERROR_LOCAL_HOST_TERM); +} + +static void terminate_big_destroy(struct hci_dev *hdev, void *data, int err) +{ + kfree(data); +} + +static int hci_le_terminate_big(struct hci_dev *hdev, u8 big, u8 bis) +{ + struct iso_list_data *d; + + bt_dev_dbg(hdev, "big 0x%2.2x bis 0x%2.2x", big, bis); + + d = kmalloc(sizeof(*d), GFP_KERNEL); + if (!d) + return -ENOMEM; + + memset(d, 0, sizeof(*d)); + d->big = big; + d->bis = bis; + + return hci_cmd_sync_queue(hdev, terminate_big_sync, d, + terminate_big_destroy); +} + +static int big_terminate_sync(struct hci_dev *hdev, void *data) +{ + struct iso_list_data *d = data; + + bt_dev_dbg(hdev, "big 0x%2.2x sync_handle 0x%4.4x", d->big, + d->sync_handle); + + /* Check if ISO connection is a BIS and terminate BIG if there are + * no other connections using it. + */ + hci_conn_hash_list_state(hdev, find_bis, ISO_LINK, BT_CONNECTED, d); + if (d->count) + return 0; + + hci_le_big_terminate_sync(hdev, d->big); + + return hci_le_pa_terminate_sync(hdev, d->sync_handle); +} + +static int hci_le_big_terminate(struct hci_dev *hdev, u8 big, u16 sync_handle) +{ + struct iso_list_data *d; + + bt_dev_dbg(hdev, "big 0x%2.2x sync_handle 0x%4.4x", big, sync_handle); + + d = kmalloc(sizeof(*d), GFP_KERNEL); + if (!d) + return -ENOMEM; + + memset(d, 0, sizeof(*d)); + d->big = big; + d->sync_handle = sync_handle; + + return hci_cmd_sync_queue(hdev, big_terminate_sync, d, + terminate_big_destroy); +} + +/* Cleanup BIS connection + * + * Detects if there any BIS left connected in a BIG + * broadcaster: Remove advertising instance and terminate BIG. + * broadcaster receiver: Teminate BIG sync and terminate PA sync. + */ +static void bis_cleanup(struct hci_conn *conn) +{ + struct hci_dev *hdev = conn->hdev; + + bt_dev_dbg(hdev, "conn %p", conn); + + if (conn->role == HCI_ROLE_MASTER) { + if (!test_and_clear_bit(HCI_CONN_PER_ADV, &conn->flags)) + return; + + hci_le_terminate_big(hdev, conn->iso_qos.big, + conn->iso_qos.bis); + } else { + hci_le_big_terminate(hdev, conn->iso_qos.big, + conn->sync_handle); + } +} + +static int remove_cig_sync(struct hci_dev *hdev, void *data) +{ + u8 handle = PTR_ERR(data); + + return hci_le_remove_cig_sync(hdev, handle); +} + +static int hci_le_remove_cig(struct hci_dev *hdev, u8 handle) +{ + bt_dev_dbg(hdev, "handle 0x%2.2x", handle); + + return hci_cmd_sync_queue(hdev, remove_cig_sync, ERR_PTR(handle), NULL); +} + +static void find_cis(struct hci_conn *conn, void *data) +{ + struct iso_list_data *d = data; + + /* Ignore broadcast */ + if (!bacmp(&conn->dst, BDADDR_ANY)) + return; + + d->count++; +} + +/* Cleanup CIS connection: + * + * Detects if there any CIS left connected in a CIG and remove it. + */ +static void cis_cleanup(struct hci_conn *conn) +{ + struct hci_dev *hdev = conn->hdev; + struct iso_list_data d; + + memset(&d, 0, sizeof(d)); + d.cig = conn->iso_qos.cig; + + /* Check if ISO connection is a CIS and remove CIG if there are + * no other connections using it. + */ + hci_conn_hash_list_state(hdev, find_cis, ISO_LINK, BT_CONNECTED, &d); + if (d.count) + return; + + hci_le_remove_cig(hdev, conn->iso_qos.cig); +} + struct hci_conn *hci_conn_add(struct hci_dev *hdev, int type, bdaddr_t *dst, u8 role) { @@ -722,6 +924,17 @@ struct hci_conn *hci_conn_add(struct hci_dev *hdev, int type, bdaddr_t *dst, /* conn->src should reflect the local identity address */ hci_copy_identity_address(hdev, &conn->src, &conn->src_type); break; + case ISO_LINK: + /* conn->src should reflect the local identity address */ + hci_copy_identity_address(hdev, &conn->src, &conn->src_type); + + /* set proper cleanup function */ + if (!bacmp(dst, BDADDR_ANY)) + conn->cleanup = bis_cleanup; + else if (conn->role == HCI_ROLE_MASTER) + conn->cleanup = cis_cleanup; + + break; case SCO_LINK: if (lmp_esco_capable(hdev)) conn->pkt_type = (hdev->esco_type & SCO_ESCO_MASK) | @@ -947,7 +1160,7 @@ static void create_le_conn_complete(struct hci_dev *hdev, void *data, int err) if (conn != hci_lookup_le_connect(hdev)) goto done; - hci_conn_failed(conn, err); + hci_conn_failed(conn, bt_status(err)); done: hci_dev_unlock(hdev); @@ -1093,6 +1306,108 @@ static int hci_explicit_conn_params_set(struct hci_dev *hdev, return 0; } +static int qos_set_big(struct hci_dev *hdev, struct bt_iso_qos *qos) +{ + struct iso_list_data data; + + /* Allocate a BIG if not set */ + if (qos->big == BT_ISO_QOS_BIG_UNSET) { + for (data.big = 0x00; data.big < 0xef; data.big++) { + data.count = 0; + data.bis = 0xff; + + hci_conn_hash_list_state(hdev, bis_list, ISO_LINK, + BT_BOUND, &data); + if (!data.count) + break; + } + + if (data.big == 0xef) + return -EADDRNOTAVAIL; + + /* Update BIG */ + qos->big = data.big; + } + + return 0; +} + +static int qos_set_bis(struct hci_dev *hdev, struct bt_iso_qos *qos) +{ + struct iso_list_data data; + + /* Allocate BIS if not set */ + if (qos->bis == BT_ISO_QOS_BIS_UNSET) { + /* Find an unused adv set to advertise BIS, skip instance 0x00 + * since it is reserved as general purpose set. + */ + for (data.bis = 0x01; data.bis < hdev->le_num_of_adv_sets; + data.bis++) { + data.count = 0; + + hci_conn_hash_list_state(hdev, bis_list, ISO_LINK, + BT_BOUND, &data); + if (!data.count) + break; + } + + if (data.bis == hdev->le_num_of_adv_sets) + return -EADDRNOTAVAIL; + + /* Update BIS */ + qos->bis = data.bis; + } + + return 0; +} + +/* This function requires the caller holds hdev->lock */ +static struct hci_conn *hci_add_bis(struct hci_dev *hdev, bdaddr_t *dst, + struct bt_iso_qos *qos) +{ + struct hci_conn *conn; + struct iso_list_data data; + int err; + + /* Let's make sure that le is enabled.*/ + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) { + if (lmp_le_capable(hdev)) + return ERR_PTR(-ECONNREFUSED); + return ERR_PTR(-EOPNOTSUPP); + } + + err = qos_set_big(hdev, qos); + if (err) + return ERR_PTR(err); + + err = qos_set_bis(hdev, qos); + if (err) + return ERR_PTR(err); + + data.big = qos->big; + data.bis = qos->bis; + data.count = 0; + + /* Check if there is already a matching BIG/BIS */ + hci_conn_hash_list_state(hdev, bis_list, ISO_LINK, BT_BOUND, &data); + if (data.count) + return ERR_PTR(-EADDRINUSE); + + conn = hci_conn_hash_lookup_bis(hdev, dst, qos->big, qos->bis); + if (conn) + return ERR_PTR(-EADDRINUSE); + + conn = hci_conn_add(hdev, ISO_LINK, dst, HCI_ROLE_MASTER); + if (!conn) + return ERR_PTR(-ENOMEM); + + set_bit(HCI_CONN_PER_ADV, &conn->flags); + conn->state = BT_CONNECT; + + hci_conn_hold(conn); + return conn; +} + /* This function requires the caller holds hdev->lock */ struct hci_conn *hci_connect_le_scan(struct hci_dev *hdev, bdaddr_t *dst, u8 dst_type, u8 sec_level, @@ -1229,6 +1544,589 @@ struct hci_conn *hci_connect_sco(struct hci_dev *hdev, int type, bdaddr_t *dst, return sco; } +static void cis_add(struct iso_list_data *d, struct bt_iso_qos *qos) +{ + struct hci_cis_params *cis = &d->pdu.cis[d->pdu.cp.num_cis]; + + cis->cis_id = qos->cis; + cis->c_sdu = cpu_to_le16(qos->out.sdu); + cis->p_sdu = cpu_to_le16(qos->in.sdu); + cis->c_phy = qos->out.phy; + cis->p_phy = qos->in.phy; + cis->c_rtn = qos->out.rtn; + cis->p_rtn = qos->in.rtn; + + d->pdu.cp.num_cis++; +} + +static void cis_list(struct hci_conn *conn, void *data) +{ + struct iso_list_data *d = data; + + /* Skip if broadcast/ANY address */ + if (!bacmp(&conn->dst, BDADDR_ANY)) + return; + + if (d->cig != conn->iso_qos.cig || d->cis == BT_ISO_QOS_CIS_UNSET || + d->cis != conn->iso_qos.cis) + return; + + d->count++; + + if (d->pdu.cp.cig_id == BT_ISO_QOS_CIG_UNSET || + d->count >= ARRAY_SIZE(d->pdu.cis)) + return; + + cis_add(d, &conn->iso_qos); +} + +static int hci_le_create_big(struct hci_conn *conn, struct bt_iso_qos *qos) +{ + struct hci_dev *hdev = conn->hdev; + struct hci_cp_le_create_big cp; + + memset(&cp, 0, sizeof(cp)); + + cp.handle = qos->big; + cp.adv_handle = qos->bis; + cp.num_bis = 0x01; + hci_cpu_to_le24(qos->out.interval, cp.bis.sdu_interval); + cp.bis.sdu = cpu_to_le16(qos->out.sdu); + cp.bis.latency = cpu_to_le16(qos->out.latency); + cp.bis.rtn = qos->out.rtn; + cp.bis.phy = qos->out.phy; + cp.bis.packing = qos->packing; + cp.bis.framing = qos->framing; + cp.bis.encryption = 0x00; + memset(&cp.bis.bcode, 0, sizeof(cp.bis.bcode)); + + return hci_send_cmd(hdev, HCI_OP_LE_CREATE_BIG, sizeof(cp), &cp); +} + +static bool hci_le_set_cig_params(struct hci_conn *conn, struct bt_iso_qos *qos) +{ + struct hci_dev *hdev = conn->hdev; + struct iso_list_data data; + + memset(&data, 0, sizeof(data)); + + /* Allocate a CIG if not set */ + if (qos->cig == BT_ISO_QOS_CIG_UNSET) { + for (data.cig = 0x00; data.cig < 0xff; data.cig++) { + data.count = 0; + data.cis = 0xff; + + hci_conn_hash_list_state(hdev, cis_list, ISO_LINK, + BT_BOUND, &data); + if (data.count) + continue; + + hci_conn_hash_list_state(hdev, cis_list, ISO_LINK, + BT_CONNECTED, &data); + if (!data.count) + break; + } + + if (data.cig == 0xff) + return false; + + /* Update CIG */ + qos->cig = data.cig; + } + + data.pdu.cp.cig_id = qos->cig; + hci_cpu_to_le24(qos->out.interval, data.pdu.cp.c_interval); + hci_cpu_to_le24(qos->in.interval, data.pdu.cp.p_interval); + data.pdu.cp.sca = qos->sca; + data.pdu.cp.packing = qos->packing; + data.pdu.cp.framing = qos->framing; + data.pdu.cp.c_latency = cpu_to_le16(qos->out.latency); + data.pdu.cp.p_latency = cpu_to_le16(qos->in.latency); + + if (qos->cis != BT_ISO_QOS_CIS_UNSET) { + data.count = 0; + data.cig = qos->cig; + data.cis = qos->cis; + + hci_conn_hash_list_state(hdev, cis_list, ISO_LINK, BT_BOUND, + &data); + if (data.count) + return false; + + cis_add(&data, qos); + } + + /* Reprogram all CIS(s) with the same CIG */ + for (data.cig = qos->cig, data.cis = 0x00; data.cis < 0x11; + data.cis++) { + data.count = 0; + + hci_conn_hash_list_state(hdev, cis_list, ISO_LINK, BT_BOUND, + &data); + if (data.count) + continue; + + /* Allocate a CIS if not set */ + if (qos->cis == BT_ISO_QOS_CIS_UNSET) { + /* Update CIS */ + qos->cis = data.cis; + cis_add(&data, qos); + } + } + + if (qos->cis == BT_ISO_QOS_CIS_UNSET || !data.pdu.cp.num_cis) + return false; + + if (hci_send_cmd(hdev, HCI_OP_LE_SET_CIG_PARAMS, + sizeof(data.pdu.cp) + + (data.pdu.cp.num_cis * sizeof(*data.pdu.cis)), + &data.pdu) < 0) + return false; + + return true; +} + +struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst, + __u8 dst_type, struct bt_iso_qos *qos) +{ + struct hci_conn *cis; + + cis = hci_conn_hash_lookup_cis(hdev, dst, dst_type); + if (!cis) { + cis = hci_conn_add(hdev, ISO_LINK, dst, HCI_ROLE_MASTER); + if (!cis) + return ERR_PTR(-ENOMEM); + cis->cleanup = cis_cleanup; + } + + if (cis->state == BT_CONNECTED) + return cis; + + /* Check if CIS has been set and the settings matches */ + if (cis->state == BT_BOUND && + !memcmp(&cis->iso_qos, qos, sizeof(*qos))) + return cis; + + /* Update LINK PHYs according to QoS preference */ + cis->le_tx_phy = qos->out.phy; + cis->le_rx_phy = qos->in.phy; + + /* If output interval is not set use the input interval as it cannot be + * 0x000000. + */ + if (!qos->out.interval) + qos->out.interval = qos->in.interval; + + /* If input interval is not set use the output interval as it cannot be + * 0x000000. + */ + if (!qos->in.interval) + qos->in.interval = qos->out.interval; + + /* If output latency is not set use the input latency as it cannot be + * 0x0000. + */ + if (!qos->out.latency) + qos->out.latency = qos->in.latency; + + /* If input latency is not set use the output latency as it cannot be + * 0x0000. + */ + if (!qos->in.latency) + qos->in.latency = qos->out.latency; + + /* Mirror PHYs that are disabled as SDU will be set to 0 */ + if (!qos->in.phy) + qos->in.phy = qos->out.phy; + + if (!qos->out.phy) + qos->out.phy = qos->in.phy; + + if (!hci_le_set_cig_params(cis, qos)) { + hci_conn_drop(cis); + return ERR_PTR(-EINVAL); + } + + cis->iso_qos = *qos; + cis->state = BT_BOUND; + + return cis; +} + +bool hci_iso_setup_path(struct hci_conn *conn) +{ + struct hci_dev *hdev = conn->hdev; + struct hci_cp_le_setup_iso_path cmd; + + memset(&cmd, 0, sizeof(cmd)); + + if (conn->iso_qos.out.sdu) { + cmd.handle = cpu_to_le16(conn->handle); + cmd.direction = 0x00; /* Input (Host to Controller) */ + cmd.path = 0x00; /* HCI path if enabled */ + cmd.codec = 0x03; /* Transparent Data */ + + if (hci_send_cmd(hdev, HCI_OP_LE_SETUP_ISO_PATH, sizeof(cmd), + &cmd) < 0) + return false; + } + + if (conn->iso_qos.in.sdu) { + cmd.handle = cpu_to_le16(conn->handle); + cmd.direction = 0x01; /* Output (Controller to Host) */ + cmd.path = 0x00; /* HCI path if enabled */ + cmd.codec = 0x03; /* Transparent Data */ + + if (hci_send_cmd(hdev, HCI_OP_LE_SETUP_ISO_PATH, sizeof(cmd), + &cmd) < 0) + return false; + } + + return true; +} + +static int hci_create_cis_sync(struct hci_dev *hdev, void *data) +{ + struct { + struct hci_cp_le_create_cis cp; + struct hci_cis cis[0x1f]; + } cmd; + struct hci_conn *conn = data; + u8 cig; + + memset(&cmd, 0, sizeof(cmd)); + cmd.cis[0].acl_handle = cpu_to_le16(conn->link->handle); + cmd.cis[0].cis_handle = cpu_to_le16(conn->handle); + cmd.cp.num_cis++; + cig = conn->iso_qos.cig; + + hci_dev_lock(hdev); + + rcu_read_lock(); + + list_for_each_entry_rcu(conn, &hdev->conn_hash.list, list) { + struct hci_cis *cis = &cmd.cis[cmd.cp.num_cis]; + + if (conn == data || conn->type != ISO_LINK || + conn->state == BT_CONNECTED || conn->iso_qos.cig != cig) + continue; + + /* Check if all CIS(s) belonging to a CIG are ready */ + if (conn->link->state != BT_CONNECTED || + conn->state != BT_CONNECT) { + cmd.cp.num_cis = 0; + break; + } + + /* Group all CIS with state BT_CONNECT since the spec don't + * allow to send them individually: + * + * BLUETOOTH CORE SPECIFICATION Version 5.3 | Vol 4, Part E + * page 2566: + * + * If the Host issues this command before all the + * HCI_LE_CIS_Established events from the previous use of the + * command have been generated, the Controller shall return the + * error code Command Disallowed (0x0C). + */ + cis->acl_handle = cpu_to_le16(conn->link->handle); + cis->cis_handle = cpu_to_le16(conn->handle); + cmd.cp.num_cis++; + } + + rcu_read_unlock(); + + hci_dev_unlock(hdev); + + if (!cmd.cp.num_cis) + return 0; + + return hci_send_cmd(hdev, HCI_OP_LE_CREATE_CIS, sizeof(cmd.cp) + + sizeof(cmd.cis[0]) * cmd.cp.num_cis, &cmd); +} + +int hci_le_create_cis(struct hci_conn *conn) +{ + struct hci_conn *cis; + struct hci_dev *hdev = conn->hdev; + int err; + + switch (conn->type) { + case LE_LINK: + if (!conn->link || conn->state != BT_CONNECTED) + return -EINVAL; + cis = conn->link; + break; + case ISO_LINK: + cis = conn; + break; + default: + return -EINVAL; + } + + if (cis->state == BT_CONNECT) + return 0; + + /* Queue Create CIS */ + err = hci_cmd_sync_queue(hdev, hci_create_cis_sync, cis, NULL); + if (err) + return err; + + cis->state = BT_CONNECT; + + return 0; +} + +static void hci_iso_qos_setup(struct hci_dev *hdev, struct hci_conn *conn, + struct bt_iso_io_qos *qos, __u8 phy) +{ + /* Only set MTU if PHY is enabled */ + if (!qos->sdu && qos->phy) { + if (hdev->iso_mtu > 0) + qos->sdu = hdev->iso_mtu; + else if (hdev->le_mtu > 0) + qos->sdu = hdev->le_mtu; + else + qos->sdu = hdev->acl_mtu; + } + + /* Use the same PHY as ACL if set to any */ + if (qos->phy == BT_ISO_PHY_ANY) + qos->phy = phy; + + /* Use LE ACL connection interval if not set */ + if (!qos->interval) + /* ACL interval unit in 1.25 ms to us */ + qos->interval = conn->le_conn_interval * 1250; + + /* Use LE ACL connection latency if not set */ + if (!qos->latency) + qos->latency = conn->le_conn_latency; +} + +static struct hci_conn *hci_bind_bis(struct hci_conn *conn, + struct bt_iso_qos *qos) +{ + /* Update LINK PHYs according to QoS preference */ + conn->le_tx_phy = qos->out.phy; + conn->le_tx_phy = qos->out.phy; + conn->iso_qos = *qos; + conn->state = BT_BOUND; + + return conn; +} + +static int create_big_sync(struct hci_dev *hdev, void *data) +{ + struct hci_conn *conn = data; + struct bt_iso_qos *qos = &conn->iso_qos; + u16 interval, sync_interval = 0; + u32 flags = 0; + int err; + + if (qos->out.phy == 0x02) + flags |= MGMT_ADV_FLAG_SEC_2M; + + /* Align intervals */ + interval = qos->out.interval / 1250; + + if (qos->bis) + sync_interval = qos->sync_interval * 1600; + + err = hci_start_per_adv_sync(hdev, qos->bis, conn->le_per_adv_data_len, + conn->le_per_adv_data, flags, interval, + interval, sync_interval); + if (err) + return err; + + return hci_le_create_big(conn, &conn->iso_qos); +} + +static void create_pa_complete(struct hci_dev *hdev, void *data, int err) +{ + struct hci_cp_le_pa_create_sync *cp = data; + + bt_dev_dbg(hdev, ""); + + if (err) + bt_dev_err(hdev, "Unable to create PA: %d", err); + + kfree(cp); +} + +static int create_pa_sync(struct hci_dev *hdev, void *data) +{ + struct hci_cp_le_pa_create_sync *cp = data; + int err; + + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_PA_CREATE_SYNC, + sizeof(*cp), cp, HCI_CMD_TIMEOUT); + if (err) { + hci_dev_clear_flag(hdev, HCI_PA_SYNC); + return err; + } + + return hci_update_passive_scan_sync(hdev); +} + +int hci_pa_create_sync(struct hci_dev *hdev, bdaddr_t *dst, __u8 dst_type, + __u8 sid) +{ + struct hci_cp_le_pa_create_sync *cp; + + if (hci_dev_test_and_set_flag(hdev, HCI_PA_SYNC)) + return -EBUSY; + + cp = kmalloc(sizeof(*cp), GFP_KERNEL); + if (!cp) { + hci_dev_clear_flag(hdev, HCI_PA_SYNC); + return -ENOMEM; + } + + /* Convert from ISO socket address type to HCI address type */ + if (dst_type == BDADDR_LE_PUBLIC) + dst_type = ADDR_LE_DEV_PUBLIC; + else + dst_type = ADDR_LE_DEV_RANDOM; + + memset(cp, 0, sizeof(*cp)); + cp->sid = sid; + cp->addr_type = dst_type; + bacpy(&cp->addr, dst); + + /* Queue start pa_create_sync and scan */ + return hci_cmd_sync_queue(hdev, create_pa_sync, cp, create_pa_complete); +} + +int hci_le_big_create_sync(struct hci_dev *hdev, struct bt_iso_qos *qos, + __u16 sync_handle, __u8 num_bis, __u8 bis[]) +{ + struct _packed { + struct hci_cp_le_big_create_sync cp; + __u8 bis[0x11]; + } pdu; + int err; + + if (num_bis > sizeof(pdu.bis)) + return -EINVAL; + + err = qos_set_big(hdev, qos); + if (err) + return err; + + memset(&pdu, 0, sizeof(pdu)); + pdu.cp.handle = qos->big; + pdu.cp.sync_handle = cpu_to_le16(sync_handle); + pdu.cp.num_bis = num_bis; + memcpy(pdu.bis, bis, num_bis); + + return hci_send_cmd(hdev, HCI_OP_LE_BIG_CREATE_SYNC, + sizeof(pdu.cp) + num_bis, &pdu); +} + +static void create_big_complete(struct hci_dev *hdev, void *data, int err) +{ + struct hci_conn *conn = data; + + bt_dev_dbg(hdev, "conn %p", conn); + + if (err) { + bt_dev_err(hdev, "Unable to create BIG: %d", err); + hci_connect_cfm(conn, err); + hci_conn_del(conn); + } +} + +struct hci_conn *hci_connect_bis(struct hci_dev *hdev, bdaddr_t *dst, + __u8 dst_type, struct bt_iso_qos *qos, + __u8 base_len, __u8 *base) +{ + struct hci_conn *conn; + int err; + + /* We need hci_conn object using the BDADDR_ANY as dst */ + conn = hci_add_bis(hdev, dst, qos); + if (IS_ERR(conn)) + return conn; + + conn = hci_bind_bis(conn, qos); + if (!conn) { + hci_conn_drop(conn); + return ERR_PTR(-ENOMEM); + } + + /* Add Basic Announcement into Peridic Adv Data if BASE is set */ + if (base_len && base) { + base_len = eir_append_service_data(conn->le_per_adv_data, 0, + 0x1851, base, base_len); + conn->le_per_adv_data_len = base_len; + } + + /* Queue start periodic advertising and create BIG */ + err = hci_cmd_sync_queue(hdev, create_big_sync, conn, + create_big_complete); + if (err < 0) { + hci_conn_drop(conn); + return ERR_PTR(err); + } + + hci_iso_qos_setup(hdev, conn, &qos->out, + conn->le_tx_phy ? conn->le_tx_phy : + hdev->le_tx_def_phys); + + return conn; +} + +struct hci_conn *hci_connect_cis(struct hci_dev *hdev, bdaddr_t *dst, + __u8 dst_type, struct bt_iso_qos *qos) +{ + struct hci_conn *le; + struct hci_conn *cis; + + /* Convert from ISO socket address type to HCI address type */ + if (dst_type == BDADDR_LE_PUBLIC) + dst_type = ADDR_LE_DEV_PUBLIC; + else + dst_type = ADDR_LE_DEV_RANDOM; + + if (hci_dev_test_flag(hdev, HCI_ADVERTISING)) + le = hci_connect_le(hdev, dst, dst_type, false, + BT_SECURITY_LOW, + HCI_LE_CONN_TIMEOUT, + HCI_ROLE_SLAVE); + else + le = hci_connect_le_scan(hdev, dst, dst_type, + BT_SECURITY_LOW, + HCI_LE_CONN_TIMEOUT, + CONN_REASON_ISO_CONNECT); + if (IS_ERR(le)) + return le; + + hci_iso_qos_setup(hdev, le, &qos->out, + le->le_tx_phy ? le->le_tx_phy : hdev->le_tx_def_phys); + hci_iso_qos_setup(hdev, le, &qos->in, + le->le_rx_phy ? le->le_rx_phy : hdev->le_rx_def_phys); + + cis = hci_bind_cis(hdev, dst, dst_type, qos); + if (IS_ERR(cis)) { + hci_conn_drop(le); + return cis; + } + + le->link = cis; + cis->link = le; + + hci_conn_hold(cis); + + /* If LE is already connected and CIS handle is already set proceed to + * Create CIS immediately. + */ + if (le->state == BT_CONNECTED && cis->handle != HCI_CONN_HANDLE_UNSET) + hci_le_create_cis(le); + + return cis; +} + /* Check link security requirement */ int hci_conn_check_link_mode(struct hci_conn *conn) { diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index a0f99baafd35..b3a5a3cc9372 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -29,6 +29,7 @@ #include <linux/rfkill.h> #include <linux/debugfs.h> #include <linux/crypto.h> +#include <linux/kcov.h> #include <linux/property.h> #include <linux/suspend.h> #include <linux/wait.h> @@ -594,6 +595,11 @@ static int hci_dev_do_reset(struct hci_dev *hdev) skb_queue_purge(&hdev->rx_q); skb_queue_purge(&hdev->cmd_q); + /* Cancel these to avoid queueing non-chained pending work */ + hci_dev_set_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE); + cancel_delayed_work(&hdev->cmd_timer); + cancel_delayed_work(&hdev->ncmd_timer); + /* Avoid potential lockdep warnings from the *_flush() calls by * ensuring the workqueue is empty up front. */ @@ -607,8 +613,13 @@ static int hci_dev_do_reset(struct hci_dev *hdev) if (hdev->flush) hdev->flush(hdev); + hci_dev_clear_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE); + atomic_set(&hdev->cmd_cnt, 1); - hdev->acl_cnt = 0; hdev->sco_cnt = 0; hdev->le_cnt = 0; + hdev->acl_cnt = 0; + hdev->sco_cnt = 0; + hdev->le_cnt = 0; + hdev->iso_cnt = 0; ret = hci_reset_sync(hdev); @@ -1691,63 +1702,77 @@ static void adv_instance_rpa_expired(struct work_struct *work) } /* This function requires the caller holds hdev->lock */ -int hci_add_adv_instance(struct hci_dev *hdev, u8 instance, u32 flags, - u16 adv_data_len, u8 *adv_data, - u16 scan_rsp_len, u8 *scan_rsp_data, - u16 timeout, u16 duration, s8 tx_power, - u32 min_interval, u32 max_interval) +struct adv_info *hci_add_adv_instance(struct hci_dev *hdev, u8 instance, + u32 flags, u16 adv_data_len, u8 *adv_data, + u16 scan_rsp_len, u8 *scan_rsp_data, + u16 timeout, u16 duration, s8 tx_power, + u32 min_interval, u32 max_interval) { - struct adv_info *adv_instance; + struct adv_info *adv; - adv_instance = hci_find_adv_instance(hdev, instance); - if (adv_instance) { - memset(adv_instance->adv_data, 0, - sizeof(adv_instance->adv_data)); - memset(adv_instance->scan_rsp_data, 0, - sizeof(adv_instance->scan_rsp_data)); + adv = hci_find_adv_instance(hdev, instance); + if (adv) { + memset(adv->adv_data, 0, sizeof(adv->adv_data)); + memset(adv->scan_rsp_data, 0, sizeof(adv->scan_rsp_data)); + memset(adv->per_adv_data, 0, sizeof(adv->per_adv_data)); } else { if (hdev->adv_instance_cnt >= hdev->le_num_of_adv_sets || instance < 1 || instance > hdev->le_num_of_adv_sets) - return -EOVERFLOW; + return ERR_PTR(-EOVERFLOW); - adv_instance = kzalloc(sizeof(*adv_instance), GFP_KERNEL); - if (!adv_instance) - return -ENOMEM; + adv = kzalloc(sizeof(*adv), GFP_KERNEL); + if (!adv) + return ERR_PTR(-ENOMEM); - adv_instance->pending = true; - adv_instance->instance = instance; - list_add(&adv_instance->list, &hdev->adv_instances); + adv->pending = true; + adv->instance = instance; + list_add(&adv->list, &hdev->adv_instances); hdev->adv_instance_cnt++; } - adv_instance->flags = flags; - adv_instance->adv_data_len = adv_data_len; - adv_instance->scan_rsp_len = scan_rsp_len; - adv_instance->min_interval = min_interval; - adv_instance->max_interval = max_interval; - adv_instance->tx_power = tx_power; + adv->flags = flags; + adv->min_interval = min_interval; + adv->max_interval = max_interval; + adv->tx_power = tx_power; - if (adv_data_len) - memcpy(adv_instance->adv_data, adv_data, adv_data_len); + hci_set_adv_instance_data(hdev, instance, adv_data_len, adv_data, + scan_rsp_len, scan_rsp_data); - if (scan_rsp_len) - memcpy(adv_instance->scan_rsp_data, - scan_rsp_data, scan_rsp_len); - - adv_instance->timeout = timeout; - adv_instance->remaining_time = timeout; + adv->timeout = timeout; + adv->remaining_time = timeout; if (duration == 0) - adv_instance->duration = hdev->def_multi_adv_rotation_duration; + adv->duration = hdev->def_multi_adv_rotation_duration; else - adv_instance->duration = duration; + adv->duration = duration; - INIT_DELAYED_WORK(&adv_instance->rpa_expired_cb, - adv_instance_rpa_expired); + INIT_DELAYED_WORK(&adv->rpa_expired_cb, adv_instance_rpa_expired); BT_DBG("%s for %dMR", hdev->name, instance); - return 0; + return adv; +} + +/* This function requires the caller holds hdev->lock */ +struct adv_info *hci_add_per_instance(struct hci_dev *hdev, u8 instance, + u32 flags, u8 data_len, u8 *data, + u32 min_interval, u32 max_interval) +{ + struct adv_info *adv; + + adv = hci_add_adv_instance(hdev, instance, flags, 0, NULL, 0, NULL, + 0, 0, HCI_ADV_TX_POWER_NO_PREFERENCE, + min_interval, max_interval); + if (IS_ERR(adv)) + return adv; + + adv->periodic = true; + adv->per_adv_data_len = data_len; + + if (data) + memcpy(adv->per_adv_data, data, data_len); + + return adv; } /* This function requires the caller holds hdev->lock */ @@ -1755,29 +1780,33 @@ int hci_set_adv_instance_data(struct hci_dev *hdev, u8 instance, u16 adv_data_len, u8 *adv_data, u16 scan_rsp_len, u8 *scan_rsp_data) { - struct adv_info *adv_instance; + struct adv_info *adv; - adv_instance = hci_find_adv_instance(hdev, instance); + adv = hci_find_adv_instance(hdev, instance); /* If advertisement doesn't exist, we can't modify its data */ - if (!adv_instance) + if (!adv) return -ENOENT; - if (adv_data_len) { - memset(adv_instance->adv_data, 0, - sizeof(adv_instance->adv_data)); - memcpy(adv_instance->adv_data, adv_data, adv_data_len); - adv_instance->adv_data_len = adv_data_len; + if (adv_data_len && ADV_DATA_CMP(adv, adv_data, adv_data_len)) { + memset(adv->adv_data, 0, sizeof(adv->adv_data)); + memcpy(adv->adv_data, adv_data, adv_data_len); + adv->adv_data_len = adv_data_len; + adv->adv_data_changed = true; } - if (scan_rsp_len) { - memset(adv_instance->scan_rsp_data, 0, - sizeof(adv_instance->scan_rsp_data)); - memcpy(adv_instance->scan_rsp_data, - scan_rsp_data, scan_rsp_len); - adv_instance->scan_rsp_len = scan_rsp_len; + if (scan_rsp_len && SCAN_RSP_CMP(adv, scan_rsp_data, scan_rsp_len)) { + memset(adv->scan_rsp_data, 0, sizeof(adv->scan_rsp_data)); + memcpy(adv->scan_rsp_data, scan_rsp_data, scan_rsp_len); + adv->scan_rsp_len = scan_rsp_len; + adv->scan_rsp_changed = true; } + /* Mark as changed if there are flags which would affect it */ + if (((adv->flags & MGMT_ADV_FLAG_APPEARANCE) && hdev->appearance) || + adv->flags & MGMT_ADV_FLAG_LOCAL_NAME) + adv->scan_rsp_changed = true; + return 0; } @@ -1874,151 +1903,120 @@ void hci_free_adv_monitor(struct hci_dev *hdev, struct adv_monitor *monitor) kfree(monitor); } -int hci_add_adv_patterns_monitor_complete(struct hci_dev *hdev, u8 status) -{ - return mgmt_add_adv_patterns_monitor_complete(hdev, status); -} - -int hci_remove_adv_monitor_complete(struct hci_dev *hdev, u8 status) -{ - return mgmt_remove_adv_monitor_complete(hdev, status); -} - /* Assigns handle to a monitor, and if offloading is supported and power is on, * also attempts to forward the request to the controller. - * Returns true if request is forwarded (result is pending), false otherwise. - * This function requires the caller holds hdev->lock. + * This function requires the caller holds hci_req_sync_lock. */ -bool hci_add_adv_monitor(struct hci_dev *hdev, struct adv_monitor *monitor, - int *err) +int hci_add_adv_monitor(struct hci_dev *hdev, struct adv_monitor *monitor) { int min, max, handle; + int status = 0; - *err = 0; + if (!monitor) + return -EINVAL; - if (!monitor) { - *err = -EINVAL; - return false; - } + hci_dev_lock(hdev); min = HCI_MIN_ADV_MONITOR_HANDLE; max = HCI_MIN_ADV_MONITOR_HANDLE + HCI_MAX_ADV_MONITOR_NUM_HANDLES; handle = idr_alloc(&hdev->adv_monitors_idr, monitor, min, max, GFP_KERNEL); - if (handle < 0) { - *err = handle; - return false; - } + + hci_dev_unlock(hdev); + + if (handle < 0) + return handle; monitor->handle = handle; if (!hdev_is_powered(hdev)) - return false; + return status; switch (hci_get_adv_monitor_offload_ext(hdev)) { case HCI_ADV_MONITOR_EXT_NONE: - hci_update_passive_scan(hdev); - bt_dev_dbg(hdev, "%s add monitor status %d", hdev->name, *err); + bt_dev_dbg(hdev, "%s add monitor %d status %d", hdev->name, + monitor->handle, status); /* Message was not forwarded to controller - not an error */ - return false; + break; + case HCI_ADV_MONITOR_EXT_MSFT: - *err = msft_add_monitor_pattern(hdev, monitor); - bt_dev_dbg(hdev, "%s add monitor msft status %d", hdev->name, - *err); + status = msft_add_monitor_pattern(hdev, monitor); + bt_dev_dbg(hdev, "%s add monitor %d msft status %d", hdev->name, + monitor->handle, status); break; } - return (*err == 0); + return status; } /* Attempts to tell the controller and free the monitor. If somehow the * controller doesn't have a corresponding handle, remove anyway. - * Returns true if request is forwarded (result is pending), false otherwise. - * This function requires the caller holds hdev->lock. + * This function requires the caller holds hci_req_sync_lock. */ -static bool hci_remove_adv_monitor(struct hci_dev *hdev, - struct adv_monitor *monitor, - u16 handle, int *err) +static int hci_remove_adv_monitor(struct hci_dev *hdev, + struct adv_monitor *monitor) { - *err = 0; + int status = 0; switch (hci_get_adv_monitor_offload_ext(hdev)) { case HCI_ADV_MONITOR_EXT_NONE: /* also goes here when powered off */ + bt_dev_dbg(hdev, "%s remove monitor %d status %d", hdev->name, + monitor->handle, status); goto free_monitor; + case HCI_ADV_MONITOR_EXT_MSFT: - *err = msft_remove_monitor(hdev, monitor, handle); + status = msft_remove_monitor(hdev, monitor); + bt_dev_dbg(hdev, "%s remove monitor %d msft status %d", + hdev->name, monitor->handle, status); break; } /* In case no matching handle registered, just free the monitor */ - if (*err == -ENOENT) + if (status == -ENOENT) goto free_monitor; - return (*err == 0); + return status; free_monitor: - if (*err == -ENOENT) + if (status == -ENOENT) bt_dev_warn(hdev, "Removing monitor with no matching handle %d", monitor->handle); hci_free_adv_monitor(hdev, monitor); - *err = 0; - return false; + return status; } -/* Returns true if request is forwarded (result is pending), false otherwise. - * This function requires the caller holds hdev->lock. - */ -bool hci_remove_single_adv_monitor(struct hci_dev *hdev, u16 handle, int *err) +/* This function requires the caller holds hci_req_sync_lock */ +int hci_remove_single_adv_monitor(struct hci_dev *hdev, u16 handle) { struct adv_monitor *monitor = idr_find(&hdev->adv_monitors_idr, handle); - bool pending; - - if (!monitor) { - *err = -EINVAL; - return false; - } - - pending = hci_remove_adv_monitor(hdev, monitor, handle, err); - if (!*err && !pending) - hci_update_passive_scan(hdev); - bt_dev_dbg(hdev, "%s remove monitor handle %d, status %d, %spending", - hdev->name, handle, *err, pending ? "" : "not "); + if (!monitor) + return -EINVAL; - return pending; + return hci_remove_adv_monitor(hdev, monitor); } -/* Returns true if request is forwarded (result is pending), false otherwise. - * This function requires the caller holds hdev->lock. - */ -bool hci_remove_all_adv_monitor(struct hci_dev *hdev, int *err) +/* This function requires the caller holds hci_req_sync_lock */ +int hci_remove_all_adv_monitor(struct hci_dev *hdev) { struct adv_monitor *monitor; int idr_next_id = 0; - bool pending = false; - bool update = false; - - *err = 0; + int status = 0; - while (!*err && !pending) { + while (1) { monitor = idr_get_next(&hdev->adv_monitors_idr, &idr_next_id); if (!monitor) break; - pending = hci_remove_adv_monitor(hdev, monitor, 0, err); + status = hci_remove_adv_monitor(hdev, monitor); + if (status) + return status; - if (!*err && !pending) - update = true; + idr_next_id++; } - if (update) - hci_update_passive_scan(hdev); - - bt_dev_dbg(hdev, "%s remove all monitors status %d, %spending", - hdev->name, *err, pending ? "" : "not "); - - return pending; + return status; } /* This function requires the caller holds hdev->lock */ @@ -2640,12 +2638,9 @@ int hci_register_dev(struct hci_dev *hdev) hci_sock_dev_event(hdev, HCI_DEV_REG); hci_dev_hold(hdev); - if (!test_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks)) { - hdev->suspend_notifier.notifier_call = hci_suspend_notifier; - error = register_pm_notifier(&hdev->suspend_notifier); - if (error) - goto err_wqueue; - } + error = hci_register_suspend_notifier(hdev); + if (error) + goto err_wqueue; queue_work(hdev->req_workqueue, &hdev->power_on); @@ -2680,8 +2675,7 @@ void hci_unregister_dev(struct hci_dev *hdev) hci_cmd_sync_clear(hdev); - if (!test_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks)) - unregister_pm_notifier(&hdev->suspend_notifier); + hci_unregister_suspend_notifier(hdev); msft_unregister(hdev); @@ -2741,10 +2735,33 @@ void hci_release_dev(struct hci_dev *hdev) ida_simple_remove(&hci_index_ida, hdev->id); kfree_skb(hdev->sent_cmd); + kfree_skb(hdev->recv_event); kfree(hdev); } EXPORT_SYMBOL(hci_release_dev); +int hci_register_suspend_notifier(struct hci_dev *hdev) +{ + int ret = 0; + + if (!test_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks)) { + hdev->suspend_notifier.notifier_call = hci_suspend_notifier; + ret = register_pm_notifier(&hdev->suspend_notifier); + } + + return ret; +} + +int hci_unregister_suspend_notifier(struct hci_dev *hdev) +{ + int ret = 0; + + if (!test_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks)) + ret = unregister_pm_notifier(&hdev->suspend_notifier); + + return ret; +} + /* Suspend HCI device */ int hci_suspend_dev(struct hci_dev *hdev) { @@ -3025,6 +3042,37 @@ void *hci_sent_cmd_data(struct hci_dev *hdev, __u16 opcode) return hdev->sent_cmd->data + HCI_COMMAND_HDR_SIZE; } +/* Get data from last received event */ +void *hci_recv_event_data(struct hci_dev *hdev, __u8 event) +{ + struct hci_event_hdr *hdr; + int offset; + + if (!hdev->recv_event) + return NULL; + + hdr = (void *)hdev->recv_event->data; + offset = sizeof(*hdr); + + if (hdr->evt != event) { + /* In case of LE metaevent check the subevent match */ + if (hdr->evt == HCI_EV_LE_META) { + struct hci_ev_le_meta *ev; + + ev = (void *)hdev->recv_event->data + offset; + offset += sizeof(*ev); + if (ev->subevent == event) + goto found; + } + return NULL; + } + +found: + bt_dev_dbg(hdev, "event 0x%2.2x", event); + + return hdev->recv_event->data + offset; +} + /* Send ACL data */ static void hci_add_acl_hdr(struct sk_buff *skb, __u16 handle, __u16 flags) { @@ -3132,9 +3180,117 @@ void hci_send_sco(struct hci_conn *conn, struct sk_buff *skb) queue_work(hdev->workqueue, &hdev->tx_work); } +/* Send ISO data */ +static void hci_add_iso_hdr(struct sk_buff *skb, __u16 handle, __u8 flags) +{ + struct hci_iso_hdr *hdr; + int len = skb->len; + + skb_push(skb, HCI_ISO_HDR_SIZE); + skb_reset_transport_header(skb); + hdr = (struct hci_iso_hdr *)skb_transport_header(skb); + hdr->handle = cpu_to_le16(hci_handle_pack(handle, flags)); + hdr->dlen = cpu_to_le16(len); +} + +static void hci_queue_iso(struct hci_conn *conn, struct sk_buff_head *queue, + struct sk_buff *skb) +{ + struct hci_dev *hdev = conn->hdev; + struct sk_buff *list; + __u16 flags; + + skb->len = skb_headlen(skb); + skb->data_len = 0; + + hci_skb_pkt_type(skb) = HCI_ISODATA_PKT; + + list = skb_shinfo(skb)->frag_list; + + flags = hci_iso_flags_pack(list ? ISO_START : ISO_SINGLE, 0x00); + hci_add_iso_hdr(skb, conn->handle, flags); + + if (!list) { + /* Non fragmented */ + BT_DBG("%s nonfrag skb %p len %d", hdev->name, skb, skb->len); + + skb_queue_tail(queue, skb); + } else { + /* Fragmented */ + BT_DBG("%s frag %p len %d", hdev->name, skb, skb->len); + + skb_shinfo(skb)->frag_list = NULL; + + __skb_queue_tail(queue, skb); + + do { + skb = list; list = list->next; + + hci_skb_pkt_type(skb) = HCI_ISODATA_PKT; + flags = hci_iso_flags_pack(list ? ISO_CONT : ISO_END, + 0x00); + hci_add_iso_hdr(skb, conn->handle, flags); + + BT_DBG("%s frag %p len %d", hdev->name, skb, skb->len); + + __skb_queue_tail(queue, skb); + } while (list); + } +} + +void hci_send_iso(struct hci_conn *conn, struct sk_buff *skb) +{ + struct hci_dev *hdev = conn->hdev; + + BT_DBG("%s len %d", hdev->name, skb->len); + + hci_queue_iso(conn, &conn->data_q, skb); + + queue_work(hdev->workqueue, &hdev->tx_work); +} + /* ---- HCI TX task (outgoing data) ---- */ /* HCI Connection scheduler */ +static inline void hci_quote_sent(struct hci_conn *conn, int num, int *quote) +{ + struct hci_dev *hdev; + int cnt, q; + + if (!conn) { + *quote = 0; + return; + } + + hdev = conn->hdev; + + switch (conn->type) { + case ACL_LINK: + cnt = hdev->acl_cnt; + break; + case AMP_LINK: + cnt = hdev->block_cnt; + break; + case SCO_LINK: + case ESCO_LINK: + cnt = hdev->sco_cnt; + break; + case LE_LINK: + cnt = hdev->le_mtu ? hdev->le_cnt : hdev->acl_cnt; + break; + case ISO_LINK: + cnt = hdev->iso_mtu ? hdev->iso_cnt : + hdev->le_mtu ? hdev->le_cnt : hdev->acl_cnt; + break; + default: + cnt = 0; + bt_dev_err(hdev, "unknown link type %d", conn->type); + } + + q = cnt / num; + *quote = q ? q : 1; +} + static struct hci_conn *hci_low_sent(struct hci_dev *hdev, __u8 type, int *quote) { @@ -3167,29 +3323,7 @@ static struct hci_conn *hci_low_sent(struct hci_dev *hdev, __u8 type, rcu_read_unlock(); - if (conn) { - int cnt, q; - - switch (conn->type) { - case ACL_LINK: - cnt = hdev->acl_cnt; - break; - case SCO_LINK: - case ESCO_LINK: - cnt = hdev->sco_cnt; - break; - case LE_LINK: - cnt = hdev->le_mtu ? hdev->le_cnt : hdev->acl_cnt; - break; - default: - cnt = 0; - bt_dev_err(hdev, "unknown link type %d", conn->type); - } - - q = cnt / num; - *quote = q ? q : 1; - } else - *quote = 0; + hci_quote_sent(conn, num, quote); BT_DBG("conn %p quote %d", conn, *quote); return conn; @@ -3223,7 +3357,7 @@ static struct hci_chan *hci_chan_sent(struct hci_dev *hdev, __u8 type, struct hci_chan *chan = NULL; unsigned int num = 0, min = ~0, cur_prio = 0; struct hci_conn *conn; - int cnt, q, conn_num = 0; + int conn_num = 0; BT_DBG("%s", hdev->name); @@ -3273,27 +3407,8 @@ static struct hci_chan *hci_chan_sent(struct hci_dev *hdev, __u8 type, if (!chan) return NULL; - switch (chan->conn->type) { - case ACL_LINK: - cnt = hdev->acl_cnt; - break; - case AMP_LINK: - cnt = hdev->block_cnt; - break; - case SCO_LINK: - case ESCO_LINK: - cnt = hdev->sco_cnt; - break; - case LE_LINK: - cnt = hdev->le_mtu ? hdev->le_cnt : hdev->acl_cnt; - break; - default: - cnt = 0; - bt_dev_err(hdev, "unknown link type %d", chan->conn->type); - } + hci_quote_sent(chan->conn, num, quote); - q = cnt / num; - *quote = q ? q : 1; BT_DBG("chan %p quote %d", chan, *quote); return chan; } @@ -3582,18 +3697,46 @@ static void hci_sched_le(struct hci_dev *hdev) hci_prio_recalculate(hdev, LE_LINK); } +/* Schedule CIS */ +static void hci_sched_iso(struct hci_dev *hdev) +{ + struct hci_conn *conn; + struct sk_buff *skb; + int quote, *cnt; + + BT_DBG("%s", hdev->name); + + if (!hci_conn_num(hdev, ISO_LINK)) + return; + + cnt = hdev->iso_pkts ? &hdev->iso_cnt : + hdev->le_pkts ? &hdev->le_cnt : &hdev->acl_cnt; + while (*cnt && (conn = hci_low_sent(hdev, ISO_LINK, "e))) { + while (quote-- && (skb = skb_dequeue(&conn->data_q))) { + BT_DBG("skb %p len %d", skb, skb->len); + hci_send_frame(hdev, skb); + + conn->sent++; + if (conn->sent == ~0) + conn->sent = 0; + (*cnt)--; + } + } +} + static void hci_tx_work(struct work_struct *work) { struct hci_dev *hdev = container_of(work, struct hci_dev, tx_work); struct sk_buff *skb; - BT_DBG("%s acl %d sco %d le %d", hdev->name, hdev->acl_cnt, - hdev->sco_cnt, hdev->le_cnt); + BT_DBG("%s acl %d sco %d le %d iso %d", hdev->name, hdev->acl_cnt, + hdev->sco_cnt, hdev->le_cnt, hdev->iso_cnt); if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL)) { /* Schedule queues and send stuff to HCI driver */ hci_sched_sco(hdev); hci_sched_esco(hdev); + hci_sched_iso(hdev); hci_sched_acl(hdev); hci_sched_le(hdev); } @@ -3676,6 +3819,43 @@ static void hci_scodata_packet(struct hci_dev *hdev, struct sk_buff *skb) kfree_skb(skb); } +static void hci_isodata_packet(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct hci_iso_hdr *hdr; + struct hci_conn *conn; + __u16 handle, flags; + + hdr = skb_pull_data(skb, sizeof(*hdr)); + if (!hdr) { + bt_dev_err(hdev, "ISO packet too small"); + goto drop; + } + + handle = __le16_to_cpu(hdr->handle); + flags = hci_flags(handle); + handle = hci_handle(handle); + + bt_dev_dbg(hdev, "len %d handle 0x%4.4x flags 0x%4.4x", skb->len, + handle, flags); + + hci_dev_lock(hdev); + conn = hci_conn_hash_lookup_handle(hdev, handle); + hci_dev_unlock(hdev); + + if (!conn) { + bt_dev_err(hdev, "ISO packet for unknown connection handle %d", + handle); + goto drop; + } + + /* Send to upper protocol */ + iso_recv(conn, skb, flags); + return; + +drop: + kfree_skb(skb); +} + static bool hci_req_is_complete(struct hci_dev *hdev) { struct sk_buff *skb; @@ -3781,7 +3961,14 @@ static void hci_rx_work(struct work_struct *work) BT_DBG("%s", hdev->name); - while ((skb = skb_dequeue(&hdev->rx_q))) { + /* The kcov_remote functions used for collecting packet parsing + * coverage information from this background thread and associate + * the coverage with the syscall's thread which originally injected + * the packet. This helps fuzzing the kernel. + */ + for (; (skb = skb_dequeue(&hdev->rx_q)); kcov_remote_stop()) { + kcov_remote_start_common(skb_get_kcov_handle(skb)); + /* Send copy to monitor */ hci_send_to_monitor(hdev, skb); @@ -3830,6 +4017,11 @@ static void hci_rx_work(struct work_struct *work) hci_scodata_packet(hdev, skb); break; + case HCI_ISODATA_PKT: + BT_DBG("%s ISO data packet", hdev->name); + hci_isodata_packet(hdev, skb); + break; + default: kfree_skb(skb); break; @@ -3864,7 +4056,8 @@ static void hci_cmd_work(struct work_struct *work) if (res < 0) __hci_cmd_sync_cancel(hdev, -res); - if (test_bit(HCI_RESET, &hdev->flags)) + if (test_bit(HCI_RESET, &hdev->flags) || + hci_dev_test_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE)) cancel_delayed_work(&hdev->cmd_timer); else schedule_delayed_work(&hdev->cmd_timer, diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index af17dfb20e01..ea33dd0cd478 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -2741,7 +2741,7 @@ static void hci_cs_disconnect(struct hci_dev *hdev, u8 status) mgmt_conn = test_and_clear_bit(HCI_CONN_MGMT_CONNECTED, &conn->flags); if (conn->type == ACL_LINK) { - if (test_bit(HCI_CONN_FLUSH_KEY, &conn->flags)) + if (test_and_clear_bit(HCI_CONN_FLUSH_KEY, &conn->flags)) hci_remove_link_key(hdev, &conn->dst); } @@ -3173,7 +3173,7 @@ static void hci_conn_complete_evt(struct hci_dev *hdev, void *data, hci_send_cmd(hdev, HCI_OP_READ_REMOTE_FEATURES, sizeof(cp), &cp); - hci_req_update_scan(hdev); + hci_update_scan(hdev); } /* Set packet type for incoming connection */ @@ -3368,10 +3368,10 @@ static void hci_disconn_complete_evt(struct hci_dev *hdev, void *data, reason, mgmt_connected); if (conn->type == ACL_LINK) { - if (test_bit(HCI_CONN_FLUSH_KEY, &conn->flags)) + if (test_and_clear_bit(HCI_CONN_FLUSH_KEY, &conn->flags)) hci_remove_link_key(hdev, &conn->dst); - hci_req_update_scan(hdev); + hci_update_scan(hdev); } params = hci_conn_params_lookup(hdev, &conn->dst, conn->dst_type); @@ -3768,12 +3768,182 @@ static inline void handle_cmd_cnt_and_timer(struct hci_dev *hdev, u8 ncmd) cancel_delayed_work(&hdev->ncmd_timer); atomic_set(&hdev->cmd_cnt, 1); } else { - schedule_delayed_work(&hdev->ncmd_timer, - HCI_NCMD_TIMEOUT); + if (!hci_dev_test_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE)) + schedule_delayed_work(&hdev->ncmd_timer, + HCI_NCMD_TIMEOUT); } } } +static u8 hci_cc_le_read_buffer_size_v2(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_rp_le_read_buffer_size_v2 *rp = data; + + bt_dev_dbg(hdev, "status 0x%2.2x", rp->status); + + if (rp->status) + return rp->status; + + hdev->le_mtu = __le16_to_cpu(rp->acl_mtu); + hdev->le_pkts = rp->acl_max_pkt; + hdev->iso_mtu = __le16_to_cpu(rp->iso_mtu); + hdev->iso_pkts = rp->iso_max_pkt; + + hdev->le_cnt = hdev->le_pkts; + hdev->iso_cnt = hdev->iso_pkts; + + BT_DBG("%s acl mtu %d:%d iso mtu %d:%d", hdev->name, hdev->acl_mtu, + hdev->acl_pkts, hdev->iso_mtu, hdev->iso_pkts); + + return rp->status; +} + +static u8 hci_cc_le_set_cig_params(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_rp_le_set_cig_params *rp = data; + struct hci_conn *conn; + int i = 0; + + bt_dev_dbg(hdev, "status 0x%2.2x", rp->status); + + hci_dev_lock(hdev); + + if (rp->status) { + while ((conn = hci_conn_hash_lookup_cig(hdev, rp->cig_id))) { + conn->state = BT_CLOSED; + hci_connect_cfm(conn, rp->status); + hci_conn_del(conn); + } + goto unlock; + } + + rcu_read_lock(); + + list_for_each_entry_rcu(conn, &hdev->conn_hash.list, list) { + if (conn->type != ISO_LINK || conn->iso_qos.cig != rp->cig_id || + conn->state == BT_CONNECTED) + continue; + + conn->handle = __le16_to_cpu(rp->handle[i++]); + + bt_dev_dbg(hdev, "%p handle 0x%4.4x link %p", conn, + conn->handle, conn->link); + + /* Create CIS if LE is already connected */ + if (conn->link && conn->link->state == BT_CONNECTED) + hci_le_create_cis(conn->link); + + if (i == rp->num_handles) + break; + } + + rcu_read_unlock(); + +unlock: + hci_dev_unlock(hdev); + + return rp->status; +} + +static u8 hci_cc_le_setup_iso_path(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_rp_le_setup_iso_path *rp = data; + struct hci_cp_le_setup_iso_path *cp; + struct hci_conn *conn; + + bt_dev_dbg(hdev, "status 0x%2.2x", rp->status); + + cp = hci_sent_cmd_data(hdev, HCI_OP_LE_SETUP_ISO_PATH); + if (!cp) + return rp->status; + + hci_dev_lock(hdev); + + conn = hci_conn_hash_lookup_handle(hdev, __le16_to_cpu(cp->handle)); + if (!conn) + goto unlock; + + if (rp->status) { + hci_connect_cfm(conn, rp->status); + hci_conn_del(conn); + goto unlock; + } + + switch (cp->direction) { + /* Input (Host to Controller) */ + case 0x00: + /* Only confirm connection if output only */ + if (conn->iso_qos.out.sdu && !conn->iso_qos.in.sdu) + hci_connect_cfm(conn, rp->status); + break; + /* Output (Controller to Host) */ + case 0x01: + /* Confirm connection since conn->iso_qos is always configured + * last. + */ + hci_connect_cfm(conn, rp->status); + break; + } + +unlock: + hci_dev_unlock(hdev); + return rp->status; +} + +static void hci_cs_le_create_big(struct hci_dev *hdev, u8 status) +{ + bt_dev_dbg(hdev, "status 0x%2.2x", status); +} + +static u8 hci_cc_set_per_adv_param(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_ev_status *rp = data; + struct hci_cp_le_set_per_adv_params *cp; + + bt_dev_dbg(hdev, "status 0x%2.2x", rp->status); + + if (rp->status) + return rp->status; + + cp = hci_sent_cmd_data(hdev, HCI_OP_LE_SET_PER_ADV_PARAMS); + if (!cp) + return rp->status; + + /* TODO: set the conn state */ + return rp->status; +} + +static u8 hci_cc_le_set_per_adv_enable(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_ev_status *rp = data; + __u8 *sent; + + bt_dev_dbg(hdev, "status 0x%2.2x", rp->status); + + if (rp->status) + return rp->status; + + sent = hci_sent_cmd_data(hdev, HCI_OP_LE_SET_PER_ADV_ENABLE); + if (!sent) + return rp->status; + + hci_dev_lock(hdev); + + if (*sent) + hci_dev_set_flag(hdev, HCI_LE_PER_ADV); + else + hci_dev_clear_flag(hdev, HCI_LE_PER_ADV); + + hci_dev_unlock(hdev); + + return rp->status; +} + #define HCI_CC_VL(_op, _func, _min, _max) \ { \ .op = _op, \ @@ -3947,9 +4117,18 @@ static const struct hci_cc { hci_cc_le_set_adv_set_random_addr), HCI_CC_STATUS(HCI_OP_LE_REMOVE_ADV_SET, hci_cc_le_remove_adv_set), HCI_CC_STATUS(HCI_OP_LE_CLEAR_ADV_SETS, hci_cc_le_clear_adv_sets), + HCI_CC_STATUS(HCI_OP_LE_SET_PER_ADV_PARAMS, hci_cc_set_per_adv_param), + HCI_CC_STATUS(HCI_OP_LE_SET_PER_ADV_ENABLE, + hci_cc_le_set_per_adv_enable), HCI_CC(HCI_OP_LE_READ_TRANSMIT_POWER, hci_cc_le_read_transmit_power, sizeof(struct hci_rp_le_read_transmit_power)), - HCI_CC_STATUS(HCI_OP_LE_SET_PRIVACY_MODE, hci_cc_le_set_privacy_mode) + HCI_CC_STATUS(HCI_OP_LE_SET_PRIVACY_MODE, hci_cc_le_set_privacy_mode), + HCI_CC(HCI_OP_LE_READ_BUFFER_SIZE_V2, hci_cc_le_read_buffer_size_v2, + sizeof(struct hci_rp_le_read_buffer_size_v2)), + HCI_CC_VL(HCI_OP_LE_SET_CIG_PARAMS, hci_cc_le_set_cig_params, + sizeof(struct hci_rp_le_set_cig_params), HCI_MAX_EVENT_SIZE), + HCI_CC(HCI_OP_LE_SETUP_ISO_PATH, hci_cc_le_setup_iso_path, + sizeof(struct hci_rp_le_setup_iso_path)), }; static u8 hci_cc_func(struct hci_dev *hdev, const struct hci_cc *cc, @@ -4012,6 +4191,40 @@ static void hci_cmd_complete_evt(struct hci_dev *hdev, void *data, queue_work(hdev->workqueue, &hdev->cmd_work); } +static void hci_cs_le_create_cis(struct hci_dev *hdev, u8 status) +{ + struct hci_cp_le_create_cis *cp; + int i; + + bt_dev_dbg(hdev, "status 0x%2.2x", status); + + if (!status) + return; + + cp = hci_sent_cmd_data(hdev, HCI_OP_LE_CREATE_CIS); + if (!cp) + return; + + hci_dev_lock(hdev); + + /* Remove connection if command failed */ + for (i = 0; cp->num_cis; cp->num_cis--, i++) { + struct hci_conn *conn; + u16 handle; + + handle = __le16_to_cpu(cp->cis[i].cis_handle); + + conn = hci_conn_hash_lookup_handle(hdev, handle); + if (conn) { + conn->state = BT_CLOSED; + hci_connect_cfm(conn, status); + hci_conn_del(conn); + } + } + + hci_dev_unlock(hdev); +} + #define HCI_CS(_op, _func) \ { \ .op = _op, \ @@ -4041,7 +4254,9 @@ static const struct hci_cs { HCI_CS(HCI_OP_LE_CREATE_CONN, hci_cs_le_create_conn), HCI_CS(HCI_OP_LE_READ_REMOTE_FEATURES, hci_cs_le_read_remote_features), HCI_CS(HCI_OP_LE_START_ENC, hci_cs_le_start_enc), - HCI_CS(HCI_OP_LE_EXT_CREATE_CONN, hci_cs_le_ext_create_conn) + HCI_CS(HCI_OP_LE_EXT_CREATE_CONN, hci_cs_le_ext_create_conn), + HCI_CS(HCI_OP_LE_CREATE_CIS, hci_cs_le_create_cis), + HCI_CS(HCI_OP_LE_CREATE_BIG, hci_cs_le_create_big), }; static void hci_cmd_status_evt(struct hci_dev *hdev, void *data, @@ -4177,6 +4392,22 @@ static void hci_num_comp_pkts_evt(struct hci_dev *hdev, void *data, hdev->sco_cnt = hdev->sco_pkts; break; + case ISO_LINK: + if (hdev->iso_pkts) { + hdev->iso_cnt += count; + if (hdev->iso_cnt > hdev->iso_pkts) + hdev->iso_cnt = hdev->iso_pkts; + } else if (hdev->le_pkts) { + hdev->le_cnt += count; + if (hdev->le_cnt > hdev->le_pkts) + hdev->le_cnt = hdev->le_pkts; + } else { + hdev->acl_cnt += count; + if (hdev->acl_cnt > hdev->acl_pkts) + hdev->acl_cnt = hdev->acl_pkts; + } + break; + default: bt_dev_err(hdev, "unknown type %d conn %p", conn->type, conn); @@ -6249,6 +6480,39 @@ static void hci_le_ext_adv_report_evt(struct hci_dev *hdev, void *data, hci_dev_unlock(hdev); } +static int hci_le_pa_term_sync(struct hci_dev *hdev, __le16 handle) +{ + struct hci_cp_le_pa_term_sync cp; + + memset(&cp, 0, sizeof(cp)); + cp.handle = handle; + + return hci_send_cmd(hdev, HCI_OP_LE_PA_TERM_SYNC, sizeof(cp), &cp); +} + +static void hci_le_pa_sync_estabilished_evt(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_ev_le_pa_sync_established *ev = data; + int mask = hdev->link_mode; + __u8 flags = 0; + + bt_dev_dbg(hdev, "status 0x%2.2x", ev->status); + + if (ev->status) + return; + + hci_dev_lock(hdev); + + hci_dev_clear_flag(hdev, HCI_PA_SYNC); + + mask |= hci_proto_connect_ind(hdev, &ev->bdaddr, ISO_LINK, &flags); + if (!(mask & HCI_LM_ACCEPT)) + hci_le_pa_term_sync(hdev, ev->handle); + + hci_dev_unlock(hdev); +} + static void hci_le_remote_feat_complete_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb) { @@ -6479,6 +6743,226 @@ unlock: hci_dev_unlock(hdev); } +static void hci_le_cis_estabilished_evt(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_evt_le_cis_established *ev = data; + struct hci_conn *conn; + u16 handle = __le16_to_cpu(ev->handle); + + bt_dev_dbg(hdev, "status 0x%2.2x", ev->status); + + hci_dev_lock(hdev); + + conn = hci_conn_hash_lookup_handle(hdev, handle); + if (!conn) { + bt_dev_err(hdev, + "Unable to find connection with handle 0x%4.4x", + handle); + goto unlock; + } + + if (conn->role == HCI_ROLE_SLAVE) { + __le32 interval; + + memset(&interval, 0, sizeof(interval)); + + memcpy(&interval, ev->c_latency, sizeof(ev->c_latency)); + conn->iso_qos.in.interval = le32_to_cpu(interval); + memcpy(&interval, ev->p_latency, sizeof(ev->p_latency)); + conn->iso_qos.out.interval = le32_to_cpu(interval); + conn->iso_qos.in.latency = le16_to_cpu(ev->interval); + conn->iso_qos.out.latency = le16_to_cpu(ev->interval); + conn->iso_qos.in.sdu = le16_to_cpu(ev->c_mtu); + conn->iso_qos.out.sdu = le16_to_cpu(ev->p_mtu); + conn->iso_qos.in.phy = ev->c_phy; + conn->iso_qos.out.phy = ev->p_phy; + } + + if (!ev->status) { + conn->state = BT_CONNECTED; + hci_debugfs_create_conn(conn); + hci_conn_add_sysfs(conn); + hci_iso_setup_path(conn); + goto unlock; + } + + hci_connect_cfm(conn, ev->status); + hci_conn_del(conn); + +unlock: + hci_dev_unlock(hdev); +} + +static void hci_le_reject_cis(struct hci_dev *hdev, __le16 handle) +{ + struct hci_cp_le_reject_cis cp; + + memset(&cp, 0, sizeof(cp)); + cp.handle = handle; + cp.reason = HCI_ERROR_REJ_BAD_ADDR; + hci_send_cmd(hdev, HCI_OP_LE_REJECT_CIS, sizeof(cp), &cp); +} + +static void hci_le_accept_cis(struct hci_dev *hdev, __le16 handle) +{ + struct hci_cp_le_accept_cis cp; + + memset(&cp, 0, sizeof(cp)); + cp.handle = handle; + hci_send_cmd(hdev, HCI_OP_LE_ACCEPT_CIS, sizeof(cp), &cp); +} + +static void hci_le_cis_req_evt(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_evt_le_cis_req *ev = data; + u16 acl_handle, cis_handle; + struct hci_conn *acl, *cis; + int mask; + __u8 flags = 0; + + acl_handle = __le16_to_cpu(ev->acl_handle); + cis_handle = __le16_to_cpu(ev->cis_handle); + + bt_dev_dbg(hdev, "acl 0x%4.4x handle 0x%4.4x cig 0x%2.2x cis 0x%2.2x", + acl_handle, cis_handle, ev->cig_id, ev->cis_id); + + hci_dev_lock(hdev); + + acl = hci_conn_hash_lookup_handle(hdev, acl_handle); + if (!acl) + goto unlock; + + mask = hci_proto_connect_ind(hdev, &acl->dst, ISO_LINK, &flags); + if (!(mask & HCI_LM_ACCEPT)) { + hci_le_reject_cis(hdev, ev->cis_handle); + goto unlock; + } + + cis = hci_conn_hash_lookup_handle(hdev, cis_handle); + if (!cis) { + cis = hci_conn_add(hdev, ISO_LINK, &acl->dst, HCI_ROLE_SLAVE); + if (!cis) { + hci_le_reject_cis(hdev, ev->cis_handle); + goto unlock; + } + cis->handle = cis_handle; + } + + cis->iso_qos.cig = ev->cig_id; + cis->iso_qos.cis = ev->cis_id; + + if (!(flags & HCI_PROTO_DEFER)) { + hci_le_accept_cis(hdev, ev->cis_handle); + } else { + cis->state = BT_CONNECT2; + hci_connect_cfm(cis, 0); + } + +unlock: + hci_dev_unlock(hdev); +} + +static void hci_le_create_big_complete_evt(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_evt_le_create_big_complete *ev = data; + struct hci_conn *conn; + + BT_DBG("%s status 0x%2.2x", hdev->name, ev->status); + + if (!hci_le_ev_skb_pull(hdev, skb, HCI_EVT_LE_CREATE_BIG_COMPLETE, + flex_array_size(ev, bis_handle, ev->num_bis))) + return; + + hci_dev_lock(hdev); + + conn = hci_conn_hash_lookup_big(hdev, ev->handle); + if (!conn) + goto unlock; + + if (ev->num_bis) + conn->handle = __le16_to_cpu(ev->bis_handle[0]); + + if (!ev->status) { + conn->state = BT_CONNECTED; + hci_debugfs_create_conn(conn); + hci_conn_add_sysfs(conn); + hci_iso_setup_path(conn); + goto unlock; + } + + hci_connect_cfm(conn, ev->status); + hci_conn_del(conn); + +unlock: + hci_dev_unlock(hdev); +} + +static void hci_le_big_sync_established_evt(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_evt_le_big_sync_estabilished *ev = data; + struct hci_conn *bis; + int i; + + bt_dev_dbg(hdev, "status 0x%2.2x", ev->status); + + if (!hci_le_ev_skb_pull(hdev, skb, HCI_EVT_LE_BIG_SYNC_ESTABILISHED, + flex_array_size(ev, bis, ev->num_bis))) + return; + + if (ev->status) + return; + + hci_dev_lock(hdev); + + for (i = 0; i < ev->num_bis; i++) { + u16 handle = le16_to_cpu(ev->bis[i]); + __le32 interval; + + bis = hci_conn_hash_lookup_handle(hdev, handle); + if (!bis) { + bis = hci_conn_add(hdev, ISO_LINK, BDADDR_ANY, + HCI_ROLE_SLAVE); + if (!bis) + continue; + bis->handle = handle; + } + + bis->iso_qos.big = ev->handle; + memset(&interval, 0, sizeof(interval)); + memcpy(&interval, ev->latency, sizeof(ev->latency)); + bis->iso_qos.in.interval = le32_to_cpu(interval); + /* Convert ISO Interval (1.25 ms slots) to latency (ms) */ + bis->iso_qos.in.latency = le16_to_cpu(ev->interval) * 125 / 100; + bis->iso_qos.in.sdu = le16_to_cpu(ev->max_pdu); + + hci_connect_cfm(bis, ev->status); + } + + hci_dev_unlock(hdev); +} + +static void hci_le_big_info_adv_report_evt(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_evt_le_big_info_adv_report *ev = data; + int mask = hdev->link_mode; + __u8 flags = 0; + + bt_dev_dbg(hdev, "sync_handle 0x%4.4x", le16_to_cpu(ev->sync_handle)); + + hci_dev_lock(hdev); + + mask |= hci_proto_connect_ind(hdev, BDADDR_ANY, ISO_LINK, &flags); + if (!(mask & HCI_LM_ACCEPT)) + hci_le_pa_term_sync(hdev, ev->sync_handle); + + hci_dev_unlock(hdev); +} + #define HCI_LE_EV_VL(_op, _func, _min_len, _max_len) \ [_op] = { \ .func = _func, \ @@ -6539,9 +7023,34 @@ static const struct hci_le_ev { HCI_LE_EV_VL(HCI_EV_LE_EXT_ADV_REPORT, hci_le_ext_adv_report_evt, sizeof(struct hci_ev_le_ext_adv_report), HCI_MAX_EVENT_SIZE), + /* [0x0e = HCI_EV_LE_PA_SYNC_ESTABLISHED] */ + HCI_LE_EV(HCI_EV_LE_PA_SYNC_ESTABLISHED, + hci_le_pa_sync_estabilished_evt, + sizeof(struct hci_ev_le_pa_sync_established)), /* [0x12 = HCI_EV_LE_EXT_ADV_SET_TERM] */ HCI_LE_EV(HCI_EV_LE_EXT_ADV_SET_TERM, hci_le_ext_adv_term_evt, sizeof(struct hci_evt_le_ext_adv_set_term)), + /* [0x19 = HCI_EVT_LE_CIS_ESTABLISHED] */ + HCI_LE_EV(HCI_EVT_LE_CIS_ESTABLISHED, hci_le_cis_estabilished_evt, + sizeof(struct hci_evt_le_cis_established)), + /* [0x1a = HCI_EVT_LE_CIS_REQ] */ + HCI_LE_EV(HCI_EVT_LE_CIS_REQ, hci_le_cis_req_evt, + sizeof(struct hci_evt_le_cis_req)), + /* [0x1b = HCI_EVT_LE_CREATE_BIG_COMPLETE] */ + HCI_LE_EV_VL(HCI_EVT_LE_CREATE_BIG_COMPLETE, + hci_le_create_big_complete_evt, + sizeof(struct hci_evt_le_create_big_complete), + HCI_MAX_EVENT_SIZE), + /* [0x1d = HCI_EV_LE_BIG_SYNC_ESTABILISHED] */ + HCI_LE_EV_VL(HCI_EVT_LE_BIG_SYNC_ESTABILISHED, + hci_le_big_sync_established_evt, + sizeof(struct hci_evt_le_big_sync_estabilished), + HCI_MAX_EVENT_SIZE), + /* [0x22 = HCI_EVT_LE_BIG_INFO_ADV_REPORT] */ + HCI_LE_EV_VL(HCI_EVT_LE_BIG_INFO_ADV_REPORT, + hci_le_big_info_adv_report_evt, + sizeof(struct hci_evt_le_big_info_adv_report), + HCI_MAX_EVENT_SIZE), }; static void hci_le_meta_evt(struct hci_dev *hdev, void *data, @@ -6580,7 +7089,6 @@ static void hci_le_meta_evt(struct hci_dev *hdev, void *data, if (skb->len > subev->max_len) bt_dev_warn(hdev, "unexpected subevent 0x%2.2x length: %u > %u", ev->subevent, skb->len, subev->max_len); - data = hci_le_ev_skb_pull(hdev, skb, ev->subevent, subev->min_len); if (!data) return; @@ -6935,6 +7443,9 @@ void hci_event_packet(struct hci_dev *hdev, struct sk_buff *skb) goto done; } + kfree_skb(hdev->recv_event); + hdev->recv_event = skb_clone(skb, GFP_KERNEL); + event = hdr->evt; if (!event) { bt_dev_warn(hdev, "Received unexpected HCI Event 0x%2.2x", diff --git a/net/bluetooth/hci_request.c b/net/bluetooth/hci_request.c index 38ecaf9264ee..e64d558e5d69 100644 --- a/net/bluetooth/hci_request.c +++ b/net/bluetooth/hci_request.c @@ -827,7 +827,6 @@ void __hci_req_disable_advertising(struct hci_request *req) { if (ext_adv_capable(req->hdev)) { __hci_req_disable_ext_adv_instance(req, 0x00); - } else { u8 enable = 0x00; @@ -1338,15 +1337,15 @@ int __hci_req_setup_ext_adv_instance(struct hci_request *req, u8 instance) bdaddr_t random_addr; u8 own_addr_type; int err; - struct adv_info *adv_instance; - bool secondary_adv; + struct adv_info *adv; + bool secondary_adv, require_privacy; if (instance > 0) { - adv_instance = hci_find_adv_instance(hdev, instance); - if (!adv_instance) + adv = hci_find_adv_instance(hdev, instance); + if (!adv) return -EINVAL; } else { - adv_instance = NULL; + adv = NULL; } flags = hci_adv_instance_flags(hdev, instance); @@ -1364,18 +1363,24 @@ int __hci_req_setup_ext_adv_instance(struct hci_request *req, u8 instance) * advertising is used. In that case it is fine to use a * non-resolvable private address. */ - err = hci_get_random_address(hdev, !connectable, - adv_use_rpa(hdev, flags), adv_instance, + require_privacy = !connectable; + + /* Don't require privacy for periodic adv? */ + if (adv && adv->periodic) + require_privacy = false; + + err = hci_get_random_address(hdev, require_privacy, + adv_use_rpa(hdev, flags), adv, &own_addr_type, &random_addr); if (err < 0) return err; memset(&cp, 0, sizeof(cp)); - if (adv_instance) { - hci_cpu_to_le24(adv_instance->min_interval, cp.min_interval); - hci_cpu_to_le24(adv_instance->max_interval, cp.max_interval); - cp.tx_power = adv_instance->tx_power; + if (adv) { + hci_cpu_to_le24(adv->min_interval, cp.min_interval); + hci_cpu_to_le24(adv->max_interval, cp.max_interval); + cp.tx_power = adv->tx_power; } else { hci_cpu_to_le24(hdev->le_adv_min_interval, cp.min_interval); hci_cpu_to_le24(hdev->le_adv_max_interval, cp.max_interval); @@ -1396,7 +1401,8 @@ int __hci_req_setup_ext_adv_instance(struct hci_request *req, u8 instance) else cp.evt_properties = cpu_to_le16(LE_LEGACY_ADV_SCAN_IND); } else { - if (secondary_adv) + /* Secondary and periodic cannot use legacy PDUs */ + if (secondary_adv || (adv && adv->periodic)) cp.evt_properties = cpu_to_le16(LE_EXT_ADV_NON_CONN_IND); else cp.evt_properties = cpu_to_le16(LE_LEGACY_NONCONN_IND); @@ -1426,8 +1432,8 @@ int __hci_req_setup_ext_adv_instance(struct hci_request *req, u8 instance) struct hci_cp_le_set_adv_set_rand_addr cp; /* Check if random address need to be updated */ - if (adv_instance) { - if (!bacmp(&random_addr, &adv_instance->random_addr)) + if (adv) { + if (!bacmp(&random_addr, &adv->random_addr)) return 0; } else { if (!bacmp(&random_addr, &hdev->random_addr)) @@ -1835,21 +1841,6 @@ void __hci_req_update_scan(struct hci_request *req) hci_req_add(req, HCI_OP_WRITE_SCAN_ENABLE, 1, &scan); } -static int update_scan(struct hci_request *req, unsigned long opt) -{ - hci_dev_lock(req->hdev); - __hci_req_update_scan(req); - hci_dev_unlock(req->hdev); - return 0; -} - -static void scan_update_work(struct work_struct *work) -{ - struct hci_dev *hdev = container_of(work, struct hci_dev, scan_update); - - hci_req_sync(hdev, update_scan, 0, HCI_CMD_TIMEOUT, NULL); -} - static u8 get_service_classes(struct hci_dev *hdev) { struct bt_uuid *uuid; @@ -1890,69 +1881,6 @@ void __hci_req_update_class(struct hci_request *req) hci_req_add(req, HCI_OP_WRITE_CLASS_OF_DEV, sizeof(cod), cod); } -static void write_iac(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - struct hci_cp_write_current_iac_lap cp; - - if (!hci_dev_test_flag(hdev, HCI_DISCOVERABLE)) - return; - - if (hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) { - /* Limited discoverable mode */ - cp.num_iac = min_t(u8, hdev->num_iac, 2); - cp.iac_lap[0] = 0x00; /* LIAC */ - cp.iac_lap[1] = 0x8b; - cp.iac_lap[2] = 0x9e; - cp.iac_lap[3] = 0x33; /* GIAC */ - cp.iac_lap[4] = 0x8b; - cp.iac_lap[5] = 0x9e; - } else { - /* General discoverable mode */ - cp.num_iac = 1; - cp.iac_lap[0] = 0x33; /* GIAC */ - cp.iac_lap[1] = 0x8b; - cp.iac_lap[2] = 0x9e; - } - - hci_req_add(req, HCI_OP_WRITE_CURRENT_IAC_LAP, - (cp.num_iac * 3) + 1, &cp); -} - -static int discoverable_update(struct hci_request *req, unsigned long opt) -{ - struct hci_dev *hdev = req->hdev; - - hci_dev_lock(hdev); - - if (hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) { - write_iac(req); - __hci_req_update_scan(req); - __hci_req_update_class(req); - } - - /* Advertising instances don't use the global discoverable setting, so - * only update AD if advertising was enabled using Set Advertising. - */ - if (hci_dev_test_flag(hdev, HCI_ADVERTISING)) { - __hci_req_update_adv_data(req, 0x00); - - /* Discoverable mode affects the local advertising - * address in limited privacy mode. - */ - if (hci_dev_test_flag(hdev, HCI_LIMITED_PRIVACY)) { - if (ext_adv_capable(hdev)) - __hci_req_start_ext_adv(req, 0x00); - else - __hci_req_enable_advertising(req); - } - } - - hci_dev_unlock(hdev); - - return 0; -} - void __hci_abort_conn(struct hci_request *req, struct hci_conn *conn, u8 reason) { @@ -2227,146 +2155,6 @@ unlock: hci_dev_unlock(hdev); } -static int active_scan(struct hci_request *req, unsigned long opt) -{ - uint16_t interval = opt; - struct hci_dev *hdev = req->hdev; - u8 own_addr_type; - /* Accept list is not used for discovery */ - u8 filter_policy = 0x00; - /* Default is to enable duplicates filter */ - u8 filter_dup = LE_SCAN_FILTER_DUP_ENABLE; - /* Discovery doesn't require controller address resolution */ - bool addr_resolv = false; - int err; - - bt_dev_dbg(hdev, ""); - - /* If controller is scanning, it means the background scanning is - * running. Thus, we should temporarily stop it in order to set the - * discovery scanning parameters. - */ - if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) { - hci_req_add_le_scan_disable(req, false); - cancel_interleave_scan(hdev); - } - - /* All active scans will be done with either a resolvable private - * address (when privacy feature has been enabled) or non-resolvable - * private address. - */ - err = hci_update_random_address(req, true, scan_use_rpa(hdev), - &own_addr_type); - if (err < 0) - own_addr_type = ADDR_LE_DEV_PUBLIC; - - hci_dev_lock(hdev); - if (hci_is_adv_monitoring(hdev)) { - /* Duplicate filter should be disabled when some advertisement - * monitor is activated, otherwise AdvMon can only receive one - * advertisement for one peer(*) during active scanning, and - * might report loss to these peers. - * - * Note that different controllers have different meanings of - * |duplicate|. Some of them consider packets with the same - * address as duplicate, and others consider packets with the - * same address and the same RSSI as duplicate. Although in the - * latter case we don't need to disable duplicate filter, but - * it is common to have active scanning for a short period of - * time, the power impact should be neglectable. - */ - filter_dup = LE_SCAN_FILTER_DUP_DISABLE; - } - hci_dev_unlock(hdev); - - hci_req_start_scan(req, LE_SCAN_ACTIVE, interval, - hdev->le_scan_window_discovery, own_addr_type, - filter_policy, filter_dup, addr_resolv); - return 0; -} - -static int interleaved_discov(struct hci_request *req, unsigned long opt) -{ - int err; - - bt_dev_dbg(req->hdev, ""); - - err = active_scan(req, opt); - if (err) - return err; - - return bredr_inquiry(req, DISCOV_BREDR_INQUIRY_LEN); -} - -static void start_discovery(struct hci_dev *hdev, u8 *status) -{ - unsigned long timeout; - - bt_dev_dbg(hdev, "type %u", hdev->discovery.type); - - switch (hdev->discovery.type) { - case DISCOV_TYPE_BREDR: - if (!hci_dev_test_flag(hdev, HCI_INQUIRY)) - hci_req_sync(hdev, bredr_inquiry, - DISCOV_BREDR_INQUIRY_LEN, HCI_CMD_TIMEOUT, - status); - return; - case DISCOV_TYPE_INTERLEAVED: - /* When running simultaneous discovery, the LE scanning time - * should occupy the whole discovery time sine BR/EDR inquiry - * and LE scanning are scheduled by the controller. - * - * For interleaving discovery in comparison, BR/EDR inquiry - * and LE scanning are done sequentially with separate - * timeouts. - */ - if (test_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, - &hdev->quirks)) { - timeout = msecs_to_jiffies(DISCOV_LE_TIMEOUT); - /* During simultaneous discovery, we double LE scan - * interval. We must leave some time for the controller - * to do BR/EDR inquiry. - */ - hci_req_sync(hdev, interleaved_discov, - hdev->le_scan_int_discovery * 2, HCI_CMD_TIMEOUT, - status); - break; - } - - timeout = msecs_to_jiffies(hdev->discov_interleaved_timeout); - hci_req_sync(hdev, active_scan, hdev->le_scan_int_discovery, - HCI_CMD_TIMEOUT, status); - break; - case DISCOV_TYPE_LE: - timeout = msecs_to_jiffies(DISCOV_LE_TIMEOUT); - hci_req_sync(hdev, active_scan, hdev->le_scan_int_discovery, - HCI_CMD_TIMEOUT, status); - break; - default: - *status = HCI_ERROR_UNSPECIFIED; - return; - } - - if (*status) - return; - - bt_dev_dbg(hdev, "timeout %u ms", jiffies_to_msecs(timeout)); - - /* When service discovery is used and the controller has a - * strict duplicate filter, it is important to remember the - * start and duration of the scan. This is required for - * restarting scanning during the discovery phase. - */ - if (test_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks) && - hdev->discovery.result_filtering) { - hdev->discovery.scan_start = jiffies; - hdev->discovery.scan_duration = timeout; - } - - queue_delayed_work(hdev->req_workqueue, &hdev->le_scan_disable, - timeout); -} - bool hci_req_stop_discovery(struct hci_request *req) { struct hci_dev *hdev = req->hdev; @@ -2462,180 +2250,8 @@ error: return err; } -static int stop_discovery(struct hci_request *req, unsigned long opt) -{ - hci_dev_lock(req->hdev); - hci_req_stop_discovery(req); - hci_dev_unlock(req->hdev); - - return 0; -} - -static void discov_update(struct work_struct *work) -{ - struct hci_dev *hdev = container_of(work, struct hci_dev, - discov_update); - u8 status = 0; - - switch (hdev->discovery.state) { - case DISCOVERY_STARTING: - start_discovery(hdev, &status); - mgmt_start_discovery_complete(hdev, status); - if (status) - hci_discovery_set_state(hdev, DISCOVERY_STOPPED); - else - hci_discovery_set_state(hdev, DISCOVERY_FINDING); - break; - case DISCOVERY_STOPPING: - hci_req_sync(hdev, stop_discovery, 0, HCI_CMD_TIMEOUT, &status); - mgmt_stop_discovery_complete(hdev, status); - if (!status) - hci_discovery_set_state(hdev, DISCOVERY_STOPPED); - break; - case DISCOVERY_STOPPED: - default: - return; - } -} - -static void discov_off(struct work_struct *work) -{ - struct hci_dev *hdev = container_of(work, struct hci_dev, - discov_off.work); - - bt_dev_dbg(hdev, ""); - - hci_dev_lock(hdev); - - /* When discoverable timeout triggers, then just make sure - * the limited discoverable flag is cleared. Even in the case - * of a timeout triggered from general discoverable, it is - * safe to unconditionally clear the flag. - */ - hci_dev_clear_flag(hdev, HCI_LIMITED_DISCOVERABLE); - hci_dev_clear_flag(hdev, HCI_DISCOVERABLE); - hdev->discov_timeout = 0; - - hci_dev_unlock(hdev); - - hci_req_sync(hdev, discoverable_update, 0, HCI_CMD_TIMEOUT, NULL); - mgmt_new_settings(hdev); -} - -static int powered_update_hci(struct hci_request *req, unsigned long opt) -{ - struct hci_dev *hdev = req->hdev; - u8 link_sec; - - hci_dev_lock(hdev); - - if (hci_dev_test_flag(hdev, HCI_SSP_ENABLED) && - !lmp_host_ssp_capable(hdev)) { - u8 mode = 0x01; - - hci_req_add(req, HCI_OP_WRITE_SSP_MODE, sizeof(mode), &mode); - - if (bredr_sc_enabled(hdev) && !lmp_host_sc_capable(hdev)) { - u8 support = 0x01; - - hci_req_add(req, HCI_OP_WRITE_SC_SUPPORT, - sizeof(support), &support); - } - } - - if (hci_dev_test_flag(hdev, HCI_LE_ENABLED) && - lmp_bredr_capable(hdev)) { - struct hci_cp_write_le_host_supported cp; - - cp.le = 0x01; - cp.simul = 0x00; - - /* Check first if we already have the right - * host state (host features set) - */ - if (cp.le != lmp_host_le_capable(hdev) || - cp.simul != lmp_host_le_br_capable(hdev)) - hci_req_add(req, HCI_OP_WRITE_LE_HOST_SUPPORTED, - sizeof(cp), &cp); - } - - if (hci_dev_test_flag(hdev, HCI_LE_ENABLED)) { - /* Make sure the controller has a good default for - * advertising data. This also applies to the case - * where BR/EDR was toggled during the AUTO_OFF phase. - */ - if (hci_dev_test_flag(hdev, HCI_ADVERTISING) || - list_empty(&hdev->adv_instances)) { - int err; - - if (ext_adv_capable(hdev)) { - err = __hci_req_setup_ext_adv_instance(req, - 0x00); - if (!err) - __hci_req_update_scan_rsp_data(req, - 0x00); - } else { - err = 0; - __hci_req_update_adv_data(req, 0x00); - __hci_req_update_scan_rsp_data(req, 0x00); - } - - if (hci_dev_test_flag(hdev, HCI_ADVERTISING)) { - if (!ext_adv_capable(hdev)) - __hci_req_enable_advertising(req); - else if (!err) - __hci_req_enable_ext_advertising(req, - 0x00); - } - } else if (!list_empty(&hdev->adv_instances)) { - struct adv_info *adv_instance; - - adv_instance = list_first_entry(&hdev->adv_instances, - struct adv_info, list); - __hci_req_schedule_adv_instance(req, - adv_instance->instance, - true); - } - } - - link_sec = hci_dev_test_flag(hdev, HCI_LINK_SECURITY); - if (link_sec != test_bit(HCI_AUTH, &hdev->flags)) - hci_req_add(req, HCI_OP_WRITE_AUTH_ENABLE, - sizeof(link_sec), &link_sec); - - if (lmp_bredr_capable(hdev)) { - if (hci_dev_test_flag(hdev, HCI_FAST_CONNECTABLE)) - __hci_req_write_fast_connectable(req, true); - else - __hci_req_write_fast_connectable(req, false); - __hci_req_update_scan(req); - __hci_req_update_class(req); - __hci_req_update_name(req); - __hci_req_update_eir(req); - } - - hci_dev_unlock(hdev); - return 0; -} - -int __hci_req_hci_power_on(struct hci_dev *hdev) -{ - /* Register the available SMP channels (BR/EDR and LE) only when - * successfully powering on the controller. This late - * registration is required so that LE SMP can clearly decide if - * the public address or static address is used. - */ - smp_register(hdev); - - return __hci_req_sync(hdev, powered_update_hci, 0, HCI_CMD_TIMEOUT, - NULL); -} - void hci_request_setup(struct hci_dev *hdev) { - INIT_WORK(&hdev->discov_update, discov_update); - INIT_WORK(&hdev->scan_update, scan_update_work); - INIT_DELAYED_WORK(&hdev->discov_off, discov_off); INIT_DELAYED_WORK(&hdev->le_scan_disable, le_scan_disable_work); INIT_DELAYED_WORK(&hdev->le_scan_restart, le_scan_restart_work); INIT_DELAYED_WORK(&hdev->adv_instance_expire, adv_timeout_expire); @@ -2646,9 +2262,6 @@ void hci_request_cancel_all(struct hci_dev *hdev) { __hci_cmd_sync_cancel(hdev, ENODEV); - cancel_work_sync(&hdev->discov_update); - cancel_work_sync(&hdev->scan_update); - cancel_delayed_work_sync(&hdev->discov_off); cancel_delayed_work_sync(&hdev->le_scan_disable); cancel_delayed_work_sync(&hdev->le_scan_restart); diff --git a/net/bluetooth/hci_request.h b/net/bluetooth/hci_request.h index 7f8df258e295..39d001fa3acf 100644 --- a/net/bluetooth/hci_request.h +++ b/net/bluetooth/hci_request.h @@ -68,8 +68,6 @@ int __hci_req_sync(struct hci_dev *hdev, int (*func)(struct hci_request *req, struct sk_buff *hci_prepare_cmd(struct hci_dev *hdev, u16 opcode, u32 plen, const void *param); -int __hci_req_hci_power_on(struct hci_dev *hdev); - void __hci_req_write_fast_connectable(struct hci_request *req, bool enable); void __hci_req_update_name(struct hci_request *req); void __hci_req_update_eir(struct hci_request *req); @@ -85,6 +83,9 @@ void __hci_req_enable_advertising(struct hci_request *req); void __hci_req_disable_advertising(struct hci_request *req); void __hci_req_update_adv_data(struct hci_request *req, u8 instance); int hci_req_update_adv_data(struct hci_dev *hdev, u8 instance); +int hci_req_start_per_adv(struct hci_dev *hdev, u8 instance, u32 flags, + u16 min_interval, u16 max_interval, + u16 sync_interval); void __hci_req_update_scan_rsp_data(struct hci_request *req, u8 instance); int __hci_req_schedule_adv_instance(struct hci_request *req, u8 instance, @@ -94,8 +95,14 @@ void hci_req_clear_adv_instance(struct hci_dev *hdev, struct sock *sk, bool force); int __hci_req_setup_ext_adv_instance(struct hci_request *req, u8 instance); +int __hci_req_setup_per_adv_instance(struct hci_request *req, u8 instance, + u16 min_interval, u16 max_interval); int __hci_req_start_ext_adv(struct hci_request *req, u8 instance); +int __hci_req_start_per_adv(struct hci_request *req, u8 instance, u32 flags, + u16 min_interval, u16 max_interval, + u16 sync_interval); int __hci_req_enable_ext_advertising(struct hci_request *req, u8 instance); +int __hci_req_enable_per_advertising(struct hci_request *req, u8 instance); int __hci_req_disable_ext_adv_instance(struct hci_request *req, u8 instance); int __hci_req_remove_ext_adv_instance(struct hci_request *req, u8 instance); void __hci_req_clear_ext_adv_sets(struct hci_request *req); @@ -110,11 +117,6 @@ bool hci_req_stop_discovery(struct hci_request *req); int hci_req_configure_datapath(struct hci_dev *hdev, struct bt_codec *codec); -static inline void hci_req_update_scan(struct hci_dev *hdev) -{ - queue_work(hdev->req_workqueue, &hdev->scan_update); -} - void __hci_req_update_scan(struct hci_request *req); int hci_update_random_address(struct hci_request *req, bool require_privacy, diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index 189e3115c8c6..0d015d4a8e41 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -869,7 +869,8 @@ static int hci_sock_release(struct socket *sock) hdev = hci_pi(sk)->hdev; if (hdev) { - if (hci_pi(sk)->channel == HCI_CHANNEL_USER) { + if (hci_pi(sk)->channel == HCI_CHANNEL_USER && + !hci_dev_test_flag(hdev, HCI_UNREGISTER)) { /* When releasing a user channel exclusive access, * call hci_dev_do_close directly instead of calling * hci_dev_close to ensure the exclusive access will @@ -878,9 +879,15 @@ static int hci_sock_release(struct socket *sock) * The checking of HCI_AUTO_OFF is not needed in this * case since it will have been cleared already when * opening the user channel. + * + * Make sure to also check that we haven't already + * unregistered since all the cleanup will have already + * been complete and hdev will get released when we put + * below. */ hci_dev_do_close(hdev); hci_dev_clear_flag(hdev, HCI_USER_CHANNEL); + hci_register_suspend_notifier(hdev); mgmt_index_added(hdev); } @@ -1209,6 +1216,7 @@ static int hci_sock_bind(struct socket *sock, struct sockaddr *addr, } mgmt_index_removed(hdev); + hci_unregister_suspend_notifier(hdev); err = hci_dev_open(hdev->id); if (err) { @@ -1223,6 +1231,7 @@ static int hci_sock_bind(struct socket *sock, struct sockaddr *addr, err = 0; } else { hci_dev_clear_flag(hdev, HCI_USER_CHANNEL); + hci_register_suspend_notifier(hdev); mgmt_index_added(hdev); hci_dev_put(hdev); goto done; diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 1739e8cb3291..148ce629a59f 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -849,26 +849,38 @@ static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) u8 data[HCI_MAX_EXT_AD_LENGTH]; } pdu; u8 len; + struct adv_info *adv = NULL; + int err; memset(&pdu, 0, sizeof(pdu)); - len = eir_create_scan_rsp(hdev, instance, pdu.data); - - if (hdev->scan_rsp_data_len == len && - !memcmp(pdu.data, hdev->scan_rsp_data, len)) - return 0; + if (instance) { + adv = hci_find_adv_instance(hdev, instance); + if (!adv || !adv->scan_rsp_changed) + return 0; + } - memcpy(hdev->scan_rsp_data, pdu.data, len); - hdev->scan_rsp_data_len = len; + len = eir_create_scan_rsp(hdev, instance, pdu.data); pdu.cp.handle = instance; pdu.cp.length = len; pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE; pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG; - return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_RSP_DATA, - sizeof(pdu.cp) + len, &pdu.cp, - HCI_CMD_TIMEOUT); + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_RSP_DATA, + sizeof(pdu.cp) + len, &pdu.cp, + HCI_CMD_TIMEOUT); + if (err) + return err; + + if (adv) { + adv->scan_rsp_changed = false; + } else { + memcpy(hdev->scan_rsp_data, pdu.data, len); + hdev->scan_rsp_data_len = len; + } + + return 0; } static int __hci_set_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) @@ -965,6 +977,187 @@ int hci_start_ext_adv_sync(struct hci_dev *hdev, u8 instance) return hci_enable_ext_advertising_sync(hdev, instance); } +static int hci_disable_per_advertising_sync(struct hci_dev *hdev, u8 instance) +{ + struct hci_cp_le_set_per_adv_enable cp; + + /* If periodic advertising already disabled there is nothing to do. */ + if (!hci_dev_test_flag(hdev, HCI_LE_PER_ADV)) + return 0; + + memset(&cp, 0, sizeof(cp)); + + cp.enable = 0x00; + cp.handle = instance; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PER_ADV_ENABLE, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_set_per_adv_params_sync(struct hci_dev *hdev, u8 instance, + u16 min_interval, u16 max_interval) +{ + struct hci_cp_le_set_per_adv_params cp; + + memset(&cp, 0, sizeof(cp)); + + if (!min_interval) + min_interval = DISCOV_LE_PER_ADV_INT_MIN; + + if (!max_interval) + max_interval = DISCOV_LE_PER_ADV_INT_MAX; + + cp.handle = instance; + cp.min_interval = cpu_to_le16(min_interval); + cp.max_interval = cpu_to_le16(max_interval); + cp.periodic_properties = 0x0000; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PER_ADV_PARAMS, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_set_per_adv_data_sync(struct hci_dev *hdev, u8 instance) +{ + struct { + struct hci_cp_le_set_per_adv_data cp; + u8 data[HCI_MAX_PER_AD_LENGTH]; + } pdu; + u8 len; + + memset(&pdu, 0, sizeof(pdu)); + + if (instance) { + struct adv_info *adv = hci_find_adv_instance(hdev, instance); + + if (!adv || !adv->periodic) + return 0; + } + + len = eir_create_per_adv_data(hdev, instance, pdu.data); + + pdu.cp.length = len; + pdu.cp.handle = instance; + pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PER_ADV_DATA, + sizeof(pdu.cp) + len, &pdu, + HCI_CMD_TIMEOUT); +} + +static int hci_enable_per_advertising_sync(struct hci_dev *hdev, u8 instance) +{ + struct hci_cp_le_set_per_adv_enable cp; + + /* If periodic advertising already enabled there is nothing to do. */ + if (hci_dev_test_flag(hdev, HCI_LE_PER_ADV)) + return 0; + + memset(&cp, 0, sizeof(cp)); + + cp.enable = 0x01; + cp.handle = instance; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PER_ADV_ENABLE, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +/* Checks if periodic advertising data contains a Basic Announcement and if it + * does generates a Broadcast ID and add Broadcast Announcement. + */ +static int hci_adv_bcast_annoucement(struct hci_dev *hdev, struct adv_info *adv) +{ + u8 bid[3]; + u8 ad[4 + 3]; + + /* Skip if NULL adv as instance 0x00 is used for general purpose + * advertising so it cannot used for the likes of Broadcast Announcement + * as it can be overwritten at any point. + */ + if (!adv) + return 0; + + /* Check if PA data doesn't contains a Basic Audio Announcement then + * there is nothing to do. + */ + if (!eir_get_service_data(adv->per_adv_data, adv->per_adv_data_len, + 0x1851, NULL)) + return 0; + + /* Check if advertising data already has a Broadcast Announcement since + * the process may want to control the Broadcast ID directly and in that + * case the kernel shall no interfere. + */ + if (eir_get_service_data(adv->adv_data, adv->adv_data_len, 0x1852, + NULL)) + return 0; + + /* Generate Broadcast ID */ + get_random_bytes(bid, sizeof(bid)); + eir_append_service_data(ad, 0, 0x1852, bid, sizeof(bid)); + hci_set_adv_instance_data(hdev, adv->instance, sizeof(ad), ad, 0, NULL); + + return hci_update_adv_data_sync(hdev, adv->instance); +} + +int hci_start_per_adv_sync(struct hci_dev *hdev, u8 instance, u8 data_len, + u8 *data, u32 flags, u16 min_interval, + u16 max_interval, u16 sync_interval) +{ + struct adv_info *adv = NULL; + int err; + bool added = false; + + hci_disable_per_advertising_sync(hdev, instance); + + if (instance) { + adv = hci_find_adv_instance(hdev, instance); + /* Create an instance if that could not be found */ + if (!adv) { + adv = hci_add_per_instance(hdev, instance, flags, + data_len, data, + sync_interval, + sync_interval); + if (IS_ERR(adv)) + return PTR_ERR(adv); + added = true; + } + } + + /* Only start advertising if instance 0 or if a dedicated instance has + * been added. + */ + if (!adv || added) { + err = hci_start_ext_adv_sync(hdev, instance); + if (err < 0) + goto fail; + + err = hci_adv_bcast_annoucement(hdev, adv); + if (err < 0) + goto fail; + } + + err = hci_set_per_adv_params_sync(hdev, instance, min_interval, + max_interval); + if (err < 0) + goto fail; + + err = hci_set_per_adv_data_sync(hdev, instance); + if (err < 0) + goto fail; + + err = hci_enable_per_advertising_sync(hdev, instance); + if (err < 0) + goto fail; + + return 0; + +fail: + if (added) + hci_remove_adv_instance(hdev, instance); + + return err; +} + static int hci_start_adv_sync(struct hci_dev *hdev, u8 instance) { int err; @@ -1104,6 +1297,42 @@ int hci_remove_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance, HCI_CMD_TIMEOUT, sk); } +static int remove_ext_adv_sync(struct hci_dev *hdev, void *data) +{ + struct adv_info *adv = data; + u8 instance = 0; + + if (adv) + instance = adv->instance; + + return hci_remove_ext_adv_instance_sync(hdev, instance, NULL); +} + +int hci_remove_ext_adv_instance(struct hci_dev *hdev, u8 instance) +{ + struct adv_info *adv = NULL; + + if (instance) { + adv = hci_find_adv_instance(hdev, instance); + if (!adv) + return -EINVAL; + } + + return hci_cmd_sync_queue(hdev, remove_ext_adv_sync, adv, NULL); +} + +int hci_le_terminate_big_sync(struct hci_dev *hdev, u8 handle, u8 reason) +{ + struct hci_cp_le_term_big cp; + + memset(&cp, 0, sizeof(cp)); + cp.handle = handle; + cp.reason = reason; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_TERM_BIG, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + static void cancel_adv_timeout(struct hci_dev *hdev) { if (hdev->adv_instance_timeout) { @@ -1119,27 +1348,39 @@ static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) u8 data[HCI_MAX_EXT_AD_LENGTH]; } pdu; u8 len; + struct adv_info *adv = NULL; + int err; memset(&pdu, 0, sizeof(pdu)); - len = eir_create_adv_data(hdev, instance, pdu.data); - - /* There's nothing to do if the data hasn't changed */ - if (hdev->adv_data_len == len && - memcmp(pdu.data, hdev->adv_data, len) == 0) - return 0; + if (instance) { + adv = hci_find_adv_instance(hdev, instance); + if (!adv || !adv->adv_data_changed) + return 0; + } - memcpy(hdev->adv_data, pdu.data, len); - hdev->adv_data_len = len; + len = eir_create_adv_data(hdev, instance, pdu.data); pdu.cp.length = len; pdu.cp.handle = instance; pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE; pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG; - return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_DATA, - sizeof(pdu.cp) + len, &pdu.cp, - HCI_CMD_TIMEOUT); + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_DATA, + sizeof(pdu.cp) + len, &pdu.cp, + HCI_CMD_TIMEOUT); + if (err) + return err; + + /* Update data if the command succeed */ + if (adv) { + adv->adv_data_changed = false; + } else { + memcpy(hdev->adv_data, pdu.data, len); + hdev->adv_data_len = len; + } + + return 0; } static int hci_set_adv_data_sync(struct hci_dev *hdev, u8 instance) @@ -1612,6 +1853,9 @@ static int hci_le_add_resolve_list_sync(struct hci_dev *hdev, bacpy(&cp.bdaddr, ¶ms->addr); memcpy(cp.peer_irk, irk->val, 16); + /* Default privacy mode is always Network */ + params->privacy_mode = HCI_NETWORK_PRIVACY; + done: if (hci_dev_test_flag(hdev, HCI_PRIVACY)) memcpy(cp.local_irk, hdev->irk, 16); @@ -1865,12 +2109,15 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) } /* Go through the current accept list programmed into the - * controller one by one and check if that address is still - * in the list of pending connections or list of devices to + * controller one by one and check if that address is connected or is + * still in the list of pending connections or list of devices to * report. If not present in either list, then remove it from * the controller. */ list_for_each_entry_safe(b, t, &hdev->le_accept_list, list) { + if (hci_conn_hash_lookup_le(hdev, &b->bdaddr, b->bdaddr_type)) + continue; + pend_conn = hci_pend_le_action_lookup(&hdev->pend_le_conns, &b->bdaddr, b->bdaddr_type); @@ -2171,7 +2418,8 @@ int hci_update_passive_scan_sync(struct hci_dev *hdev) if (list_empty(&hdev->pend_le_conns) && list_empty(&hdev->pend_le_reports) && - !hci_is_adv_monitoring(hdev)) { + !hci_is_adv_monitoring(hdev) && + !hci_dev_test_flag(hdev, HCI_PA_SYNC)) { /* If there is no pending LE connections or devices * to be scanned for or no ADV monitors, we should stop the * background scanning. @@ -2206,6 +2454,16 @@ int hci_update_passive_scan_sync(struct hci_dev *hdev) return err; } +static int update_scan_sync(struct hci_dev *hdev, void *data) +{ + return hci_update_scan_sync(hdev); +} + +int hci_update_scan(struct hci_dev *hdev) +{ + return hci_cmd_sync_queue(hdev, update_scan_sync, NULL, NULL); +} + static int update_passive_scan_sync(struct hci_dev *hdev, void *data) { return hci_update_passive_scan_sync(hdev); @@ -2760,6 +3018,12 @@ static const struct hci_init_stage amp_init2[] = { /* Read Buffer Size (ACL mtu, max pkt, etc.) */ static int hci_read_buffer_size_sync(struct hci_dev *hdev) { + /* Use Read LE Buffer Size V2 if supported */ + if (hdev->commands[41] & 0x20) + return __hci_cmd_sync_status(hdev, + HCI_OP_LE_READ_BUFFER_SIZE_V2, + 0, NULL, HCI_CMD_TIMEOUT); + return __hci_cmd_sync_status(hdev, HCI_OP_READ_BUFFER_SIZE, 0, NULL, HCI_CMD_TIMEOUT); } @@ -3011,6 +3275,10 @@ static int hci_init2_sync(struct hci_dev *hdev) if (hdev->dev_type == HCI_AMP) return hci_init_stage_sync(hdev, amp_init2); + err = hci_init_stage_sync(hdev, hci_init2); + if (err) + return err; + if (lmp_bredr_capable(hdev)) { err = hci_init_stage_sync(hdev, br_init2); if (err) @@ -3028,7 +3296,7 @@ static int hci_init2_sync(struct hci_dev *hdev) hci_dev_set_flag(hdev, HCI_LE_ENABLED); } - return hci_init_stage_sync(hdev, hci_init2); + return 0; } static int hci_set_event_mask_sync(struct hci_dev *hdev) @@ -3191,7 +3459,7 @@ static int hci_read_page_scan_activity_sync(struct hci_dev *hdev) static int hci_read_def_err_data_reporting_sync(struct hci_dev *hdev) { if (!(hdev->commands[18] & 0x04) || - test_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks)) + !(hdev->features[0][6] & LMP_ERR_DATA_REPORTING)) return 0; return __hci_cmd_sync_status(hdev, HCI_OP_READ_DEF_ERR_DATA_REPORTING, @@ -3349,6 +3617,19 @@ static int hci_le_set_event_mask_sync(struct hci_dev *hdev) if (ext_adv_capable(hdev)) events[2] |= 0x02; /* LE Advertising Set Terminated */ + if (cis_capable(hdev)) { + events[3] |= 0x01; /* LE CIS Established */ + if (cis_peripheral_capable(hdev)) + events[3] |= 0x02; /* LE CIS Request */ + } + + if (bis_capable(hdev)) { + events[3] |= 0x04; /* LE Create BIG Complete */ + events[3] |= 0x08; /* LE Terminate BIG Complete */ + events[3] |= 0x10; /* LE BIG Sync Established */ + events[3] |= 0x20; /* LE BIG Sync Loss */ + } + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EVENT_MASK, sizeof(events), events, HCI_CMD_TIMEOUT); } @@ -3489,6 +3770,24 @@ static int hci_set_le_support_sync(struct hci_dev *hdev) sizeof(cp), &cp, HCI_CMD_TIMEOUT); } +/* LE Set Host Feature */ +static int hci_le_set_host_feature_sync(struct hci_dev *hdev) +{ + struct hci_cp_le_set_host_feature cp; + + if (!iso_capable(hdev)) + return 0; + + memset(&cp, 0, sizeof(cp)); + + /* Isochronous Channels (Host Support) */ + cp.bit_number = 32; + cp.bit_value = 1; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_HOST_FEATURE, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + /* LE Controller init stage 3 command sequence */ static const struct hci_init_stage le_init3[] = { /* HCI_OP_LE_SET_EVENT_MASK */ @@ -3515,6 +3814,8 @@ static const struct hci_init_stage le_init3[] = { HCI_INIT(hci_le_read_num_support_adv_sets_sync), /* HCI_OP_WRITE_LE_HOST_SUPPORTED */ HCI_INIT(hci_set_le_support_sync), + /* HCI_OP_LE_SET_HOST_FEATURE */ + HCI_INIT(hci_le_set_host_feature_sync), {} }; @@ -3578,7 +3879,7 @@ static int hci_set_event_mask_page_2_sync(struct hci_dev *hdev) if (lmp_cpb_central_capable(hdev)) { events[1] |= 0x40; /* Triggered Clock Capture */ events[1] |= 0x80; /* Synchronization Train Complete */ - events[2] |= 0x10; /* Peripheral Page Response Timeout */ + events[2] |= 0x08; /* Truncated Page Complete */ events[2] |= 0x20; /* CPB Channel Map Change */ changed = true; } @@ -3590,7 +3891,7 @@ static int hci_set_event_mask_page_2_sync(struct hci_dev *hdev) events[2] |= 0x01; /* Synchronization Train Received */ events[2] |= 0x02; /* CPB Receive */ events[2] |= 0x04; /* CPB Timeout */ - events[2] |= 0x08; /* Truncated Page Complete */ + events[2] |= 0x10; /* Peripheral Page Response Timeout */ changed = true; } @@ -3676,7 +3977,7 @@ static int hci_set_err_data_report_sync(struct hci_dev *hdev) bool enabled = hci_dev_test_flag(hdev, HCI_WIDEBAND_SPEECH_ENABLED); if (!(hdev->commands[18] & 0x08) || - test_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks)) + !(hdev->features[0][6] & LMP_ERR_DATA_REPORTING)) return 0; if (enabled == hdev->err_data_reporting) @@ -3835,9 +4136,6 @@ static const struct { HCI_QUIRK_BROKEN(STORED_LINK_KEY, "HCI Delete Stored Link Key command is advertised, " "but not supported."), - HCI_QUIRK_BROKEN(ERR_DATA_REPORTING, - "HCI Read Default Erroneous Data Reporting command is " - "advertised, but not supported."), HCI_QUIRK_BROKEN(READ_TRANSMIT_POWER, "HCI Read Transmit Power Level command is advertised, " "but not supported."), @@ -3848,90 +4146,40 @@ static const struct { "advertised, but not supported.") }; -int hci_dev_open_sync(struct hci_dev *hdev) +/* This function handles hdev setup stage: + * + * Calls hdev->setup + * Setup address if HCI_QUIRK_USE_BDADDR_PROPERTY is set. + */ +static int hci_dev_setup_sync(struct hci_dev *hdev) { int ret = 0; - - bt_dev_dbg(hdev, ""); - - if (hci_dev_test_flag(hdev, HCI_UNREGISTER)) { - ret = -ENODEV; - goto done; - } + bool invalid_bdaddr; + size_t i; if (!hci_dev_test_flag(hdev, HCI_SETUP) && - !hci_dev_test_flag(hdev, HCI_CONFIG)) { - /* Check for rfkill but allow the HCI setup stage to - * proceed (which in itself doesn't cause any RF activity). - */ - if (hci_dev_test_flag(hdev, HCI_RFKILLED)) { - ret = -ERFKILL; - goto done; - } - - /* Check for valid public address or a configured static - * random address, but let the HCI setup proceed to - * be able to determine if there is a public address - * or not. - * - * In case of user channel usage, it is not important - * if a public address or static random address is - * available. - * - * This check is only valid for BR/EDR controllers - * since AMP controllers do not have an address. - */ - if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && - hdev->dev_type == HCI_PRIMARY && - !bacmp(&hdev->bdaddr, BDADDR_ANY) && - !bacmp(&hdev->static_addr, BDADDR_ANY)) { - ret = -EADDRNOTAVAIL; - goto done; - } - } - - if (test_bit(HCI_UP, &hdev->flags)) { - ret = -EALREADY; - goto done; - } - - if (hdev->open(hdev)) { - ret = -EIO; - goto done; - } - - set_bit(HCI_RUNNING, &hdev->flags); - hci_sock_dev_event(hdev, HCI_DEV_OPEN); - - atomic_set(&hdev->cmd_cnt, 1); - set_bit(HCI_INIT, &hdev->flags); + !test_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks)) + return 0; - if (hci_dev_test_flag(hdev, HCI_SETUP) || - test_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks)) { - bool invalid_bdaddr; - size_t i; + bt_dev_dbg(hdev, ""); - hci_sock_dev_event(hdev, HCI_DEV_SETUP); + hci_sock_dev_event(hdev, HCI_DEV_SETUP); - if (hdev->setup) - ret = hdev->setup(hdev); + if (hdev->setup) + ret = hdev->setup(hdev); - for (i = 0; i < ARRAY_SIZE(hci_broken_table); i++) { - if (test_bit(hci_broken_table[i].quirk, &hdev->quirks)) - bt_dev_warn(hdev, "%s", - hci_broken_table[i].desc); - } - - /* The transport driver can set the quirk to mark the - * BD_ADDR invalid before creating the HCI device or in - * its setup callback. - */ - invalid_bdaddr = test_bit(HCI_QUIRK_INVALID_BDADDR, - &hdev->quirks); + for (i = 0; i < ARRAY_SIZE(hci_broken_table); i++) { + if (test_bit(hci_broken_table[i].quirk, &hdev->quirks)) + bt_dev_warn(hdev, "%s", hci_broken_table[i].desc); + } - if (ret) - goto setup_failed; + /* The transport driver can set the quirk to mark the + * BD_ADDR invalid before creating the HCI device or in + * its setup callback. + */ + invalid_bdaddr = test_bit(HCI_QUIRK_INVALID_BDADDR, &hdev->quirks); + if (!ret) { if (test_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &hdev->quirks)) { if (!bacmp(&hdev->public_addr, BDADDR_ANY)) hci_dev_get_bd_addr_from_property(hdev); @@ -3950,33 +4198,51 @@ int hci_dev_open_sync(struct hci_dev *hdev) invalid_bdaddr = false; } } + } -setup_failed: - /* The transport driver can set these quirks before - * creating the HCI device or in its setup callback. - * - * For the invalid BD_ADDR quirk it is possible that - * it becomes a valid address if the bootloader does - * provide it (see above). - * - * In case any of them is set, the controller has to - * start up as unconfigured. - */ - if (test_bit(HCI_QUIRK_EXTERNAL_CONFIG, &hdev->quirks) || - invalid_bdaddr) - hci_dev_set_flag(hdev, HCI_UNCONFIGURED); + /* The transport driver can set these quirks before + * creating the HCI device or in its setup callback. + * + * For the invalid BD_ADDR quirk it is possible that + * it becomes a valid address if the bootloader does + * provide it (see above). + * + * In case any of them is set, the controller has to + * start up as unconfigured. + */ + if (test_bit(HCI_QUIRK_EXTERNAL_CONFIG, &hdev->quirks) || + invalid_bdaddr) + hci_dev_set_flag(hdev, HCI_UNCONFIGURED); - /* For an unconfigured controller it is required to - * read at least the version information provided by - * the Read Local Version Information command. - * - * If the set_bdaddr driver callback is provided, then - * also the original Bluetooth public device address - * will be read using the Read BD Address command. - */ - if (hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) - ret = hci_unconf_init_sync(hdev); - } + /* For an unconfigured controller it is required to + * read at least the version information provided by + * the Read Local Version Information command. + * + * If the set_bdaddr driver callback is provided, then + * also the original Bluetooth public device address + * will be read using the Read BD Address command. + */ + if (hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) + return hci_unconf_init_sync(hdev); + + return ret; +} + +/* This function handles hdev init stage: + * + * Calls hci_dev_setup_sync to perform setup stage + * Calls hci_init_sync to perform HCI command init sequence + */ +static int hci_dev_init_sync(struct hci_dev *hdev) +{ + int ret; + + bt_dev_dbg(hdev, ""); + + atomic_set(&hdev->cmd_cnt, 1); + set_bit(HCI_INIT, &hdev->flags); + + ret = hci_dev_setup_sync(hdev); if (hci_dev_test_flag(hdev, HCI_CONFIG)) { /* If public address change is configured, ensure that @@ -4016,6 +4282,65 @@ setup_failed: clear_bit(HCI_INIT, &hdev->flags); + return ret; +} + +int hci_dev_open_sync(struct hci_dev *hdev) +{ + int ret; + + bt_dev_dbg(hdev, ""); + + if (hci_dev_test_flag(hdev, HCI_UNREGISTER)) { + ret = -ENODEV; + goto done; + } + + if (!hci_dev_test_flag(hdev, HCI_SETUP) && + !hci_dev_test_flag(hdev, HCI_CONFIG)) { + /* Check for rfkill but allow the HCI setup stage to + * proceed (which in itself doesn't cause any RF activity). + */ + if (hci_dev_test_flag(hdev, HCI_RFKILLED)) { + ret = -ERFKILL; + goto done; + } + + /* Check for valid public address or a configured static + * random address, but let the HCI setup proceed to + * be able to determine if there is a public address + * or not. + * + * In case of user channel usage, it is not important + * if a public address or static random address is + * available. + * + * This check is only valid for BR/EDR controllers + * since AMP controllers do not have an address. + */ + if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && + hdev->dev_type == HCI_PRIMARY && + !bacmp(&hdev->bdaddr, BDADDR_ANY) && + !bacmp(&hdev->static_addr, BDADDR_ANY)) { + ret = -EADDRNOTAVAIL; + goto done; + } + } + + if (test_bit(HCI_UP, &hdev->flags)) { + ret = -EALREADY; + goto done; + } + + if (hdev->open(hdev)) { + ret = -EIO; + goto done; + } + + set_bit(HCI_RUNNING, &hdev->flags); + hci_sock_dev_event(hdev, HCI_DEV_OPEN); + + ret = hci_dev_init_sync(hdev); if (!ret) { hci_dev_hold(hdev); hci_dev_set_flag(hdev, HCI_RPA_EXPIRED); @@ -4435,8 +4760,7 @@ static int hci_reject_conn_sync(struct hci_dev *hdev, struct hci_conn *conn, sizeof(cp), &cp, HCI_CMD_TIMEOUT); } -static int hci_abort_conn_sync(struct hci_dev *hdev, struct hci_conn *conn, - u8 reason) +int hci_abort_conn_sync(struct hci_dev *hdev, struct hci_conn *conn, u8 reason) { int err; @@ -5039,13 +5363,13 @@ static int hci_resume_scan_sync(struct hci_dev *hdev) if (!hdev->scanning_paused) return 0; + hdev->scanning_paused = false; + hci_update_scan_sync(hdev); /* Reset passive scanning to normal */ hci_update_passive_scan_sync(hdev); - hdev->scanning_paused = false; - return 0; } @@ -5064,7 +5388,6 @@ int hci_resume_sync(struct hci_dev *hdev) return 0; hdev->suspended = false; - hdev->scanning_paused = false; /* Restore event mask */ hci_set_event_mask_sync(hdev); @@ -5375,3 +5698,36 @@ done: hci_resume_advertising_sync(hdev); return err; } + +int hci_le_remove_cig_sync(struct hci_dev *hdev, u8 handle) +{ + struct hci_cp_le_remove_cig cp; + + memset(&cp, 0, sizeof(cp)); + cp.cig_id = handle; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_REMOVE_CIG, sizeof(cp), + &cp, HCI_CMD_TIMEOUT); +} + +int hci_le_big_terminate_sync(struct hci_dev *hdev, u8 handle) +{ + struct hci_cp_le_big_term_sync cp; + + memset(&cp, 0, sizeof(cp)); + cp.handle = handle; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_BIG_TERM_SYNC, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +int hci_le_pa_terminate_sync(struct hci_dev *hdev, u16 handle) +{ + struct hci_cp_le_pa_term_sync cp; + + memset(&cp, 0, sizeof(cp)); + cp.handle = cpu_to_le16(handle); + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_PA_TERM_SYNC, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c new file mode 100644 index 000000000000..ff09c353e64e --- /dev/null +++ b/net/bluetooth/iso.c @@ -0,0 +1,1824 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * BlueZ - Bluetooth protocol stack for Linux + * + * Copyright (C) 2022 Intel Corporation + */ + +#include <linux/module.h> +#include <linux/debugfs.h> +#include <linux/seq_file.h> +#include <linux/sched/signal.h> + +#include <net/bluetooth/bluetooth.h> +#include <net/bluetooth/hci_core.h> +#include <net/bluetooth/iso.h> + +static const struct proto_ops iso_sock_ops; + +static struct bt_sock_list iso_sk_list = { + .lock = __RW_LOCK_UNLOCKED(iso_sk_list.lock) +}; + +/* ---- ISO connections ---- */ +struct iso_conn { + struct hci_conn *hcon; + + /* @lock: spinlock protecting changes to iso_conn fields */ + spinlock_t lock; + struct sock *sk; + + struct delayed_work timeout_work; + + struct sk_buff *rx_skb; + __u32 rx_len; + __u16 tx_sn; +}; + +#define iso_conn_lock(c) spin_lock(&(c)->lock) +#define iso_conn_unlock(c) spin_unlock(&(c)->lock) + +static void iso_sock_close(struct sock *sk); +static void iso_sock_kill(struct sock *sk); + +/* ----- ISO socket info ----- */ +#define iso_pi(sk) ((struct iso_pinfo *)sk) + +struct iso_pinfo { + struct bt_sock bt; + bdaddr_t src; + __u8 src_type; + bdaddr_t dst; + __u8 dst_type; + __u8 bc_sid; + __u8 bc_num_bis; + __u8 bc_bis[ISO_MAX_NUM_BIS]; + __u16 sync_handle; + __u32 flags; + struct bt_iso_qos qos; + __u8 base_len; + __u8 base[HCI_MAX_PER_AD_LENGTH]; + struct iso_conn *conn; +}; + +/* ---- ISO timers ---- */ +#define ISO_CONN_TIMEOUT (HZ * 40) +#define ISO_DISCONN_TIMEOUT (HZ * 2) + +static void iso_sock_timeout(struct work_struct *work) +{ + struct iso_conn *conn = container_of(work, struct iso_conn, + timeout_work.work); + struct sock *sk; + + iso_conn_lock(conn); + sk = conn->sk; + if (sk) + sock_hold(sk); + iso_conn_unlock(conn); + + if (!sk) + return; + + BT_DBG("sock %p state %d", sk, sk->sk_state); + + lock_sock(sk); + sk->sk_err = ETIMEDOUT; + sk->sk_state_change(sk); + release_sock(sk); + sock_put(sk); +} + +static void iso_sock_set_timer(struct sock *sk, long timeout) +{ + if (!iso_pi(sk)->conn) + return; + + BT_DBG("sock %p state %d timeout %ld", sk, sk->sk_state, timeout); + cancel_delayed_work(&iso_pi(sk)->conn->timeout_work); + schedule_delayed_work(&iso_pi(sk)->conn->timeout_work, timeout); +} + +static void iso_sock_clear_timer(struct sock *sk) +{ + if (!iso_pi(sk)->conn) + return; + + BT_DBG("sock %p state %d", sk, sk->sk_state); + cancel_delayed_work(&iso_pi(sk)->conn->timeout_work); +} + +/* ---- ISO connections ---- */ +static struct iso_conn *iso_conn_add(struct hci_conn *hcon) +{ + struct iso_conn *conn = hcon->iso_data; + + if (conn) + return conn; + + conn = kzalloc(sizeof(*conn), GFP_KERNEL); + if (!conn) + return NULL; + + spin_lock_init(&conn->lock); + INIT_DELAYED_WORK(&conn->timeout_work, iso_sock_timeout); + + hcon->iso_data = conn; + conn->hcon = hcon; + conn->tx_sn = 0; + + BT_DBG("hcon %p conn %p", hcon, conn); + + return conn; +} + +/* Delete channel. Must be called on the locked socket. */ +static void iso_chan_del(struct sock *sk, int err) +{ + struct iso_conn *conn; + struct sock *parent; + + conn = iso_pi(sk)->conn; + + BT_DBG("sk %p, conn %p, err %d", sk, conn, err); + + if (conn) { + iso_conn_lock(conn); + conn->sk = NULL; + iso_pi(sk)->conn = NULL; + iso_conn_unlock(conn); + + if (conn->hcon) + hci_conn_drop(conn->hcon); + } + + sk->sk_state = BT_CLOSED; + sk->sk_err = err; + + parent = bt_sk(sk)->parent; + if (parent) { + bt_accept_unlink(sk); + parent->sk_data_ready(parent); + } else { + sk->sk_state_change(sk); + } + + sock_set_flag(sk, SOCK_ZAPPED); +} + +static void iso_conn_del(struct hci_conn *hcon, int err) +{ + struct iso_conn *conn = hcon->iso_data; + struct sock *sk; + + if (!conn) + return; + + BT_DBG("hcon %p conn %p, err %d", hcon, conn, err); + + /* Kill socket */ + iso_conn_lock(conn); + sk = conn->sk; + if (sk) + sock_hold(sk); + iso_conn_unlock(conn); + + if (sk) { + lock_sock(sk); + iso_sock_clear_timer(sk); + iso_chan_del(sk, err); + release_sock(sk); + sock_put(sk); + } + + /* Ensure no more work items will run before freeing conn. */ + cancel_delayed_work_sync(&conn->timeout_work); + + hcon->iso_data = NULL; + kfree(conn); +} + +static int __iso_chan_add(struct iso_conn *conn, struct sock *sk, + struct sock *parent) +{ + BT_DBG("conn %p", conn); + + if (iso_pi(sk)->conn == conn && conn->sk == sk) + return 0; + + if (conn->sk) { + BT_ERR("conn->sk already set"); + return -EBUSY; + } + + iso_pi(sk)->conn = conn; + conn->sk = sk; + + if (parent) + bt_accept_enqueue(parent, sk, true); + + return 0; +} + +static int iso_chan_add(struct iso_conn *conn, struct sock *sk, + struct sock *parent) +{ + int err; + + iso_conn_lock(conn); + err = __iso_chan_add(conn, sk, parent); + iso_conn_unlock(conn); + + return err; +} + +static int iso_connect_bis(struct sock *sk) +{ + struct iso_conn *conn; + struct hci_conn *hcon; + struct hci_dev *hdev; + int err; + + BT_DBG("%pMR", &iso_pi(sk)->src); + + hdev = hci_get_route(&iso_pi(sk)->dst, &iso_pi(sk)->src, + iso_pi(sk)->src_type); + if (!hdev) + return -EHOSTUNREACH; + + hci_dev_lock(hdev); + + if (!bis_capable(hdev)) { + err = -EOPNOTSUPP; + goto done; + } + + /* Fail if out PHYs are marked as disabled */ + if (!iso_pi(sk)->qos.out.phy) { + err = -EINVAL; + goto done; + } + + hcon = hci_connect_bis(hdev, &iso_pi(sk)->dst, iso_pi(sk)->dst_type, + &iso_pi(sk)->qos, iso_pi(sk)->base_len, + iso_pi(sk)->base); + if (IS_ERR(hcon)) { + err = PTR_ERR(hcon); + goto done; + } + + conn = iso_conn_add(hcon); + if (!conn) { + hci_conn_drop(hcon); + err = -ENOMEM; + goto done; + } + + /* Update source addr of the socket */ + bacpy(&iso_pi(sk)->src, &hcon->src); + + err = iso_chan_add(conn, sk, NULL); + if (err) + goto done; + + if (hcon->state == BT_CONNECTED) { + iso_sock_clear_timer(sk); + sk->sk_state = BT_CONNECTED; + } else { + sk->sk_state = BT_CONNECT; + iso_sock_set_timer(sk, sk->sk_sndtimeo); + } + +done: + hci_dev_unlock(hdev); + hci_dev_put(hdev); + return err; +} + +static int iso_connect_cis(struct sock *sk) +{ + struct iso_conn *conn; + struct hci_conn *hcon; + struct hci_dev *hdev; + int err; + + BT_DBG("%pMR -> %pMR", &iso_pi(sk)->src, &iso_pi(sk)->dst); + + hdev = hci_get_route(&iso_pi(sk)->dst, &iso_pi(sk)->src, + iso_pi(sk)->src_type); + if (!hdev) + return -EHOSTUNREACH; + + hci_dev_lock(hdev); + + if (!cis_central_capable(hdev)) { + err = -EOPNOTSUPP; + goto done; + } + + /* Fail if either PHYs are marked as disabled */ + if (!iso_pi(sk)->qos.in.phy && !iso_pi(sk)->qos.out.phy) { + err = -EINVAL; + goto done; + } + + /* Just bind if DEFER_SETUP has been set */ + if (test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags)) { + hcon = hci_bind_cis(hdev, &iso_pi(sk)->dst, + iso_pi(sk)->dst_type, &iso_pi(sk)->qos); + if (IS_ERR(hcon)) { + err = PTR_ERR(hcon); + goto done; + } + } else { + hcon = hci_connect_cis(hdev, &iso_pi(sk)->dst, + iso_pi(sk)->dst_type, &iso_pi(sk)->qos); + if (IS_ERR(hcon)) { + err = PTR_ERR(hcon); + goto done; + } + } + + conn = iso_conn_add(hcon); + if (!conn) { + hci_conn_drop(hcon); + err = -ENOMEM; + goto done; + } + + /* Update source addr of the socket */ + bacpy(&iso_pi(sk)->src, &hcon->src); + + err = iso_chan_add(conn, sk, NULL); + if (err) + goto done; + + if (hcon->state == BT_CONNECTED) { + iso_sock_clear_timer(sk); + sk->sk_state = BT_CONNECTED; + } else if (test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags)) { + iso_sock_clear_timer(sk); + sk->sk_state = BT_CONNECT; + } else { + sk->sk_state = BT_CONNECT; + iso_sock_set_timer(sk, sk->sk_sndtimeo); + } + +done: + hci_dev_unlock(hdev); + hci_dev_put(hdev); + return err; +} + +static int iso_send_frame(struct sock *sk, struct sk_buff *skb) +{ + struct iso_conn *conn = iso_pi(sk)->conn; + struct hci_iso_data_hdr *hdr; + int len = 0; + + BT_DBG("sk %p len %d", sk, skb->len); + + if (skb->len > iso_pi(sk)->qos.out.sdu) + return -EMSGSIZE; + + len = skb->len; + + /* Push ISO data header */ + hdr = skb_push(skb, HCI_ISO_DATA_HDR_SIZE); + hdr->sn = cpu_to_le16(conn->tx_sn++); + hdr->slen = cpu_to_le16(hci_iso_data_len_pack(len, + HCI_ISO_STATUS_VALID)); + + if (sk->sk_state == BT_CONNECTED) + hci_send_iso(conn->hcon, skb); + else + len = -ENOTCONN; + + return len; +} + +static void iso_recv_frame(struct iso_conn *conn, struct sk_buff *skb) +{ + struct sock *sk; + + iso_conn_lock(conn); + sk = conn->sk; + iso_conn_unlock(conn); + + if (!sk) + goto drop; + + BT_DBG("sk %p len %d", sk, skb->len); + + if (sk->sk_state != BT_CONNECTED) + goto drop; + + if (!sock_queue_rcv_skb(sk, skb)) + return; + +drop: + kfree_skb(skb); +} + +/* -------- Socket interface ---------- */ +static struct sock *__iso_get_sock_listen_by_addr(bdaddr_t *ba) +{ + struct sock *sk; + + sk_for_each(sk, &iso_sk_list.head) { + if (sk->sk_state != BT_LISTEN) + continue; + + if (!bacmp(&iso_pi(sk)->src, ba)) + return sk; + } + + return NULL; +} + +static struct sock *__iso_get_sock_listen_by_sid(bdaddr_t *ba, bdaddr_t *bc, + __u8 sid) +{ + struct sock *sk; + + sk_for_each(sk, &iso_sk_list.head) { + if (sk->sk_state != BT_LISTEN) + continue; + + if (bacmp(&iso_pi(sk)->src, ba)) + continue; + + if (bacmp(&iso_pi(sk)->dst, bc)) + continue; + + if (iso_pi(sk)->bc_sid == sid) + return sk; + } + + return NULL; +} + +typedef bool (*iso_sock_match_t)(struct sock *sk, void *data); + +/* Find socket listening: + * source bdaddr (Unicast) + * destination bdaddr (Broadcast only) + * match func - pass NULL to ignore + * match func data - pass -1 to ignore + * Returns closest match. + */ +static struct sock *iso_get_sock_listen(bdaddr_t *src, bdaddr_t *dst, + iso_sock_match_t match, void *data) +{ + struct sock *sk = NULL, *sk1 = NULL; + + read_lock(&iso_sk_list.lock); + + sk_for_each(sk, &iso_sk_list.head) { + if (sk->sk_state != BT_LISTEN) + continue; + + /* Match Broadcast destination */ + if (bacmp(dst, BDADDR_ANY) && bacmp(&iso_pi(sk)->dst, dst)) + continue; + + /* Use Match function if provided */ + if (match && !match(sk, data)) + continue; + + /* Exact match. */ + if (!bacmp(&iso_pi(sk)->src, src)) + break; + + /* Closest match */ + if (!bacmp(&iso_pi(sk)->src, BDADDR_ANY)) + sk1 = sk; + } + + read_unlock(&iso_sk_list.lock); + + return sk ? sk : sk1; +} + +static void iso_sock_destruct(struct sock *sk) +{ + BT_DBG("sk %p", sk); + + skb_queue_purge(&sk->sk_receive_queue); + skb_queue_purge(&sk->sk_write_queue); +} + +static void iso_sock_cleanup_listen(struct sock *parent) +{ + struct sock *sk; + + BT_DBG("parent %p", parent); + + /* Close not yet accepted channels */ + while ((sk = bt_accept_dequeue(parent, NULL))) { + iso_sock_close(sk); + iso_sock_kill(sk); + } + + parent->sk_state = BT_CLOSED; + sock_set_flag(parent, SOCK_ZAPPED); +} + +/* Kill socket (only if zapped and orphan) + * Must be called on unlocked socket. + */ +static void iso_sock_kill(struct sock *sk) +{ + if (!sock_flag(sk, SOCK_ZAPPED) || sk->sk_socket || + sock_flag(sk, SOCK_DEAD)) + return; + + BT_DBG("sk %p state %d", sk, sk->sk_state); + + /* Kill poor orphan */ + bt_sock_unlink(&iso_sk_list, sk); + sock_set_flag(sk, SOCK_DEAD); + sock_put(sk); +} + +static void iso_conn_defer_reject(struct hci_conn *conn) +{ + struct hci_cp_le_reject_cis cp; + + BT_DBG("conn %p", conn); + + memset(&cp, 0, sizeof(cp)); + cp.handle = cpu_to_le16(conn->handle); + cp.reason = HCI_ERROR_REJ_BAD_ADDR; + hci_send_cmd(conn->hdev, HCI_OP_LE_REJECT_CIS, sizeof(cp), &cp); +} + +static void __iso_sock_close(struct sock *sk) +{ + BT_DBG("sk %p state %d socket %p", sk, sk->sk_state, sk->sk_socket); + + switch (sk->sk_state) { + case BT_LISTEN: + iso_sock_cleanup_listen(sk); + break; + + case BT_CONNECTED: + case BT_CONFIG: + if (iso_pi(sk)->conn->hcon) { + sk->sk_state = BT_DISCONN; + iso_sock_set_timer(sk, ISO_DISCONN_TIMEOUT); + iso_conn_lock(iso_pi(sk)->conn); + hci_conn_drop(iso_pi(sk)->conn->hcon); + iso_pi(sk)->conn->hcon = NULL; + iso_conn_unlock(iso_pi(sk)->conn); + } else { + iso_chan_del(sk, ECONNRESET); + } + break; + + case BT_CONNECT2: + if (iso_pi(sk)->conn->hcon) + iso_conn_defer_reject(iso_pi(sk)->conn->hcon); + iso_chan_del(sk, ECONNRESET); + break; + case BT_CONNECT: + /* In case of DEFER_SETUP the hcon would be bound to CIG which + * needs to be removed so just call hci_conn_del so the cleanup + * callback do what is needed. + */ + if (test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags) && + iso_pi(sk)->conn->hcon) { + hci_conn_del(iso_pi(sk)->conn->hcon); + iso_pi(sk)->conn->hcon = NULL; + } + + iso_chan_del(sk, ECONNRESET); + break; + case BT_DISCONN: + iso_chan_del(sk, ECONNRESET); + break; + + default: + sock_set_flag(sk, SOCK_ZAPPED); + break; + } +} + +/* Must be called on unlocked socket. */ +static void iso_sock_close(struct sock *sk) +{ + iso_sock_clear_timer(sk); + lock_sock(sk); + __iso_sock_close(sk); + release_sock(sk); + iso_sock_kill(sk); +} + +static void iso_sock_init(struct sock *sk, struct sock *parent) +{ + BT_DBG("sk %p", sk); + + if (parent) { + sk->sk_type = parent->sk_type; + bt_sk(sk)->flags = bt_sk(parent)->flags; + security_sk_clone(parent, sk); + } +} + +static struct proto iso_proto = { + .name = "ISO", + .owner = THIS_MODULE, + .obj_size = sizeof(struct iso_pinfo) +}; + +#define DEFAULT_IO_QOS \ +{ \ + .interval = 10000u, \ + .latency = 10u, \ + .sdu = 40u, \ + .phy = BT_ISO_PHY_2M, \ + .rtn = 2u, \ +} + +static struct bt_iso_qos default_qos = { + .cig = BT_ISO_QOS_CIG_UNSET, + .cis = BT_ISO_QOS_CIS_UNSET, + .sca = 0x00, + .packing = 0x00, + .framing = 0x00, + .in = DEFAULT_IO_QOS, + .out = DEFAULT_IO_QOS, +}; + +static struct sock *iso_sock_alloc(struct net *net, struct socket *sock, + int proto, gfp_t prio, int kern) +{ + struct sock *sk; + + sk = sk_alloc(net, PF_BLUETOOTH, prio, &iso_proto, kern); + if (!sk) + return NULL; + + sock_init_data(sock, sk); + INIT_LIST_HEAD(&bt_sk(sk)->accept_q); + + sk->sk_destruct = iso_sock_destruct; + sk->sk_sndtimeo = ISO_CONN_TIMEOUT; + + sock_reset_flag(sk, SOCK_ZAPPED); + + sk->sk_protocol = proto; + sk->sk_state = BT_OPEN; + + /* Set address type as public as default src address is BDADDR_ANY */ + iso_pi(sk)->src_type = BDADDR_LE_PUBLIC; + + iso_pi(sk)->qos = default_qos; + + bt_sock_link(&iso_sk_list, sk); + return sk; +} + +static int iso_sock_create(struct net *net, struct socket *sock, int protocol, + int kern) +{ + struct sock *sk; + + BT_DBG("sock %p", sock); + + sock->state = SS_UNCONNECTED; + + if (sock->type != SOCK_SEQPACKET) + return -ESOCKTNOSUPPORT; + + sock->ops = &iso_sock_ops; + + sk = iso_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern); + if (!sk) + return -ENOMEM; + + iso_sock_init(sk, NULL); + return 0; +} + +static int iso_sock_bind_bc(struct socket *sock, struct sockaddr *addr, + int addr_len) +{ + struct sockaddr_iso *sa = (struct sockaddr_iso *)addr; + struct sock *sk = sock->sk; + int i; + + BT_DBG("sk %p bc_sid %u bc_num_bis %u", sk, sa->iso_bc->bc_sid, + sa->iso_bc->bc_num_bis); + + if (addr_len > sizeof(*sa) + sizeof(*sa->iso_bc) || + sa->iso_bc->bc_num_bis < 0x01 || sa->iso_bc->bc_num_bis > 0x1f) + return -EINVAL; + + bacpy(&iso_pi(sk)->dst, &sa->iso_bc->bc_bdaddr); + iso_pi(sk)->dst_type = sa->iso_bc->bc_bdaddr_type; + iso_pi(sk)->sync_handle = -1; + iso_pi(sk)->bc_sid = sa->iso_bc->bc_sid; + iso_pi(sk)->bc_num_bis = sa->iso_bc->bc_num_bis; + + for (i = 0; i < iso_pi(sk)->bc_num_bis; i++) { + if (sa->iso_bc->bc_bis[i] < 0x01 || + sa->iso_bc->bc_bis[i] > 0x1f) + return -EINVAL; + + memcpy(iso_pi(sk)->bc_bis, sa->iso_bc->bc_bis, + iso_pi(sk)->bc_num_bis); + } + + return 0; +} + +static int iso_sock_bind(struct socket *sock, struct sockaddr *addr, + int addr_len) +{ + struct sockaddr_iso *sa = (struct sockaddr_iso *)addr; + struct sock *sk = sock->sk; + int err = 0; + + BT_DBG("sk %p %pMR type %u", sk, &sa->iso_bdaddr, sa->iso_bdaddr_type); + + if (!addr || addr_len < sizeof(struct sockaddr_iso) || + addr->sa_family != AF_BLUETOOTH) + return -EINVAL; + + lock_sock(sk); + + if (sk->sk_state != BT_OPEN) { + err = -EBADFD; + goto done; + } + + if (sk->sk_type != SOCK_SEQPACKET) { + err = -EINVAL; + goto done; + } + + /* Check if the address type is of LE type */ + if (!bdaddr_type_is_le(sa->iso_bdaddr_type)) { + err = -EINVAL; + goto done; + } + + bacpy(&iso_pi(sk)->src, &sa->iso_bdaddr); + iso_pi(sk)->src_type = sa->iso_bdaddr_type; + + /* Check for Broadcast address */ + if (addr_len > sizeof(*sa)) { + err = iso_sock_bind_bc(sock, addr, addr_len); + if (err) + goto done; + } + + sk->sk_state = BT_BOUND; + +done: + release_sock(sk); + return err; +} + +static int iso_sock_connect(struct socket *sock, struct sockaddr *addr, + int alen, int flags) +{ + struct sockaddr_iso *sa = (struct sockaddr_iso *)addr; + struct sock *sk = sock->sk; + int err; + + BT_DBG("sk %p", sk); + + if (alen < sizeof(struct sockaddr_iso) || + addr->sa_family != AF_BLUETOOTH) + return -EINVAL; + + if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) + return -EBADFD; + + if (sk->sk_type != SOCK_SEQPACKET) + return -EINVAL; + + /* Check if the address type is of LE type */ + if (!bdaddr_type_is_le(sa->iso_bdaddr_type)) + return -EINVAL; + + lock_sock(sk); + + bacpy(&iso_pi(sk)->dst, &sa->iso_bdaddr); + iso_pi(sk)->dst_type = sa->iso_bdaddr_type; + + if (bacmp(&iso_pi(sk)->dst, BDADDR_ANY)) + err = iso_connect_cis(sk); + else + err = iso_connect_bis(sk); + + if (err) + goto done; + + if (!test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags)) { + err = bt_sock_wait_state(sk, BT_CONNECTED, + sock_sndtimeo(sk, flags & O_NONBLOCK)); + } + +done: + release_sock(sk); + return err; +} + +static int iso_listen_bis(struct sock *sk) +{ + struct hci_dev *hdev; + int err = 0; + + BT_DBG("%pMR -> %pMR (SID 0x%2.2x)", &iso_pi(sk)->src, + &iso_pi(sk)->dst, iso_pi(sk)->bc_sid); + + write_lock(&iso_sk_list.lock); + + if (__iso_get_sock_listen_by_sid(&iso_pi(sk)->src, &iso_pi(sk)->dst, + iso_pi(sk)->bc_sid)) + err = -EADDRINUSE; + + write_unlock(&iso_sk_list.lock); + + if (err) + return err; + + hdev = hci_get_route(&iso_pi(sk)->dst, &iso_pi(sk)->src, + iso_pi(sk)->src_type); + if (!hdev) + return -EHOSTUNREACH; + + hci_dev_lock(hdev); + + err = hci_pa_create_sync(hdev, &iso_pi(sk)->dst, iso_pi(sk)->dst_type, + iso_pi(sk)->bc_sid); + + hci_dev_unlock(hdev); + + return err; +} + +static int iso_listen_cis(struct sock *sk) +{ + int err = 0; + + BT_DBG("%pMR", &iso_pi(sk)->src); + + write_lock(&iso_sk_list.lock); + + if (__iso_get_sock_listen_by_addr(&iso_pi(sk)->src)) + err = -EADDRINUSE; + + write_unlock(&iso_sk_list.lock); + + return err; +} + +static int iso_sock_listen(struct socket *sock, int backlog) +{ + struct sock *sk = sock->sk; + int err = 0; + + BT_DBG("sk %p backlog %d", sk, backlog); + + lock_sock(sk); + + if (sk->sk_state != BT_BOUND) { + err = -EBADFD; + goto done; + } + + if (sk->sk_type != SOCK_SEQPACKET) { + err = -EINVAL; + goto done; + } + + if (!bacmp(&iso_pi(sk)->dst, BDADDR_ANY)) + err = iso_listen_cis(sk); + else + err = iso_listen_bis(sk); + + if (err) + goto done; + + sk->sk_max_ack_backlog = backlog; + sk->sk_ack_backlog = 0; + + sk->sk_state = BT_LISTEN; + +done: + release_sock(sk); + return err; +} + +static int iso_sock_accept(struct socket *sock, struct socket *newsock, + int flags, bool kern) +{ + DEFINE_WAIT_FUNC(wait, woken_wake_function); + struct sock *sk = sock->sk, *ch; + long timeo; + int err = 0; + + lock_sock(sk); + + timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); + + BT_DBG("sk %p timeo %ld", sk, timeo); + + /* Wait for an incoming connection. (wake-one). */ + add_wait_queue_exclusive(sk_sleep(sk), &wait); + while (1) { + if (sk->sk_state != BT_LISTEN) { + err = -EBADFD; + break; + } + + ch = bt_accept_dequeue(sk, newsock); + if (ch) + break; + + if (!timeo) { + err = -EAGAIN; + break; + } + + if (signal_pending(current)) { + err = sock_intr_errno(timeo); + break; + } + + release_sock(sk); + + timeo = wait_woken(&wait, TASK_INTERRUPTIBLE, timeo); + lock_sock(sk); + } + remove_wait_queue(sk_sleep(sk), &wait); + + if (err) + goto done; + + newsock->state = SS_CONNECTED; + + BT_DBG("new socket %p", ch); + +done: + release_sock(sk); + return err; +} + +static int iso_sock_getname(struct socket *sock, struct sockaddr *addr, + int peer) +{ + struct sockaddr_iso *sa = (struct sockaddr_iso *)addr; + struct sock *sk = sock->sk; + + BT_DBG("sock %p, sk %p", sock, sk); + + addr->sa_family = AF_BLUETOOTH; + + if (peer) { + bacpy(&sa->iso_bdaddr, &iso_pi(sk)->dst); + sa->iso_bdaddr_type = iso_pi(sk)->dst_type; + } else { + bacpy(&sa->iso_bdaddr, &iso_pi(sk)->src); + sa->iso_bdaddr_type = iso_pi(sk)->src_type; + } + + return sizeof(struct sockaddr_iso); +} + +static int iso_sock_sendmsg(struct socket *sock, struct msghdr *msg, + size_t len) +{ + struct sock *sk = sock->sk; + struct iso_conn *conn = iso_pi(sk)->conn; + struct sk_buff *skb, **frag; + int err; + + BT_DBG("sock %p, sk %p", sock, sk); + + err = sock_error(sk); + if (err) + return err; + + if (msg->msg_flags & MSG_OOB) + return -EOPNOTSUPP; + + if (sk->sk_state != BT_CONNECTED) + return -ENOTCONN; + + skb = bt_skb_sendmsg(sk, msg, len, conn->hcon->hdev->iso_mtu, + HCI_ISO_DATA_HDR_SIZE, 0); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + len -= skb->len; + + BT_DBG("skb %p len %d", sk, skb->len); + + /* Continuation fragments */ + frag = &skb_shinfo(skb)->frag_list; + while (len) { + struct sk_buff *tmp; + + tmp = bt_skb_sendmsg(sk, msg, len, conn->hcon->hdev->iso_mtu, + 0, 0); + if (IS_ERR(tmp)) { + kfree_skb(skb); + return PTR_ERR(tmp); + } + + *frag = tmp; + + len -= tmp->len; + + skb->len += tmp->len; + skb->data_len += tmp->len; + + BT_DBG("frag %p len %d", *frag, tmp->len); + + frag = &(*frag)->next; + } + + lock_sock(sk); + + if (sk->sk_state == BT_CONNECTED) + err = iso_send_frame(sk, skb); + else + err = -ENOTCONN; + + release_sock(sk); + + if (err < 0) + kfree_skb(skb); + return err; +} + +static void iso_conn_defer_accept(struct hci_conn *conn) +{ + struct hci_cp_le_accept_cis cp; + struct hci_dev *hdev = conn->hdev; + + BT_DBG("conn %p", conn); + + conn->state = BT_CONFIG; + + cp.handle = cpu_to_le16(conn->handle); + + hci_send_cmd(hdev, HCI_OP_LE_ACCEPT_CIS, sizeof(cp), &cp); +} + +static int iso_sock_recvmsg(struct socket *sock, struct msghdr *msg, + size_t len, int flags) +{ + struct sock *sk = sock->sk; + struct iso_pinfo *pi = iso_pi(sk); + int err; + + BT_DBG("sk %p", sk); + + lock_sock(sk); + + if (test_and_clear_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags)) { + switch (sk->sk_state) { + case BT_CONNECT2: + iso_conn_defer_accept(pi->conn->hcon); + sk->sk_state = BT_CONFIG; + release_sock(sk); + return 0; + case BT_CONNECT: + err = iso_connect_cis(sk); + release_sock(sk); + return err; + } + } + + release_sock(sk); + + return bt_sock_recvmsg(sock, msg, len, flags); +} + +static bool check_io_qos(struct bt_iso_io_qos *qos) +{ + /* If no PHY is enable SDU must be 0 */ + if (!qos->phy && qos->sdu) + return false; + + if (qos->interval && (qos->interval < 0xff || qos->interval > 0xfffff)) + return false; + + if (qos->latency && (qos->latency < 0x05 || qos->latency > 0xfa0)) + return false; + + if (qos->phy > BT_ISO_PHY_ANY) + return false; + + return true; +} + +static bool check_qos(struct bt_iso_qos *qos) +{ + if (qos->sca > 0x07) + return false; + + if (qos->packing > 0x01) + return false; + + if (qos->framing > 0x01) + return false; + + if (!check_io_qos(&qos->in)) + return false; + + if (!check_io_qos(&qos->out)) + return false; + + return true; +} + +static int iso_sock_setsockopt(struct socket *sock, int level, int optname, + sockptr_t optval, unsigned int optlen) +{ + struct sock *sk = sock->sk; + int len, err = 0; + struct bt_iso_qos qos; + u32 opt; + + BT_DBG("sk %p", sk); + + lock_sock(sk); + + switch (optname) { + case BT_DEFER_SETUP: + if (sk->sk_state != BT_BOUND && sk->sk_state != BT_LISTEN) { + err = -EINVAL; + break; + } + + if (copy_from_sockptr(&opt, optval, sizeof(u32))) { + err = -EFAULT; + break; + } + + if (opt) + set_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags); + else + clear_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags); + break; + + case BT_ISO_QOS: + if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND && + sk->sk_state != BT_CONNECT2) { + err = -EINVAL; + break; + } + + len = min_t(unsigned int, sizeof(qos), optlen); + if (len != sizeof(qos)) + return -EINVAL; + + memset(&qos, 0, sizeof(qos)); + + if (copy_from_sockptr(&qos, optval, len)) { + err = -EFAULT; + break; + } + + if (!check_qos(&qos)) { + err = -EINVAL; + break; + } + + iso_pi(sk)->qos = qos; + + break; + + case BT_ISO_BASE: + if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND && + sk->sk_state != BT_CONNECT2) { + err = -EINVAL; + break; + } + + if (optlen > sizeof(iso_pi(sk)->base)) { + err = -EOVERFLOW; + break; + } + + len = min_t(unsigned int, sizeof(iso_pi(sk)->base), optlen); + + if (copy_from_sockptr(iso_pi(sk)->base, optval, len)) { + err = -EFAULT; + break; + } + + iso_pi(sk)->base_len = len; + + break; + + default: + err = -ENOPROTOOPT; + break; + } + + release_sock(sk); + return err; +} + +static int iso_sock_getsockopt(struct socket *sock, int level, int optname, + char __user *optval, int __user *optlen) +{ + struct sock *sk = sock->sk; + int len, err = 0; + struct bt_iso_qos qos; + u8 base_len; + u8 *base; + + BT_DBG("sk %p", sk); + + if (get_user(len, optlen)) + return -EFAULT; + + lock_sock(sk); + + switch (optname) { + case BT_DEFER_SETUP: + if (sk->sk_state != BT_BOUND && sk->sk_state != BT_LISTEN) { + err = -EINVAL; + break; + } + + if (put_user(test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags), + (u32 __user *)optval)) + err = -EFAULT; + + break; + + case BT_ISO_QOS: + if (sk->sk_state == BT_CONNECTED || sk->sk_state == BT_CONNECT2) + qos = iso_pi(sk)->conn->hcon->iso_qos; + else + qos = iso_pi(sk)->qos; + + len = min_t(unsigned int, len, sizeof(qos)); + if (copy_to_user(optval, (char *)&qos, len)) + err = -EFAULT; + + break; + + case BT_ISO_BASE: + if (sk->sk_state == BT_CONNECTED) { + base_len = iso_pi(sk)->conn->hcon->le_per_adv_data_len; + base = iso_pi(sk)->conn->hcon->le_per_adv_data; + } else { + base_len = iso_pi(sk)->base_len; + base = iso_pi(sk)->base; + } + + len = min_t(unsigned int, len, base_len); + if (copy_to_user(optval, base, len)) + err = -EFAULT; + + break; + + default: + err = -ENOPROTOOPT; + break; + } + + release_sock(sk); + return err; +} + +static int iso_sock_shutdown(struct socket *sock, int how) +{ + struct sock *sk = sock->sk; + int err = 0; + + BT_DBG("sock %p, sk %p", sock, sk); + + if (!sk) + return 0; + + sock_hold(sk); + lock_sock(sk); + + if (!sk->sk_shutdown) { + sk->sk_shutdown = SHUTDOWN_MASK; + iso_sock_clear_timer(sk); + __iso_sock_close(sk); + + if (sock_flag(sk, SOCK_LINGER) && sk->sk_lingertime && + !(current->flags & PF_EXITING)) + err = bt_sock_wait_state(sk, BT_CLOSED, + sk->sk_lingertime); + } + + release_sock(sk); + sock_put(sk); + + return err; +} + +static int iso_sock_release(struct socket *sock) +{ + struct sock *sk = sock->sk; + int err = 0; + + BT_DBG("sock %p, sk %p", sock, sk); + + if (!sk) + return 0; + + iso_sock_close(sk); + + if (sock_flag(sk, SOCK_LINGER) && sk->sk_lingertime && + !(current->flags & PF_EXITING)) { + lock_sock(sk); + err = bt_sock_wait_state(sk, BT_CLOSED, sk->sk_lingertime); + release_sock(sk); + } + + sock_orphan(sk); + iso_sock_kill(sk); + return err; +} + +static void iso_sock_ready(struct sock *sk) +{ + BT_DBG("sk %p", sk); + + if (!sk) + return; + + lock_sock(sk); + iso_sock_clear_timer(sk); + sk->sk_state = BT_CONNECTED; + sk->sk_state_change(sk); + release_sock(sk); +} + +struct iso_list_data { + struct hci_conn *hcon; + int count; +}; + +static bool iso_match_big(struct sock *sk, void *data) +{ + struct hci_evt_le_big_sync_estabilished *ev = data; + + return ev->handle == iso_pi(sk)->qos.big; +} + +static void iso_conn_ready(struct iso_conn *conn) +{ + struct sock *parent; + struct sock *sk = conn->sk; + struct hci_ev_le_big_sync_estabilished *ev; + + BT_DBG("conn %p", conn); + + if (sk) { + iso_sock_ready(conn->sk); + } else { + iso_conn_lock(conn); + + if (!conn->hcon) { + iso_conn_unlock(conn); + return; + } + + ev = hci_recv_event_data(conn->hcon->hdev, + HCI_EVT_LE_BIG_SYNC_ESTABILISHED); + if (ev) + parent = iso_get_sock_listen(&conn->hcon->src, + &conn->hcon->dst, + iso_match_big, ev); + else + parent = iso_get_sock_listen(&conn->hcon->src, + BDADDR_ANY, NULL, NULL); + + if (!parent) { + iso_conn_unlock(conn); + return; + } + + lock_sock(parent); + + sk = iso_sock_alloc(sock_net(parent), NULL, + BTPROTO_ISO, GFP_ATOMIC, 0); + if (!sk) { + release_sock(parent); + iso_conn_unlock(conn); + return; + } + + iso_sock_init(sk, parent); + + bacpy(&iso_pi(sk)->src, &conn->hcon->src); + iso_pi(sk)->src_type = conn->hcon->src_type; + + /* If hcon has no destination address (BDADDR_ANY) it means it + * was created by HCI_EV_LE_BIG_SYNC_ESTABILISHED so we need to + * initialize using the parent socket destination address. + */ + if (!bacmp(&conn->hcon->dst, BDADDR_ANY)) { + bacpy(&conn->hcon->dst, &iso_pi(parent)->dst); + conn->hcon->dst_type = iso_pi(parent)->dst_type; + conn->hcon->sync_handle = iso_pi(parent)->sync_handle; + } + + bacpy(&iso_pi(sk)->dst, &conn->hcon->dst); + iso_pi(sk)->dst_type = conn->hcon->dst_type; + + hci_conn_hold(conn->hcon); + __iso_chan_add(conn, sk, parent); + + if (test_bit(BT_SK_DEFER_SETUP, &bt_sk(parent)->flags)) + sk->sk_state = BT_CONNECT2; + else + sk->sk_state = BT_CONNECTED; + + /* Wake up parent */ + parent->sk_data_ready(parent); + + release_sock(parent); + + iso_conn_unlock(conn); + } +} + +static bool iso_match_sid(struct sock *sk, void *data) +{ + struct hci_ev_le_pa_sync_established *ev = data; + + return ev->sid == iso_pi(sk)->bc_sid; +} + +static bool iso_match_sync_handle(struct sock *sk, void *data) +{ + struct hci_evt_le_big_info_adv_report *ev = data; + + return le16_to_cpu(ev->sync_handle) == iso_pi(sk)->sync_handle; +} + +/* ----- ISO interface with lower layer (HCI) ----- */ + +int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) +{ + struct hci_ev_le_pa_sync_established *ev1; + struct hci_evt_le_big_info_adv_report *ev2; + struct sock *sk; + int lm = 0; + + bt_dev_dbg(hdev, "bdaddr %pMR", bdaddr); + + /* Broadcast receiver requires handling of some events before it can + * proceed to establishing a BIG sync: + * + * 1. HCI_EV_LE_PA_SYNC_ESTABLISHED: The socket may specify a specific + * SID to listen to and once sync is estabilished its handle needs to + * be stored in iso_pi(sk)->sync_handle so it can be matched once + * receiving the BIG Info. + * 2. HCI_EVT_LE_BIG_INFO_ADV_REPORT: When connect_ind is triggered by a + * a BIG Info it attempts to check if there any listening socket with + * the same sync_handle and if it does then attempt to create a sync. + */ + ev1 = hci_recv_event_data(hdev, HCI_EV_LE_PA_SYNC_ESTABLISHED); + if (ev1) { + sk = iso_get_sock_listen(&hdev->bdaddr, bdaddr, iso_match_sid, + ev1); + if (sk) + iso_pi(sk)->sync_handle = le16_to_cpu(ev1->handle); + + goto done; + } + + ev2 = hci_recv_event_data(hdev, HCI_EVT_LE_BIG_INFO_ADV_REPORT); + if (ev2) { + sk = iso_get_sock_listen(&hdev->bdaddr, bdaddr, + iso_match_sync_handle, ev2); + if (sk) { + int err; + + if (ev2->num_bis < iso_pi(sk)->bc_num_bis) + iso_pi(sk)->bc_num_bis = ev2->num_bis; + + err = hci_le_big_create_sync(hdev, + &iso_pi(sk)->qos, + iso_pi(sk)->sync_handle, + iso_pi(sk)->bc_num_bis, + iso_pi(sk)->bc_bis); + if (err) { + bt_dev_err(hdev, "hci_le_big_create_sync: %d", + err); + sk = NULL; + } + } + } else { + sk = iso_get_sock_listen(&hdev->bdaddr, BDADDR_ANY, NULL, NULL); + } + +done: + if (!sk) + return lm; + + lm |= HCI_LM_ACCEPT; + + if (test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags)) + *flags |= HCI_PROTO_DEFER; + + return lm; +} + +static void iso_connect_cfm(struct hci_conn *hcon, __u8 status) +{ + if (hcon->type != ISO_LINK) { + if (hcon->type != LE_LINK) + return; + + /* Check if LE link has failed */ + if (status) { + if (hcon->link) + iso_conn_del(hcon->link, bt_to_errno(status)); + return; + } + + /* Create CIS if pending */ + hci_le_create_cis(hcon); + return; + } + + BT_DBG("hcon %p bdaddr %pMR status %d", hcon, &hcon->dst, status); + + if (!status) { + struct iso_conn *conn; + + conn = iso_conn_add(hcon); + if (conn) + iso_conn_ready(conn); + } else { + iso_conn_del(hcon, bt_to_errno(status)); + } +} + +static void iso_disconn_cfm(struct hci_conn *hcon, __u8 reason) +{ + if (hcon->type != ISO_LINK) + return; + + BT_DBG("hcon %p reason %d", hcon, reason); + + iso_conn_del(hcon, bt_to_errno(reason)); +} + +void iso_recv(struct hci_conn *hcon, struct sk_buff *skb, u16 flags) +{ + struct iso_conn *conn = hcon->iso_data; + struct hci_iso_data_hdr *hdr; + __u16 pb, ts, len; + + if (!conn) + goto drop; + + pb = hci_iso_flags_pb(flags); + ts = hci_iso_flags_ts(flags); + + BT_DBG("conn %p len %d pb 0x%x ts 0x%x", conn, skb->len, pb, ts); + + switch (pb) { + case ISO_START: + case ISO_SINGLE: + if (conn->rx_len) { + BT_ERR("Unexpected start frame (len %d)", skb->len); + kfree_skb(conn->rx_skb); + conn->rx_skb = NULL; + conn->rx_len = 0; + } + + if (ts) { + /* TODO: add timestamp to the packet? */ + hdr = skb_pull_data(skb, HCI_ISO_TS_DATA_HDR_SIZE); + if (!hdr) { + BT_ERR("Frame is too short (len %d)", skb->len); + goto drop; + } + + } else { + hdr = skb_pull_data(skb, HCI_ISO_DATA_HDR_SIZE); + if (!hdr) { + BT_ERR("Frame is too short (len %d)", skb->len); + goto drop; + } + } + + len = __le16_to_cpu(hdr->slen); + flags = hci_iso_data_flags(len); + len = hci_iso_data_len(len); + + BT_DBG("Start: total len %d, frag len %d flags 0x%4.4x", len, + skb->len, flags); + + if (len == skb->len) { + /* Complete frame received */ + iso_recv_frame(conn, skb); + return; + } + + if (pb == ISO_SINGLE) { + BT_ERR("Frame malformed (len %d, expected len %d)", + skb->len, len); + goto drop; + } + + if (skb->len > len) { + BT_ERR("Frame is too long (len %d, expected len %d)", + skb->len, len); + goto drop; + } + + /* Allocate skb for the complete frame (with header) */ + conn->rx_skb = bt_skb_alloc(len, GFP_KERNEL); + if (!conn->rx_skb) + goto drop; + + skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), + skb->len); + conn->rx_len = len - skb->len; + break; + + case ISO_CONT: + BT_DBG("Cont: frag len %d (expecting %d)", skb->len, + conn->rx_len); + + if (!conn->rx_len) { + BT_ERR("Unexpected continuation frame (len %d)", + skb->len); + goto drop; + } + + if (skb->len > conn->rx_len) { + BT_ERR("Fragment is too long (len %d, expected %d)", + skb->len, conn->rx_len); + kfree_skb(conn->rx_skb); + conn->rx_skb = NULL; + conn->rx_len = 0; + goto drop; + } + + skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), + skb->len); + conn->rx_len -= skb->len; + return; + + case ISO_END: + skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), + skb->len); + conn->rx_len -= skb->len; + + if (!conn->rx_len) { + struct sk_buff *rx_skb = conn->rx_skb; + + /* Complete frame received. iso_recv_frame + * takes ownership of the skb so set the global + * rx_skb pointer to NULL first. + */ + conn->rx_skb = NULL; + iso_recv_frame(conn, rx_skb); + } + break; + } + +drop: + kfree_skb(skb); +} + +static struct hci_cb iso_cb = { + .name = "ISO", + .connect_cfm = iso_connect_cfm, + .disconn_cfm = iso_disconn_cfm, +}; + +static int iso_debugfs_show(struct seq_file *f, void *p) +{ + struct sock *sk; + + read_lock(&iso_sk_list.lock); + + sk_for_each(sk, &iso_sk_list.head) { + seq_printf(f, "%pMR %pMR %d\n", &iso_pi(sk)->src, + &iso_pi(sk)->dst, sk->sk_state); + } + + read_unlock(&iso_sk_list.lock); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(iso_debugfs); + +static struct dentry *iso_debugfs; + +static const struct proto_ops iso_sock_ops = { + .family = PF_BLUETOOTH, + .owner = THIS_MODULE, + .release = iso_sock_release, + .bind = iso_sock_bind, + .connect = iso_sock_connect, + .listen = iso_sock_listen, + .accept = iso_sock_accept, + .getname = iso_sock_getname, + .sendmsg = iso_sock_sendmsg, + .recvmsg = iso_sock_recvmsg, + .poll = bt_sock_poll, + .ioctl = bt_sock_ioctl, + .mmap = sock_no_mmap, + .socketpair = sock_no_socketpair, + .shutdown = iso_sock_shutdown, + .setsockopt = iso_sock_setsockopt, + .getsockopt = iso_sock_getsockopt +}; + +static const struct net_proto_family iso_sock_family_ops = { + .family = PF_BLUETOOTH, + .owner = THIS_MODULE, + .create = iso_sock_create, +}; + +static bool iso_inited; + +bool iso_enabled(void) +{ + return iso_inited; +} + +int iso_init(void) +{ + int err; + + BUILD_BUG_ON(sizeof(struct sockaddr_iso) > sizeof(struct sockaddr)); + + if (iso_inited) + return -EALREADY; + + err = proto_register(&iso_proto, 0); + if (err < 0) + return err; + + err = bt_sock_register(BTPROTO_ISO, &iso_sock_family_ops); + if (err < 0) { + BT_ERR("ISO socket registration failed"); + goto error; + } + + err = bt_procfs_init(&init_net, "iso", &iso_sk_list, NULL); + if (err < 0) { + BT_ERR("Failed to create ISO proc file"); + bt_sock_unregister(BTPROTO_ISO); + goto error; + } + + BT_INFO("ISO socket layer initialized"); + + hci_register_cb(&iso_cb); + + if (IS_ERR_OR_NULL(bt_debugfs)) + return 0; + + if (!iso_debugfs) { + iso_debugfs = debugfs_create_file("iso", 0444, bt_debugfs, + NULL, &iso_debugfs_fops); + } + + iso_inited = true; + + return 0; + +error: + proto_unregister(&iso_proto); + return err; +} + +int iso_exit(void) +{ + if (!iso_inited) + return -EALREADY; + + bt_procfs_cleanup(&init_net, "iso"); + + debugfs_remove(iso_debugfs); + iso_debugfs = NULL; + + hci_unregister_cb(&iso_cb); + + bt_sock_unregister(BTPROTO_ISO); + + proto_unregister(&iso_proto); + + iso_inited = false; + + return 0; +} diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index ae78490ecd3d..09ecaf556de5 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -1369,6 +1369,7 @@ static void l2cap_le_connect(struct l2cap_chan *chan) l2cap_le_flowctl_init(chan, 0); + memset(&req, 0, sizeof(req)); req.psm = chan->psm; req.scid = cpu_to_le16(chan->scid); req.mtu = cpu_to_le16(chan->imtu); diff --git a/net/bluetooth/lib.c b/net/bluetooth/lib.c index 5326f41a58b7..469a0c95b6e8 100644 --- a/net/bluetooth/lib.c +++ b/net/bluetooth/lib.c @@ -135,6 +135,77 @@ int bt_to_errno(__u16 code) } EXPORT_SYMBOL(bt_to_errno); +/* Unix errno to Bluetooth error codes mapping */ +__u8 bt_status(int err) +{ + /* Don't convert if already positive value */ + if (err >= 0) + return err; + + switch (err) { + case -EBADRQC: + return 0x01; + + case -ENOTCONN: + return 0x02; + + case -EIO: + return 0x03; + + case -EHOSTDOWN: + return 0x04; + + case -EACCES: + return 0x05; + + case -EBADE: + return 0x06; + + case -ENOMEM: + return 0x07; + + case -ETIMEDOUT: + return 0x08; + + case -EMLINK: + return 0x09; + + case EALREADY: + return 0x0b; + + case -EBUSY: + return 0x0c; + + case -ECONNREFUSED: + return 0x0d; + + case -EOPNOTSUPP: + return 0x11; + + case -EINVAL: + return 0x12; + + case -ECONNRESET: + return 0x13; + + case -ECONNABORTED: + return 0x16; + + case ELOOP: + return 0x17; + + case -EPROTONOSUPPORT: + return 0x1a; + + case -EPROTO: + return 0x19; + + default: + return 0x1f; + } +} +EXPORT_SYMBOL(bt_status); + void bt_info(const char *format, ...) { struct va_format vaf; diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index ae758ab1b558..8cfafd7a0576 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -1023,11 +1023,39 @@ static void rpa_expired(struct work_struct *work) hci_cmd_sync_queue(hdev, rpa_expired_sync, NULL, NULL); } +static void discov_off(struct work_struct *work) +{ + struct hci_dev *hdev = container_of(work, struct hci_dev, + discov_off.work); + + bt_dev_dbg(hdev, ""); + + hci_dev_lock(hdev); + + /* When discoverable timeout triggers, then just make sure + * the limited discoverable flag is cleared. Even in the case + * of a timeout triggered from general discoverable, it is + * safe to unconditionally clear the flag. + */ + hci_dev_clear_flag(hdev, HCI_LIMITED_DISCOVERABLE); + hci_dev_clear_flag(hdev, HCI_DISCOVERABLE); + hdev->discov_timeout = 0; + + hci_update_discoverable(hdev); + + mgmt_new_settings(hdev); + + hci_dev_unlock(hdev); +} + static void mgmt_init_hdev(struct sock *sk, struct hci_dev *hdev) { if (hci_dev_test_and_set_flag(hdev, HCI_MGMT)) return; + BT_INFO("MGMT ver %d.%d", MGMT_VERSION, MGMT_REVISION); + + INIT_DELAYED_WORK(&hdev->discov_off, discov_off); INIT_DELAYED_WORK(&hdev->service_cache, service_cache_off); INIT_DELAYED_WORK(&hdev->rpa_expired, rpa_expired); @@ -1082,11 +1110,11 @@ static u16 append_eir_data_to_buf(struct hci_dev *hdev, u8 *eir) eir_len = eir_append_le16(eir, eir_len, EIR_APPEARANCE, hdev->appearance); - name_len = strlen(hdev->dev_name); + name_len = strnlen(hdev->dev_name, sizeof(hdev->dev_name)); eir_len = eir_append_data(eir, eir_len, EIR_NAME_COMPLETE, hdev->dev_name, name_len); - name_len = strlen(hdev->short_name); + name_len = strnlen(hdev->short_name, sizeof(hdev->short_name)); eir_len = eir_append_data(eir, eir_len, EIR_NAME_SHORT, hdev->short_name, name_len); @@ -1611,7 +1639,7 @@ static int set_connectable_update_settings(struct hci_dev *hdev, return err; if (changed) { - hci_req_update_scan(hdev); + hci_update_scan(hdev); hci_update_passive_scan(hdev); return new_settings(hdev, sk); } @@ -2528,6 +2556,37 @@ static int device_unpaired(struct hci_dev *hdev, bdaddr_t *bdaddr, skip_sk); } +static void unpair_device_complete(struct hci_dev *hdev, void *data, int err) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_unpair_device *cp = cmd->param; + + if (!err) + device_unpaired(hdev, &cp->addr.bdaddr, cp->addr.type, cmd->sk); + + cmd->cmd_complete(cmd, err); + mgmt_pending_free(cmd); +} + +static int unpair_device_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_unpair_device *cp = cmd->param; + struct hci_conn *conn; + + if (cp->addr.type == BDADDR_BREDR) + conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, + &cp->addr.bdaddr); + else + conn = hci_conn_hash_lookup_le(hdev, &cp->addr.bdaddr, + le_addr_type(cp->addr.type)); + + if (!conn) + return 0; + + return hci_abort_conn_sync(hdev, conn, HCI_ERROR_REMOTE_USER_TERM); +} + static int unpair_device(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -2638,7 +2697,7 @@ done: goto unlock; } - cmd = mgmt_pending_add(sk, MGMT_OP_UNPAIR_DEVICE, hdev, cp, + cmd = mgmt_pending_new(sk, MGMT_OP_UNPAIR_DEVICE, hdev, cp, sizeof(*cp)); if (!cmd) { err = -ENOMEM; @@ -2647,9 +2706,10 @@ done: cmd->cmd_complete = addr_cmd_complete; - err = hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM); + err = hci_cmd_sync_queue(hdev, unpair_device_sync, cmd, + unpair_device_complete); if (err < 0) - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); unlock: hci_dev_unlock(hdev); @@ -3925,10 +3985,16 @@ static const u8 rpa_resolution_uuid[16] = { 0xea, 0x11, 0x73, 0xc2, 0x48, 0xa1, 0xc0, 0x15, }; +/* 6fbaf188-05e0-496a-9885-d6ddfdb4e03e */ +static const u8 iso_socket_uuid[16] = { + 0x3e, 0xe0, 0xb4, 0xfd, 0xdd, 0xd6, 0x85, 0x98, + 0x6a, 0x49, 0xe0, 0x05, 0x88, 0xf1, 0xba, 0x6f, +}; + static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev, void *data, u16 data_len) { - char buf[102]; /* Enough space for 5 features: 2 + 20 * 5 */ + char buf[122]; /* Enough space for 6 features: 2 + 20 * 6 */ struct mgmt_rp_read_exp_features_info *rp = (void *)buf; u16 idx = 0; u32 flags; @@ -3992,6 +4058,13 @@ static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev, idx++; } + if (IS_ENABLED(CONFIG_BT_LE)) { + flags = iso_enabled() ? BIT(0) : 0; + memcpy(rp->features[idx].uuid, iso_socket_uuid, 16); + rp->features[idx].flags = cpu_to_le32(flags); + idx++; + } + rp->feature_count = cpu_to_le16(idx); /* After reading the experimental features information, enable @@ -4384,6 +4457,57 @@ static int set_le_simultaneous_roles_func(struct sock *sk, struct hci_dev *hdev, return err; } +#ifdef CONFIG_BT_LE +static int set_iso_socket_func(struct sock *sk, struct hci_dev *hdev, + struct mgmt_cp_set_exp_feature *cp, u16 data_len) +{ + struct mgmt_rp_set_exp_feature rp; + bool val, changed = false; + int err; + + /* Command requires to use the non-controller index */ + if (hdev) + return mgmt_cmd_status(sk, hdev->id, + MGMT_OP_SET_EXP_FEATURE, + MGMT_STATUS_INVALID_INDEX); + + /* Parameters are limited to a single octet */ + if (data_len != MGMT_SET_EXP_FEATURE_SIZE + 1) + return mgmt_cmd_status(sk, MGMT_INDEX_NONE, + MGMT_OP_SET_EXP_FEATURE, + MGMT_STATUS_INVALID_PARAMS); + + /* Only boolean on/off is supported */ + if (cp->param[0] != 0x00 && cp->param[0] != 0x01) + return mgmt_cmd_status(sk, MGMT_INDEX_NONE, + MGMT_OP_SET_EXP_FEATURE, + MGMT_STATUS_INVALID_PARAMS); + + val = cp->param[0] ? true : false; + if (val) + err = iso_init(); + else + err = iso_exit(); + + if (!err) + changed = true; + + memcpy(rp.uuid, iso_socket_uuid, 16); + rp.flags = cpu_to_le32(val ? BIT(0) : 0); + + hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS); + + err = mgmt_cmd_complete(sk, MGMT_INDEX_NONE, + MGMT_OP_SET_EXP_FEATURE, 0, + &rp, sizeof(rp)); + + if (changed) + exp_feature_changed(hdev, iso_socket_uuid, val, sk); + + return err; +} +#endif + static const struct mgmt_exp_feature { const u8 *uuid; int (*set_func)(struct sock *sk, struct hci_dev *hdev, @@ -4397,6 +4521,9 @@ static const struct mgmt_exp_feature { EXP_FEAT(quality_report_uuid, set_quality_report_func), EXP_FEAT(offload_codecs_uuid, set_offload_codec_func), EXP_FEAT(le_simultaneous_roles_uuid, set_le_simultaneous_roles_func), +#ifdef CONFIG_BT_LE + EXP_FEAT(iso_socket_uuid, set_iso_socket_func), +#endif /* end with a null feature */ EXP_FEAT(NULL, NULL) @@ -4646,23 +4773,15 @@ static int read_adv_mon_features(struct sock *sk, struct hci_dev *hdev, return err; } -int mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, u8 status) +static void mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, + void *data, int status) { struct mgmt_rp_add_adv_patterns_monitor rp; - struct mgmt_pending_cmd *cmd; - struct adv_monitor *monitor; - int err = 0; + struct mgmt_pending_cmd *cmd = data; + struct adv_monitor *monitor = cmd->user_data; hci_dev_lock(hdev); - cmd = pending_find(MGMT_OP_ADD_ADV_PATTERNS_MONITOR_RSSI, hdev); - if (!cmd) { - cmd = pending_find(MGMT_OP_ADD_ADV_PATTERNS_MONITOR, hdev); - if (!cmd) - goto done; - } - - monitor = cmd->user_data; rp.monitor_handle = cpu_to_le16(monitor->handle); if (!status) { @@ -4673,26 +4792,29 @@ int mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, u8 status) hci_update_passive_scan(hdev); } - err = mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, - mgmt_status(status), &rp, sizeof(rp)); + mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, + mgmt_status(status), &rp, sizeof(rp)); mgmt_pending_remove(cmd); -done: hci_dev_unlock(hdev); - bt_dev_dbg(hdev, "add monitor %d complete, status %u", + bt_dev_dbg(hdev, "add monitor %d complete, status %d", rp.monitor_handle, status); +} - return err; +static int mgmt_add_adv_patterns_monitor_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct adv_monitor *monitor = cmd->user_data; + + return hci_add_adv_monitor(hdev, monitor); } static int __add_adv_patterns_monitor(struct sock *sk, struct hci_dev *hdev, struct adv_monitor *m, u8 status, void *data, u16 len, u16 op) { - struct mgmt_rp_add_adv_patterns_monitor rp; struct mgmt_pending_cmd *cmd; int err; - bool pending; hci_dev_lock(hdev); @@ -4714,12 +4836,11 @@ static int __add_adv_patterns_monitor(struct sock *sk, struct hci_dev *hdev, } cmd->user_data = m; - pending = hci_add_adv_monitor(hdev, m, &err); + err = hci_cmd_sync_queue(hdev, mgmt_add_adv_patterns_monitor_sync, cmd, + mgmt_add_adv_patterns_monitor_complete); if (err) { - if (err == -ENOSPC || err == -ENOMEM) + if (err == -ENOMEM) status = MGMT_STATUS_NO_RESOURCES; - else if (err == -EINVAL) - status = MGMT_STATUS_INVALID_PARAMS; else status = MGMT_STATUS_FAILED; @@ -4727,18 +4848,6 @@ static int __add_adv_patterns_monitor(struct sock *sk, struct hci_dev *hdev, goto unlock; } - if (!pending) { - mgmt_pending_remove(cmd); - rp.monitor_handle = cpu_to_le16(m->handle); - mgmt_adv_monitor_added(sk, hdev, m->handle); - m->state = ADV_MONITOR_STATE_REGISTERED; - hdev->adv_monitors_cnt++; - - hci_dev_unlock(hdev); - return mgmt_cmd_complete(sk, hdev->id, op, MGMT_STATUS_SUCCESS, - &rp, sizeof(rp)); - } - hci_dev_unlock(hdev); return 0; @@ -4879,49 +4988,46 @@ done: MGMT_OP_ADD_ADV_PATTERNS_MONITOR_RSSI); } -int mgmt_remove_adv_monitor_complete(struct hci_dev *hdev, u8 status) +static void mgmt_remove_adv_monitor_complete(struct hci_dev *hdev, + void *data, int status) { struct mgmt_rp_remove_adv_monitor rp; - struct mgmt_cp_remove_adv_monitor *cp; - struct mgmt_pending_cmd *cmd; - int err = 0; + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_remove_adv_monitor *cp = cmd->param; hci_dev_lock(hdev); - cmd = pending_find(MGMT_OP_REMOVE_ADV_MONITOR, hdev); - if (!cmd) - goto done; - - cp = cmd->param; rp.monitor_handle = cp->monitor_handle; if (!status) hci_update_passive_scan(hdev); - err = mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, - mgmt_status(status), &rp, sizeof(rp)); + mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, + mgmt_status(status), &rp, sizeof(rp)); mgmt_pending_remove(cmd); -done: hci_dev_unlock(hdev); - bt_dev_dbg(hdev, "remove monitor %d complete, status %u", + bt_dev_dbg(hdev, "remove monitor %d complete, status %d", rp.monitor_handle, status); +} - return err; +static int mgmt_remove_adv_monitor_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_remove_adv_monitor *cp = cmd->param; + u16 handle = __le16_to_cpu(cp->monitor_handle); + + if (!handle) + return hci_remove_all_adv_monitor(hdev); + + return hci_remove_single_adv_monitor(hdev, handle); } static int remove_adv_monitor(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { - struct mgmt_cp_remove_adv_monitor *cp = data; - struct mgmt_rp_remove_adv_monitor rp; struct mgmt_pending_cmd *cmd; - u16 handle = __le16_to_cpu(cp->monitor_handle); int err, status; - bool pending; - - BT_DBG("request for %s", hdev->name); - rp.monitor_handle = cp->monitor_handle; hci_dev_lock(hdev); @@ -4939,34 +5045,23 @@ static int remove_adv_monitor(struct sock *sk, struct hci_dev *hdev, goto unlock; } - if (handle) - pending = hci_remove_single_adv_monitor(hdev, handle, &err); - else - pending = hci_remove_all_adv_monitor(hdev, &err); + err = hci_cmd_sync_queue(hdev, mgmt_remove_adv_monitor_sync, cmd, + mgmt_remove_adv_monitor_complete); if (err) { mgmt_pending_remove(cmd); - if (err == -ENOENT) - status = MGMT_STATUS_INVALID_INDEX; + if (err == -ENOMEM) + status = MGMT_STATUS_NO_RESOURCES; else status = MGMT_STATUS_FAILED; - goto unlock; - } - - /* monitor can be removed without forwarding request to controller */ - if (!pending) { mgmt_pending_remove(cmd); - hci_dev_unlock(hdev); - - return mgmt_cmd_complete(sk, hdev->id, - MGMT_OP_REMOVE_ADV_MONITOR, - MGMT_STATUS_SUCCESS, - &rp, sizeof(rp)); + goto unlock; } hci_dev_unlock(hdev); + return 0; unlock: @@ -6711,11 +6806,6 @@ static void get_conn_info_complete(struct hci_dev *hdev, void *data, int err) mgmt_cmd_complete(cmd->sk, cmd->index, MGMT_OP_GET_CONN_INFO, status, &rp, sizeof(rp)); - if (conn) { - hci_conn_drop(conn); - hci_conn_put(conn); - } - mgmt_pending_free(cmd); } @@ -6734,15 +6824,10 @@ static int get_conn_info_sync(struct hci_dev *hdev, void *data) else conn = hci_conn_hash_lookup_ba(hdev, LE_LINK, &cp->addr.bdaddr); - if (!conn || conn != cmd->user_data || conn->state != BT_CONNECTED) { - if (cmd->user_data) { - hci_conn_drop(cmd->user_data); - hci_conn_put(cmd->user_data); - cmd->user_data = NULL; - } + if (!conn || conn->state != BT_CONNECTED) return MGMT_STATUS_NOT_CONNECTED; - } + cmd->user_data = conn; handle = cpu_to_le16(conn->handle); /* Refresh RSSI each time */ @@ -6821,11 +6906,12 @@ static int get_conn_info(struct sock *sk, struct hci_dev *hdev, void *data, cmd = mgmt_pending_new(sk, MGMT_OP_GET_CONN_INFO, hdev, data, len); - if (!cmd) + if (!cmd) { err = -ENOMEM; - else + } else { err = hci_cmd_sync_queue(hdev, get_conn_info_sync, cmd, get_conn_info_complete); + } if (err < 0) { mgmt_cmd_complete(sk, hdev->id, MGMT_OP_GET_CONN_INFO, @@ -6837,9 +6923,6 @@ static int get_conn_info(struct sock *sk, struct hci_dev *hdev, void *data, goto unlock; } - hci_conn_hold(conn); - cmd->user_data = hci_conn_get(conn); - conn->conn_info_timestamp = jiffies; } else { /* Cache is valid, just reply with values cached in hci_conn */ @@ -6878,8 +6961,6 @@ static void get_clock_info_complete(struct hci_dev *hdev, void *data, int err) if (conn) { rp.piconet_clock = cpu_to_le32(conn->clock); rp.accuracy = cpu_to_le16(conn->clock_accuracy); - hci_conn_drop(conn); - hci_conn_put(conn); } complete: @@ -6894,30 +6975,21 @@ static int get_clock_info_sync(struct hci_dev *hdev, void *data) struct mgmt_pending_cmd *cmd = data; struct mgmt_cp_get_clock_info *cp = cmd->param; struct hci_cp_read_clock hci_cp; - struct hci_conn *conn = cmd->user_data; - int err; + struct hci_conn *conn; memset(&hci_cp, 0, sizeof(hci_cp)); - err = hci_read_clock_sync(hdev, &hci_cp); + hci_read_clock_sync(hdev, &hci_cp); - if (conn) { - /* Make sure connection still exists */ - conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, - &cp->addr.bdaddr); + /* Make sure connection still exists */ + conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &cp->addr.bdaddr); + if (!conn || conn->state != BT_CONNECTED) + return MGMT_STATUS_NOT_CONNECTED; - if (conn && conn == cmd->user_data && - conn->state == BT_CONNECTED) { - hci_cp.handle = cpu_to_le16(conn->handle); - hci_cp.which = 0x01; /* Piconet clock */ - err = hci_read_clock_sync(hdev, &hci_cp); - } else if (cmd->user_data) { - hci_conn_drop(cmd->user_data); - hci_conn_put(cmd->user_data); - cmd->user_data = NULL; - } - } + cmd->user_data = conn; + hci_cp.handle = cpu_to_le16(conn->handle); + hci_cp.which = 0x01; /* Piconet clock */ - return err; + return hci_read_clock_sync(hdev, &hci_cp); } static int get_clock_info(struct sock *sk, struct hci_dev *hdev, void *data, @@ -6976,10 +7048,6 @@ static int get_clock_info(struct sock *sk, struct hci_dev *hdev, void *data, if (cmd) mgmt_pending_free(cmd); - - } else if (conn) { - hci_conn_hold(conn); - cmd->user_data = hci_conn_get(conn); } @@ -7108,7 +7176,7 @@ static int add_device(struct sock *sk, struct hci_dev *hdev, if (err) goto unlock; - hci_req_update_scan(hdev); + hci_update_scan(hdev); goto added; } @@ -7220,7 +7288,7 @@ static int remove_device(struct sock *sk, struct hci_dev *hdev, goto unlock; } - hci_req_update_scan(hdev); + hci_update_scan(hdev); device_removed(sk, hdev, &cp->addr.bdaddr, cp->addr.type); @@ -7284,7 +7352,7 @@ static int remove_device(struct sock *sk, struct hci_dev *hdev, kfree(b); } - hci_req_update_scan(hdev); + hci_update_scan(hdev); list_for_each_entry_safe(p, tmp, &hdev->le_conn_params, list) { if (p->auto_connect == HCI_AUTO_CONN_DISABLED) @@ -8090,7 +8158,7 @@ static int add_advertising(struct sock *sk, struct hci_dev *hdev, u16 timeout, duration; unsigned int prev_instance_cnt; u8 schedule_instance = 0; - struct adv_info *next_instance; + struct adv_info *adv, *next_instance; int err; struct mgmt_pending_cmd *cmd; @@ -8141,7 +8209,7 @@ static int add_advertising(struct sock *sk, struct hci_dev *hdev, prev_instance_cnt = hdev->adv_instance_cnt; - err = hci_add_adv_instance(hdev, cp->instance, flags, + adv = hci_add_adv_instance(hdev, cp->instance, flags, cp->adv_data_len, cp->data, cp->scan_rsp_len, cp->data + cp->adv_data_len, @@ -8149,7 +8217,7 @@ static int add_advertising(struct sock *sk, struct hci_dev *hdev, HCI_ADV_TX_POWER_NO_PREFERENCE, hdev->le_adv_min_interval, hdev->le_adv_max_interval); - if (err < 0) { + if (IS_ERR(adv)) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_ADVERTISING, MGMT_STATUS_FAILED); goto unlock; @@ -8280,6 +8348,7 @@ static int add_ext_adv_params(struct sock *sk, struct hci_dev *hdev, struct mgmt_cp_add_ext_adv_params *cp = data; struct mgmt_rp_add_ext_adv_params rp; struct mgmt_pending_cmd *cmd = NULL; + struct adv_info *adv; u32 flags, min_interval, max_interval; u16 timeout, duration; u8 status; @@ -8349,11 +8418,11 @@ static int add_ext_adv_params(struct sock *sk, struct hci_dev *hdev, HCI_ADV_TX_POWER_NO_PREFERENCE; /* Create advertising instance with no advertising or response data */ - err = hci_add_adv_instance(hdev, cp->instance, flags, - 0, NULL, 0, NULL, timeout, duration, - tx_power, min_interval, max_interval); + adv = hci_add_adv_instance(hdev, cp->instance, flags, 0, NULL, 0, NULL, + timeout, duration, tx_power, min_interval, + max_interval); - if (err < 0) { + if (IS_ERR(adv)) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_EXT_ADV_PARAMS, MGMT_STATUS_FAILED); goto unlock; @@ -8866,6 +8935,11 @@ void mgmt_index_removed(struct hci_dev *hdev) mgmt_index_event(MGMT_EV_EXT_INDEX_REMOVED, hdev, &ev, sizeof(ev), HCI_MGMT_EXT_INDEX_EVENTS); + + /* Cancel any remaining timed work */ + cancel_delayed_work_sync(&hdev->discov_off); + cancel_delayed_work_sync(&hdev->service_cache); + cancel_delayed_work_sync(&hdev->rpa_expired); } void mgmt_power_on(struct hci_dev *hdev, int err) diff --git a/net/bluetooth/msft.c b/net/bluetooth/msft.c index f43994523b1f..14975769f678 100644 --- a/net/bluetooth/msft.c +++ b/net/bluetooth/msft.c @@ -99,18 +99,11 @@ struct msft_data { __u8 evt_prefix_len; __u8 *evt_prefix; struct list_head handle_map; - __u16 pending_add_handle; - __u16 pending_remove_handle; __u8 resuming; __u8 suspending; __u8 filter_enabled; }; -static int __msft_add_monitor_pattern(struct hci_dev *hdev, - struct adv_monitor *monitor); -static int __msft_remove_monitor(struct hci_dev *hdev, - struct adv_monitor *monitor, u16 handle); - bool msft_monitor_supported(struct hci_dev *hdev) { return !!(msft_get_features(hdev) & MSFT_FEATURE_MASK_LE_ADV_MONITOR); @@ -164,34 +157,6 @@ failed: return false; } -static void reregister_monitor(struct hci_dev *hdev, int handle) -{ - struct adv_monitor *monitor; - struct msft_data *msft = hdev->msft_data; - int err; - - while (1) { - monitor = idr_get_next(&hdev->adv_monitors_idr, &handle); - if (!monitor) { - /* All monitors have been resumed */ - msft->resuming = false; - hci_update_passive_scan(hdev); - return; - } - - msft->pending_add_handle = (u16)handle; - err = __msft_add_monitor_pattern(hdev, monitor); - - /* If success, we return and wait for monitor added callback */ - if (!err) - return; - - /* Otherwise remove the monitor and keep registering */ - hci_free_adv_monitor(hdev, monitor); - handle++; - } -} - /* is_mgmt = true matches the handle exposed to userspace via mgmt. * is_mgmt = false matches the handle used by the msft controller. * This function requires the caller holds hdev->lock @@ -243,34 +208,27 @@ static int msft_monitor_device_del(struct hci_dev *hdev, __u16 mgmt_handle, return count; } -static void msft_le_monitor_advertisement_cb(struct hci_dev *hdev, - u8 status, u16 opcode, - struct sk_buff *skb) +static int msft_le_monitor_advertisement_cb(struct hci_dev *hdev, u16 opcode, + struct adv_monitor *monitor, + struct sk_buff *skb) { struct msft_rp_le_monitor_advertisement *rp; - struct adv_monitor *monitor; struct msft_monitor_advertisement_handle_data *handle_data; struct msft_data *msft = hdev->msft_data; + int status = 0; hci_dev_lock(hdev); - monitor = idr_find(&hdev->adv_monitors_idr, msft->pending_add_handle); - if (!monitor) { - bt_dev_err(hdev, "msft add advmon: monitor %u is not found!", - msft->pending_add_handle); + rp = (struct msft_rp_le_monitor_advertisement *)skb->data; + if (skb->len < sizeof(*rp)) { status = HCI_ERROR_UNSPECIFIED; goto unlock; } + status = rp->status; if (status) goto unlock; - rp = (struct msft_rp_le_monitor_advertisement *)skb->data; - if (skb->len < sizeof(*rp)) { - status = HCI_ERROR_UNSPECIFIED; - goto unlock; - } - handle_data = kmalloc(sizeof(*handle_data), GFP_KERNEL); if (!handle_data) { status = HCI_ERROR_UNSPECIFIED; @@ -285,29 +243,23 @@ static void msft_le_monitor_advertisement_cb(struct hci_dev *hdev, monitor->state = ADV_MONITOR_STATE_OFFLOADED; unlock: - if (status && monitor) + if (status) hci_free_adv_monitor(hdev, monitor); hci_dev_unlock(hdev); - if (!msft->resuming) - hci_add_adv_patterns_monitor_complete(hdev, status); + return status; } -static void msft_le_cancel_monitor_advertisement_cb(struct hci_dev *hdev, - u8 status, u16 opcode, - struct sk_buff *skb) +static int msft_le_cancel_monitor_advertisement_cb(struct hci_dev *hdev, + u16 opcode, + struct adv_monitor *monitor, + struct sk_buff *skb) { - struct msft_cp_le_cancel_monitor_advertisement *cp; struct msft_rp_le_cancel_monitor_advertisement *rp; - struct adv_monitor *monitor; struct msft_monitor_advertisement_handle_data *handle_data; struct msft_data *msft = hdev->msft_data; - int err; - bool pending; - - if (status) - goto done; + int status = 0; rp = (struct msft_rp_le_cancel_monitor_advertisement *)skb->data; if (skb->len < sizeof(*rp)) { @@ -315,22 +267,22 @@ static void msft_le_cancel_monitor_advertisement_cb(struct hci_dev *hdev, goto done; } + status = rp->status; + if (status) + goto done; + hci_dev_lock(hdev); - cp = hci_sent_cmd_data(hdev, hdev->msft_opcode); - handle_data = msft_find_handle_data(hdev, cp->handle, false); + handle_data = msft_find_handle_data(hdev, monitor->handle, true); if (handle_data) { - monitor = idr_find(&hdev->adv_monitors_idr, - handle_data->mgmt_handle); - - if (monitor && monitor->state == ADV_MONITOR_STATE_OFFLOADED) + if (monitor->state == ADV_MONITOR_STATE_OFFLOADED) monitor->state = ADV_MONITOR_STATE_REGISTERED; /* Do not free the monitor if it is being removed due to * suspend. It will be re-monitored on resume. */ - if (monitor && !msft->suspending) { + if (!msft->suspending) { hci_free_adv_monitor(hdev, monitor); /* Clear any monitored devices by this Adv Monitor */ @@ -342,35 +294,19 @@ static void msft_le_cancel_monitor_advertisement_cb(struct hci_dev *hdev, kfree(handle_data); } - /* If remove all monitors is required, we need to continue the process - * here because the earlier it was paused when waiting for the - * response from controller. - */ - if (msft->pending_remove_handle == 0) { - pending = hci_remove_all_adv_monitor(hdev, &err); - if (pending) { - hci_dev_unlock(hdev); - return; - } - - if (err) - status = HCI_ERROR_UNSPECIFIED; - } - hci_dev_unlock(hdev); done: - if (!msft->suspending) - hci_remove_adv_monitor_complete(hdev, status); + return status; } +/* This function requires the caller holds hci_req_sync_lock */ static int msft_remove_monitor_sync(struct hci_dev *hdev, struct adv_monitor *monitor) { struct msft_cp_le_cancel_monitor_advertisement cp; struct msft_monitor_advertisement_handle_data *handle_data; struct sk_buff *skb; - u8 status; handle_data = msft_find_handle_data(hdev, monitor->handle, true); @@ -386,13 +322,8 @@ static int msft_remove_monitor_sync(struct hci_dev *hdev, if (IS_ERR(skb)) return PTR_ERR(skb); - status = skb->data[0]; - skb_pull(skb, 1); - - msft_le_cancel_monitor_advertisement_cb(hdev, status, hdev->msft_opcode, - skb); - - return status; + return msft_le_cancel_monitor_advertisement_cb(hdev, hdev->msft_opcode, + monitor, skb); } /* This function requires the caller holds hci_req_sync_lock */ @@ -463,7 +394,6 @@ static int msft_add_monitor_sync(struct hci_dev *hdev, ptrdiff_t offset = 0; u8 pattern_count = 0; struct sk_buff *skb; - u8 status; if (!msft_monitor_pattern_valid(monitor)) return -EINVAL; @@ -505,20 +435,40 @@ static int msft_add_monitor_sync(struct hci_dev *hdev, if (IS_ERR(skb)) return PTR_ERR(skb); - status = skb->data[0]; - skb_pull(skb, 1); + return msft_le_monitor_advertisement_cb(hdev, hdev->msft_opcode, + monitor, skb); +} - msft_le_monitor_advertisement_cb(hdev, status, hdev->msft_opcode, skb); +/* This function requires the caller holds hci_req_sync_lock */ +static void reregister_monitor(struct hci_dev *hdev) +{ + struct adv_monitor *monitor; + struct msft_data *msft = hdev->msft_data; + int handle = 0; - return status; + if (!msft) + return; + + msft->resuming = true; + + while (1) { + monitor = idr_get_next(&hdev->adv_monitors_idr, &handle); + if (!monitor) + break; + + msft_add_monitor_sync(hdev, monitor); + + handle++; + } + + /* All monitors have been reregistered */ + msft->resuming = false; } /* This function requires the caller holds hci_req_sync_lock */ int msft_resume_sync(struct hci_dev *hdev) { struct msft_data *msft = hdev->msft_data; - struct adv_monitor *monitor; - int handle = 0; if (!msft || !msft_monitor_supported(hdev)) return 0; @@ -533,24 +483,12 @@ int msft_resume_sync(struct hci_dev *hdev) hci_dev_unlock(hdev); - msft->resuming = true; - - while (1) { - monitor = idr_get_next(&hdev->adv_monitors_idr, &handle); - if (!monitor) - break; - - msft_add_monitor_sync(hdev, monitor); - - handle++; - } - - /* All monitors have been resumed */ - msft->resuming = false; + reregister_monitor(hdev); return 0; } +/* This function requires the caller holds hci_req_sync_lock */ void msft_do_open(struct hci_dev *hdev) { struct msft_data *msft = hdev->msft_data; @@ -583,7 +521,7 @@ void msft_do_open(struct hci_dev *hdev) /* Monitors get removed on power off, so we need to explicitly * tell the controller to re-monitor. */ - reregister_monitor(hdev, 0); + reregister_monitor(hdev); } } @@ -829,66 +767,7 @@ static void msft_le_set_advertisement_filter_enable_cb(struct hci_dev *hdev, hci_dev_unlock(hdev); } -/* This function requires the caller holds hdev->lock */ -static int __msft_add_monitor_pattern(struct hci_dev *hdev, - struct adv_monitor *monitor) -{ - struct msft_cp_le_monitor_advertisement *cp; - struct msft_le_monitor_advertisement_pattern_data *pattern_data; - struct msft_le_monitor_advertisement_pattern *pattern; - struct adv_pattern *entry; - struct hci_request req; - struct msft_data *msft = hdev->msft_data; - size_t total_size = sizeof(*cp) + sizeof(*pattern_data); - ptrdiff_t offset = 0; - u8 pattern_count = 0; - int err = 0; - - if (!msft_monitor_pattern_valid(monitor)) - return -EINVAL; - - list_for_each_entry(entry, &monitor->patterns, list) { - pattern_count++; - total_size += sizeof(*pattern) + entry->length; - } - - cp = kmalloc(total_size, GFP_KERNEL); - if (!cp) - return -ENOMEM; - - cp->sub_opcode = MSFT_OP_LE_MONITOR_ADVERTISEMENT; - cp->rssi_high = monitor->rssi.high_threshold; - cp->rssi_low = monitor->rssi.low_threshold; - cp->rssi_low_interval = (u8)monitor->rssi.low_threshold_timeout; - cp->rssi_sampling_period = monitor->rssi.sampling_period; - - cp->cond_type = MSFT_MONITOR_ADVERTISEMENT_TYPE_PATTERN; - - pattern_data = (void *)cp->data; - pattern_data->count = pattern_count; - - list_for_each_entry(entry, &monitor->patterns, list) { - pattern = (void *)(pattern_data->data + offset); - /* the length also includes data_type and offset */ - pattern->length = entry->length + 2; - pattern->data_type = entry->ad_type; - pattern->start_byte = entry->offset; - memcpy(pattern->pattern, entry->value, entry->length); - offset += sizeof(*pattern) + entry->length; - } - - hci_req_init(&req, hdev); - hci_req_add(&req, hdev->msft_opcode, total_size, cp); - err = hci_req_run_skb(&req, msft_le_monitor_advertisement_cb); - kfree(cp); - - if (!err) - msft->pending_add_handle = monitor->handle; - - return err; -} - -/* This function requires the caller holds hdev->lock */ +/* This function requires the caller holds hci_req_sync_lock */ int msft_add_monitor_pattern(struct hci_dev *hdev, struct adv_monitor *monitor) { struct msft_data *msft = hdev->msft_data; @@ -899,41 +778,11 @@ int msft_add_monitor_pattern(struct hci_dev *hdev, struct adv_monitor *monitor) if (msft->resuming || msft->suspending) return -EBUSY; - return __msft_add_monitor_pattern(hdev, monitor); + return msft_add_monitor_sync(hdev, monitor); } -/* This function requires the caller holds hdev->lock */ -static int __msft_remove_monitor(struct hci_dev *hdev, - struct adv_monitor *monitor, u16 handle) -{ - struct msft_cp_le_cancel_monitor_advertisement cp; - struct msft_monitor_advertisement_handle_data *handle_data; - struct hci_request req; - struct msft_data *msft = hdev->msft_data; - int err = 0; - - handle_data = msft_find_handle_data(hdev, monitor->handle, true); - - /* If no matched handle, just remove without telling controller */ - if (!handle_data) - return -ENOENT; - - cp.sub_opcode = MSFT_OP_LE_CANCEL_MONITOR_ADVERTISEMENT; - cp.handle = handle_data->msft_handle; - - hci_req_init(&req, hdev); - hci_req_add(&req, hdev->msft_opcode, sizeof(cp), &cp); - err = hci_req_run_skb(&req, msft_le_cancel_monitor_advertisement_cb); - - if (!err) - msft->pending_remove_handle = handle; - - return err; -} - -/* This function requires the caller holds hdev->lock */ -int msft_remove_monitor(struct hci_dev *hdev, struct adv_monitor *monitor, - u16 handle) +/* This function requires the caller holds hci_req_sync_lock */ +int msft_remove_monitor(struct hci_dev *hdev, struct adv_monitor *monitor) { struct msft_data *msft = hdev->msft_data; @@ -943,7 +792,7 @@ int msft_remove_monitor(struct hci_dev *hdev, struct adv_monitor *monitor, if (msft->resuming || msft->suspending) return -EBUSY; - return __msft_remove_monitor(hdev, monitor, handle); + return msft_remove_monitor_sync(hdev, monitor); } void msft_req_add_set_filter_enable(struct hci_request *req, bool enable) diff --git a/net/bluetooth/msft.h b/net/bluetooth/msft.h index afcaf7d3b1cb..2a63205b377b 100644 --- a/net/bluetooth/msft.h +++ b/net/bluetooth/msft.h @@ -20,8 +20,7 @@ void msft_do_close(struct hci_dev *hdev); void msft_vendor_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb); __u64 msft_get_features(struct hci_dev *hdev); int msft_add_monitor_pattern(struct hci_dev *hdev, struct adv_monitor *monitor); -int msft_remove_monitor(struct hci_dev *hdev, struct adv_monitor *monitor, - u16 handle); +int msft_remove_monitor(struct hci_dev *hdev, struct adv_monitor *monitor); void msft_req_add_set_filter_enable(struct hci_request *req, bool enable); int msft_set_filter_enable(struct hci_dev *hdev, bool enable); int msft_suspend_sync(struct hci_dev *hdev); @@ -49,8 +48,7 @@ static inline int msft_add_monitor_pattern(struct hci_dev *hdev, } static inline int msft_remove_monitor(struct hci_dev *hdev, - struct adv_monitor *monitor, - u16 handle) + struct adv_monitor *monitor) { return -EOPNOTSUPP; } diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 2ca96acbc50a..cbc9cd5058cb 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -691,52 +691,35 @@ noinline void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len) { } +noinline void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p) +{ +} + __diag_pop(); ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); -BTF_SET_START(test_sk_check_kfunc_ids) -BTF_ID(func, bpf_kfunc_call_test1) -BTF_ID(func, bpf_kfunc_call_test2) -BTF_ID(func, bpf_kfunc_call_test3) -BTF_ID(func, bpf_kfunc_call_test_acquire) -BTF_ID(func, bpf_kfunc_call_memb_acquire) -BTF_ID(func, bpf_kfunc_call_test_release) -BTF_ID(func, bpf_kfunc_call_memb_release) -BTF_ID(func, bpf_kfunc_call_memb1_release) -BTF_ID(func, bpf_kfunc_call_test_kptr_get) -BTF_ID(func, bpf_kfunc_call_test_pass_ctx) -BTF_ID(func, bpf_kfunc_call_test_pass1) -BTF_ID(func, bpf_kfunc_call_test_pass2) -BTF_ID(func, bpf_kfunc_call_test_fail1) -BTF_ID(func, bpf_kfunc_call_test_fail2) -BTF_ID(func, bpf_kfunc_call_test_fail3) -BTF_ID(func, bpf_kfunc_call_test_mem_len_pass1) -BTF_ID(func, bpf_kfunc_call_test_mem_len_fail1) -BTF_ID(func, bpf_kfunc_call_test_mem_len_fail2) -BTF_SET_END(test_sk_check_kfunc_ids) - -BTF_SET_START(test_sk_acquire_kfunc_ids) -BTF_ID(func, bpf_kfunc_call_test_acquire) -BTF_ID(func, bpf_kfunc_call_memb_acquire) -BTF_ID(func, bpf_kfunc_call_test_kptr_get) -BTF_SET_END(test_sk_acquire_kfunc_ids) - -BTF_SET_START(test_sk_release_kfunc_ids) -BTF_ID(func, bpf_kfunc_call_test_release) -BTF_ID(func, bpf_kfunc_call_memb_release) -BTF_ID(func, bpf_kfunc_call_memb1_release) -BTF_SET_END(test_sk_release_kfunc_ids) - -BTF_SET_START(test_sk_ret_null_kfunc_ids) -BTF_ID(func, bpf_kfunc_call_test_acquire) -BTF_ID(func, bpf_kfunc_call_memb_acquire) -BTF_ID(func, bpf_kfunc_call_test_kptr_get) -BTF_SET_END(test_sk_ret_null_kfunc_ids) - -BTF_SET_START(test_sk_kptr_acquire_kfunc_ids) -BTF_ID(func, bpf_kfunc_call_test_kptr_get) -BTF_SET_END(test_sk_kptr_acquire_kfunc_ids) +BTF_SET8_START(test_sk_check_kfunc_ids) +BTF_ID_FLAGS(func, bpf_kfunc_call_test1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test2) +BTF_ID_FLAGS(func, bpf_kfunc_call_test3) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_acquire, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_kfunc_call_memb_acquire, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE) +BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE) +BTF_ID_FLAGS(func, bpf_kfunc_call_memb1_release, KF_RELEASE) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_kptr_get, KF_ACQUIRE | KF_RET_NULL | KF_KPTR_GET) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass_ctx) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail2) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail1) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail2) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ref, KF_TRUSTED_ARGS) +BTF_SET8_END(test_sk_check_kfunc_ids) static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, u32 size, u32 headroom, u32 tailroom) @@ -955,6 +938,9 @@ static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) { struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; + if (!skb->len) + return -EINVAL; + if (!__skb) return 0; @@ -1617,12 +1603,8 @@ out: } static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &test_sk_check_kfunc_ids, - .acquire_set = &test_sk_acquire_kfunc_ids, - .release_set = &test_sk_release_kfunc_ids, - .ret_null_set = &test_sk_ret_null_kfunc_ids, - .kptr_acquire_set = &test_sk_kptr_acquire_kfunc_ids + .owner = THIS_MODULE, + .set = &test_sk_check_kfunc_ids, }; BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids) diff --git a/net/bridge/netfilter/nft_meta_bridge.c b/net/bridge/netfilter/nft_meta_bridge.c index 8c3eaba87ad2..c3ecd77e25cb 100644 --- a/net/bridge/netfilter/nft_meta_bridge.c +++ b/net/bridge/netfilter/nft_meta_bridge.c @@ -53,7 +53,7 @@ static void nft_meta_bridge_get_eval(const struct nft_expr *expr, goto err; br_vlan_get_proto(br_dev, &p_proto); - nft_reg_store16(dest, htons(p_proto)); + nft_reg_store_be16(dest, htons(p_proto)); return; } default: diff --git a/net/compat.c b/net/compat.c index 210fc3b4d0d8..6cd2e7683dd0 100644 --- a/net/compat.c +++ b/net/compat.c @@ -80,6 +80,7 @@ int __get_compat_msghdr(struct msghdr *kmsg, return -EMSGSIZE; kmsg->msg_iocb = NULL; + kmsg->msg_ubuf = NULL; *ptr = msg.msg_iov; *len = msg.msg_iovlen; return 0; diff --git a/net/core/datagram.c b/net/core/datagram.c index 35791f86bd1a..f3988ef8e9af 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -610,10 +610,16 @@ fault: } EXPORT_SYMBOL(skb_copy_datagram_from_iter); -int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb, - struct iov_iter *from, size_t length) +int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, + struct sk_buff *skb, struct iov_iter *from, + size_t length) { - int frag = skb_shinfo(skb)->nr_frags; + int frag; + + if (msg && msg->msg_ubuf && msg->sg_from_iter) + return msg->sg_from_iter(sk, skb, from, length); + + frag = skb_shinfo(skb)->nr_frags; while (length && iov_iter_count(from)) { struct page *pages[MAX_SKB_FRAGS]; @@ -699,7 +705,7 @@ int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *from) if (skb_copy_datagram_from_iter(skb, 0, from, copy)) return -EFAULT; - return __zerocopy_sg_from_iter(NULL, skb, from, ~0U); + return __zerocopy_sg_from_iter(NULL, NULL, skb, from, ~0U); } EXPORT_SYMBOL(zerocopy_sg_from_iter); diff --git a/net/core/dev.c b/net/core/dev.c index d588fd0a54ce..716df64fcfa5 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4168,6 +4168,7 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) bool again = false; skb_reset_mac_header(skb); + skb_assert_len(skb); if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED); diff --git a/net/core/devlink.c b/net/core/devlink.c index a9776ea923ae..98d79feeb3dc 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -66,6 +66,7 @@ struct devlink { * port, sb, dpipe, resource, params, region, traps and more. */ struct mutex lock; + struct lock_class_key lock_key; u8 reload_failed:1; refcount_t refcount; struct completion comp; @@ -265,6 +266,12 @@ void devl_lock(struct devlink *devlink) } EXPORT_SYMBOL_GPL(devl_lock); +int devl_trylock(struct devlink *devlink) +{ + return mutex_trylock(&devlink->lock); +} +EXPORT_SYMBOL_GPL(devl_trylock); + void devl_unlock(struct devlink *devlink) { mutex_unlock(&devlink->lock); @@ -9472,7 +9479,9 @@ struct devlink *devlink_alloc_ns(const struct devlink_ops *ops, INIT_LIST_HEAD(&devlink->trap_list); INIT_LIST_HEAD(&devlink->trap_group_list); INIT_LIST_HEAD(&devlink->trap_policer_list); + lockdep_register_key(&devlink->lock_key); mutex_init(&devlink->lock); + lockdep_set_class(&devlink->lock, &devlink->lock_key); mutex_init(&devlink->reporters_lock); mutex_init(&devlink->linecards_lock); refcount_set(&devlink->refcount, 1); @@ -9619,6 +9628,7 @@ void devlink_free(struct devlink *devlink) mutex_destroy(&devlink->linecards_lock); mutex_destroy(&devlink->reporters_lock); mutex_destroy(&devlink->lock); + lockdep_unregister_key(&devlink->lock_key); WARN_ON(!list_empty(&devlink->trap_policer_list)); WARN_ON(!list_empty(&devlink->trap_group_list)); WARN_ON(!list_empty(&devlink->trap_list)); @@ -10371,25 +10381,21 @@ void devlink_linecard_deactivate(struct devlink_linecard *linecard) } EXPORT_SYMBOL_GPL(devlink_linecard_deactivate); -int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, - u32 size, u16 ingress_pools_count, - u16 egress_pools_count, u16 ingress_tc_count, - u16 egress_tc_count) +int devl_sb_register(struct devlink *devlink, unsigned int sb_index, + u32 size, u16 ingress_pools_count, + u16 egress_pools_count, u16 ingress_tc_count, + u16 egress_tc_count) { struct devlink_sb *devlink_sb; - int err = 0; - devl_lock(devlink); - if (devlink_sb_index_exists(devlink, sb_index)) { - err = -EEXIST; - goto unlock; - } + lockdep_assert_held(&devlink->lock); + + if (devlink_sb_index_exists(devlink, sb_index)) + return -EEXIST; devlink_sb = kzalloc(sizeof(*devlink_sb), GFP_KERNEL); - if (!devlink_sb) { - err = -ENOMEM; - goto unlock; - } + if (!devlink_sb) + return -ENOMEM; devlink_sb->index = sb_index; devlink_sb->size = size; devlink_sb->ingress_pools_count = ingress_pools_count; @@ -10397,56 +10403,78 @@ int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, devlink_sb->ingress_tc_count = ingress_tc_count; devlink_sb->egress_tc_count = egress_tc_count; list_add_tail(&devlink_sb->list, &devlink->sb_list); -unlock: + return 0; +} +EXPORT_SYMBOL_GPL(devl_sb_register); + +int devlink_sb_register(struct devlink *devlink, unsigned int sb_index, + u32 size, u16 ingress_pools_count, + u16 egress_pools_count, u16 ingress_tc_count, + u16 egress_tc_count) +{ + int err; + + devl_lock(devlink); + err = devl_sb_register(devlink, sb_index, size, ingress_pools_count, + egress_pools_count, ingress_tc_count, + egress_tc_count); devl_unlock(devlink); return err; } EXPORT_SYMBOL_GPL(devlink_sb_register); -void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index) +void devl_sb_unregister(struct devlink *devlink, unsigned int sb_index) { struct devlink_sb *devlink_sb; - devl_lock(devlink); + lockdep_assert_held(&devlink->lock); + devlink_sb = devlink_sb_get_by_index(devlink, sb_index); WARN_ON(!devlink_sb); list_del(&devlink_sb->list); - devl_unlock(devlink); kfree(devlink_sb); } +EXPORT_SYMBOL_GPL(devl_sb_unregister); + +void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index) +{ + devl_lock(devlink); + devl_sb_unregister(devlink, sb_index); + devl_unlock(devlink); +} EXPORT_SYMBOL_GPL(devlink_sb_unregister); /** - * devlink_dpipe_headers_register - register dpipe headers + * devl_dpipe_headers_register - register dpipe headers * - * @devlink: devlink - * @dpipe_headers: dpipe header array + * @devlink: devlink + * @dpipe_headers: dpipe header array * - * Register the headers supported by hardware. + * Register the headers supported by hardware. */ -void devlink_dpipe_headers_register(struct devlink *devlink, - struct devlink_dpipe_headers *dpipe_headers) +void devl_dpipe_headers_register(struct devlink *devlink, + struct devlink_dpipe_headers *dpipe_headers) { - devl_lock(devlink); + lockdep_assert_held(&devlink->lock); + devlink->dpipe_headers = dpipe_headers; - devl_unlock(devlink); } -EXPORT_SYMBOL_GPL(devlink_dpipe_headers_register); +EXPORT_SYMBOL_GPL(devl_dpipe_headers_register); /** - * devlink_dpipe_headers_unregister - unregister dpipe headers + * devl_dpipe_headers_unregister - unregister dpipe headers * - * @devlink: devlink + * @devlink: devlink * - * Unregister the headers supported by hardware. + * Unregister the headers supported by hardware. */ -void devlink_dpipe_headers_unregister(struct devlink *devlink) +void devl_dpipe_headers_unregister(struct devlink *devlink) { - devl_lock(devlink); + lockdep_assert_held(&devlink->lock); + devlink->dpipe_headers = NULL; - devl_unlock(devlink); } -EXPORT_SYMBOL_GPL(devlink_dpipe_headers_unregister); +EXPORT_SYMBOL_GPL(devl_dpipe_headers_unregister); /** * devlink_dpipe_table_counter_enabled - check if counter allocation @@ -10480,38 +10508,33 @@ bool devlink_dpipe_table_counter_enabled(struct devlink *devlink, EXPORT_SYMBOL_GPL(devlink_dpipe_table_counter_enabled); /** - * devlink_dpipe_table_register - register dpipe table + * devl_dpipe_table_register - register dpipe table * - * @devlink: devlink - * @table_name: table name - * @table_ops: table ops - * @priv: priv - * @counter_control_extern: external control for counters + * @devlink: devlink + * @table_name: table name + * @table_ops: table ops + * @priv: priv + * @counter_control_extern: external control for counters */ -int devlink_dpipe_table_register(struct devlink *devlink, - const char *table_name, - struct devlink_dpipe_table_ops *table_ops, - void *priv, bool counter_control_extern) +int devl_dpipe_table_register(struct devlink *devlink, + const char *table_name, + struct devlink_dpipe_table_ops *table_ops, + void *priv, bool counter_control_extern) { struct devlink_dpipe_table *table; - int err = 0; + + lockdep_assert_held(&devlink->lock); if (WARN_ON(!table_ops->size_get)) return -EINVAL; - devl_lock(devlink); - if (devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name, - devlink)) { - err = -EEXIST; - goto unlock; - } + devlink)) + return -EEXIST; table = kzalloc(sizeof(*table), GFP_KERNEL); - if (!table) { - err = -ENOMEM; - goto unlock; - } + if (!table) + return -ENOMEM; table->name = table_name; table->table_ops = table_ops; @@ -10519,77 +10542,69 @@ int devlink_dpipe_table_register(struct devlink *devlink, table->counter_control_extern = counter_control_extern; list_add_tail_rcu(&table->list, &devlink->dpipe_table_list); -unlock: - devl_unlock(devlink); - return err; + + return 0; } -EXPORT_SYMBOL_GPL(devlink_dpipe_table_register); +EXPORT_SYMBOL_GPL(devl_dpipe_table_register); /** - * devlink_dpipe_table_unregister - unregister dpipe table + * devl_dpipe_table_unregister - unregister dpipe table * - * @devlink: devlink - * @table_name: table name + * @devlink: devlink + * @table_name: table name */ -void devlink_dpipe_table_unregister(struct devlink *devlink, - const char *table_name) +void devl_dpipe_table_unregister(struct devlink *devlink, + const char *table_name) { struct devlink_dpipe_table *table; - devl_lock(devlink); + lockdep_assert_held(&devlink->lock); + table = devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name, devlink); if (!table) - goto unlock; + return; list_del_rcu(&table->list); - devl_unlock(devlink); kfree_rcu(table, rcu); - return; -unlock: - devl_unlock(devlink); } -EXPORT_SYMBOL_GPL(devlink_dpipe_table_unregister); +EXPORT_SYMBOL_GPL(devl_dpipe_table_unregister); /** - * devlink_resource_register - devlink resource register + * devl_resource_register - devlink resource register * - * @devlink: devlink - * @resource_name: resource's name - * @resource_size: resource's size - * @resource_id: resource's id - * @parent_resource_id: resource's parent id - * @size_params: size parameters + * @devlink: devlink + * @resource_name: resource's name + * @resource_size: resource's size + * @resource_id: resource's id + * @parent_resource_id: resource's parent id + * @size_params: size parameters * - * Generic resources should reuse the same names across drivers. - * Please see the generic resources list at: - * Documentation/networking/devlink/devlink-resource.rst + * Generic resources should reuse the same names across drivers. + * Please see the generic resources list at: + * Documentation/networking/devlink/devlink-resource.rst */ -int devlink_resource_register(struct devlink *devlink, - const char *resource_name, - u64 resource_size, - u64 resource_id, - u64 parent_resource_id, - const struct devlink_resource_size_params *size_params) +int devl_resource_register(struct devlink *devlink, + const char *resource_name, + u64 resource_size, + u64 resource_id, + u64 parent_resource_id, + const struct devlink_resource_size_params *size_params) { struct devlink_resource *resource; struct list_head *resource_list; bool top_hierarchy; - int err = 0; + + lockdep_assert_held(&devlink->lock); top_hierarchy = parent_resource_id == DEVLINK_RESOURCE_ID_PARENT_TOP; - devl_lock(devlink); resource = devlink_resource_find(devlink, NULL, resource_id); - if (resource) { - err = -EINVAL; - goto out; - } + if (resource) + return -EINVAL; resource = kzalloc(sizeof(*resource), GFP_KERNEL); - if (!resource) { - err = -ENOMEM; - goto out; - } + if (!resource) + return -ENOMEM; if (top_hierarchy) { resource_list = &devlink->resource_list; @@ -10603,8 +10618,7 @@ int devlink_resource_register(struct devlink *devlink, resource->parent = parent_resource; } else { kfree(resource); - err = -EINVAL; - goto out; + return -EINVAL; } } @@ -10617,7 +10631,39 @@ int devlink_resource_register(struct devlink *devlink, sizeof(resource->size_params)); INIT_LIST_HEAD(&resource->resource_list); list_add_tail(&resource->list, resource_list); -out: + + return 0; +} +EXPORT_SYMBOL_GPL(devl_resource_register); + +/** + * devlink_resource_register - devlink resource register + * + * @devlink: devlink + * @resource_name: resource's name + * @resource_size: resource's size + * @resource_id: resource's id + * @parent_resource_id: resource's parent id + * @size_params: size parameters + * + * Generic resources should reuse the same names across drivers. + * Please see the generic resources list at: + * Documentation/networking/devlink/devlink-resource.rst + * + * Context: Takes and release devlink->lock <mutex>. + */ +int devlink_resource_register(struct devlink *devlink, + const char *resource_name, + u64 resource_size, + u64 resource_id, + u64 parent_resource_id, + const struct devlink_resource_size_params *size_params) +{ + int err; + + devl_lock(devlink); + err = devl_resource_register(devlink, resource_name, resource_size, + resource_id, parent_resource_id, size_params); devl_unlock(devlink); return err; } @@ -10637,15 +10683,15 @@ static void devlink_resource_unregister(struct devlink *devlink, } /** - * devlink_resources_unregister - free all resources + * devl_resources_unregister - free all resources * - * @devlink: devlink + * @devlink: devlink */ -void devlink_resources_unregister(struct devlink *devlink) +void devl_resources_unregister(struct devlink *devlink) { struct devlink_resource *tmp, *child_resource; - devl_lock(devlink); + lockdep_assert_held(&devlink->lock); list_for_each_entry_safe(child_resource, tmp, &devlink->resource_list, list) { @@ -10653,69 +10699,100 @@ void devlink_resources_unregister(struct devlink *devlink) list_del(&child_resource->list); kfree(child_resource); } +} +EXPORT_SYMBOL_GPL(devl_resources_unregister); +/** + * devlink_resources_unregister - free all resources + * + * @devlink: devlink + * + * Context: Takes and release devlink->lock <mutex>. + */ +void devlink_resources_unregister(struct devlink *devlink) +{ + devl_lock(devlink); + devl_resources_unregister(devlink); devl_unlock(devlink); } EXPORT_SYMBOL_GPL(devlink_resources_unregister); /** - * devlink_resource_size_get - get and update size + * devl_resource_size_get - get and update size * - * @devlink: devlink - * @resource_id: the requested resource id - * @p_resource_size: ptr to update + * @devlink: devlink + * @resource_id: the requested resource id + * @p_resource_size: ptr to update */ -int devlink_resource_size_get(struct devlink *devlink, - u64 resource_id, - u64 *p_resource_size) +int devl_resource_size_get(struct devlink *devlink, + u64 resource_id, + u64 *p_resource_size) { struct devlink_resource *resource; - int err = 0; - devl_lock(devlink); + lockdep_assert_held(&devlink->lock); + resource = devlink_resource_find(devlink, NULL, resource_id); - if (!resource) { - err = -EINVAL; - goto out; - } + if (!resource) + return -EINVAL; *p_resource_size = resource->size_new; resource->size = resource->size_new; -out: - devl_unlock(devlink); - return err; + return 0; } -EXPORT_SYMBOL_GPL(devlink_resource_size_get); +EXPORT_SYMBOL_GPL(devl_resource_size_get); /** - * devlink_dpipe_table_resource_set - set the resource id + * devl_dpipe_table_resource_set - set the resource id * - * @devlink: devlink - * @table_name: table name - * @resource_id: resource id - * @resource_units: number of resource's units consumed per table's entry + * @devlink: devlink + * @table_name: table name + * @resource_id: resource id + * @resource_units: number of resource's units consumed per table's entry */ -int devlink_dpipe_table_resource_set(struct devlink *devlink, - const char *table_name, u64 resource_id, - u64 resource_units) +int devl_dpipe_table_resource_set(struct devlink *devlink, + const char *table_name, u64 resource_id, + u64 resource_units) { struct devlink_dpipe_table *table; - int err = 0; - devl_lock(devlink); table = devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name, devlink); - if (!table) { - err = -EINVAL; - goto out; - } + if (!table) + return -EINVAL; + table->resource_id = resource_id; table->resource_units = resource_units; table->resource_valid = true; -out: - devl_unlock(devlink); - return err; + return 0; +} +EXPORT_SYMBOL_GPL(devl_dpipe_table_resource_set); + +/** + * devl_resource_occ_get_register - register occupancy getter + * + * @devlink: devlink + * @resource_id: resource id + * @occ_get: occupancy getter callback + * @occ_get_priv: occupancy getter callback priv + */ +void devl_resource_occ_get_register(struct devlink *devlink, + u64 resource_id, + devlink_resource_occ_get_t *occ_get, + void *occ_get_priv) +{ + struct devlink_resource *resource; + + lockdep_assert_held(&devlink->lock); + + resource = devlink_resource_find(devlink, NULL, resource_id); + if (WARN_ON(!resource)) + return; + WARN_ON(resource->occ_get); + + resource->occ_get = occ_get; + resource->occ_get_priv = occ_get_priv; } -EXPORT_SYMBOL_GPL(devlink_dpipe_table_resource_set); +EXPORT_SYMBOL_GPL(devl_resource_occ_get_register); /** * devlink_resource_occ_get_register - register occupancy getter @@ -10724,47 +10801,57 @@ EXPORT_SYMBOL_GPL(devlink_dpipe_table_resource_set); * @resource_id: resource id * @occ_get: occupancy getter callback * @occ_get_priv: occupancy getter callback priv + * + * Context: Takes and release devlink->lock <mutex>. */ void devlink_resource_occ_get_register(struct devlink *devlink, u64 resource_id, devlink_resource_occ_get_t *occ_get, void *occ_get_priv) { - struct devlink_resource *resource; - devl_lock(devlink); - resource = devlink_resource_find(devlink, NULL, resource_id); - if (WARN_ON(!resource)) - goto out; - WARN_ON(resource->occ_get); - - resource->occ_get = occ_get; - resource->occ_get_priv = occ_get_priv; -out: + devl_resource_occ_get_register(devlink, resource_id, + occ_get, occ_get_priv); devl_unlock(devlink); } EXPORT_SYMBOL_GPL(devlink_resource_occ_get_register); /** - * devlink_resource_occ_get_unregister - unregister occupancy getter + * devl_resource_occ_get_unregister - unregister occupancy getter * - * @devlink: devlink - * @resource_id: resource id + * @devlink: devlink + * @resource_id: resource id */ -void devlink_resource_occ_get_unregister(struct devlink *devlink, - u64 resource_id) +void devl_resource_occ_get_unregister(struct devlink *devlink, + u64 resource_id) { struct devlink_resource *resource; - devl_lock(devlink); + lockdep_assert_held(&devlink->lock); + resource = devlink_resource_find(devlink, NULL, resource_id); if (WARN_ON(!resource)) - goto out; + return; WARN_ON(!resource->occ_get); resource->occ_get = NULL; resource->occ_get_priv = NULL; -out: +} +EXPORT_SYMBOL_GPL(devl_resource_occ_get_unregister); + +/** + * devlink_resource_occ_get_unregister - unregister occupancy getter + * + * @devlink: devlink + * @resource_id: resource id + * + * Context: Takes and release devlink->lock <mutex>. + */ +void devlink_resource_occ_get_unregister(struct devlink *devlink, + u64 resource_id) +{ + devl_lock(devlink); + devl_resource_occ_get_unregister(devlink, resource_id); devl_unlock(devlink); } EXPORT_SYMBOL_GPL(devlink_resource_occ_get_unregister); @@ -10986,36 +11073,31 @@ void devlink_param_value_changed(struct devlink *devlink, u32 param_id) EXPORT_SYMBOL_GPL(devlink_param_value_changed); /** - * devlink_region_create - create a new address region + * devl_region_create - create a new address region * - * @devlink: devlink - * @ops: region operations and name - * @region_max_snapshots: Maximum supported number of snapshots for region - * @region_size: size of region + * @devlink: devlink + * @ops: region operations and name + * @region_max_snapshots: Maximum supported number of snapshots for region + * @region_size: size of region */ -struct devlink_region * -devlink_region_create(struct devlink *devlink, - const struct devlink_region_ops *ops, - u32 region_max_snapshots, u64 region_size) +struct devlink_region *devl_region_create(struct devlink *devlink, + const struct devlink_region_ops *ops, + u32 region_max_snapshots, + u64 region_size) { struct devlink_region *region; - int err = 0; + + devl_assert_locked(devlink); if (WARN_ON(!ops) || WARN_ON(!ops->destructor)) return ERR_PTR(-EINVAL); - devl_lock(devlink); - - if (devlink_region_get_by_name(devlink, ops->name)) { - err = -EEXIST; - goto unlock; - } + if (devlink_region_get_by_name(devlink, ops->name)) + return ERR_PTR(-EEXIST); region = kzalloc(sizeof(*region), GFP_KERNEL); - if (!region) { - err = -ENOMEM; - goto unlock; - } + if (!region) + return ERR_PTR(-ENOMEM); region->devlink = devlink; region->max_snapshots = region_max_snapshots; @@ -11025,12 +11107,32 @@ devlink_region_create(struct devlink *devlink, list_add_tail(®ion->list, &devlink->region_list); devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW); - devl_unlock(devlink); return region; +} +EXPORT_SYMBOL_GPL(devl_region_create); -unlock: +/** + * devlink_region_create - create a new address region + * + * @devlink: devlink + * @ops: region operations and name + * @region_max_snapshots: Maximum supported number of snapshots for region + * @region_size: size of region + * + * Context: Takes and release devlink->lock <mutex>. + */ +struct devlink_region * +devlink_region_create(struct devlink *devlink, + const struct devlink_region_ops *ops, + u32 region_max_snapshots, u64 region_size) +{ + struct devlink_region *region; + + devl_lock(devlink); + region = devl_region_create(devlink, ops, region_max_snapshots, + region_size); devl_unlock(devlink); - return ERR_PTR(err); + return region; } EXPORT_SYMBOL_GPL(devlink_region_create); @@ -11041,6 +11143,8 @@ EXPORT_SYMBOL_GPL(devlink_region_create); * @ops: region operations and name * @region_max_snapshots: Maximum supported number of snapshots for region * @region_size: size of region + * + * Context: Takes and release devlink->lock <mutex>. */ struct devlink_region * devlink_port_region_create(struct devlink_port *port, @@ -11086,16 +11190,16 @@ unlock: EXPORT_SYMBOL_GPL(devlink_port_region_create); /** - * devlink_region_destroy - destroy address region + * devl_region_destroy - destroy address region * - * @region: devlink region to destroy + * @region: devlink region to destroy */ -void devlink_region_destroy(struct devlink_region *region) +void devl_region_destroy(struct devlink_region *region) { struct devlink *devlink = region->devlink; struct devlink_snapshot *snapshot, *ts; - devl_lock(devlink); + devl_assert_locked(devlink); /* Free all snapshots of region */ list_for_each_entry_safe(snapshot, ts, ®ion->snapshot_list, list) @@ -11104,9 +11208,25 @@ void devlink_region_destroy(struct devlink_region *region) list_del(®ion->list); devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL); - devl_unlock(devlink); kfree(region); } +EXPORT_SYMBOL_GPL(devl_region_destroy); + +/** + * devlink_region_destroy - destroy address region + * + * @region: devlink region to destroy + * + * Context: Takes and release devlink->lock <mutex>. + */ +void devlink_region_destroy(struct devlink_region *region) +{ + struct devlink *devlink = region->devlink; + + devl_lock(devlink); + devl_region_destroy(region); + devl_unlock(devlink); +} EXPORT_SYMBOL_GPL(devlink_region_destroy); /** @@ -11540,7 +11660,7 @@ static void devlink_trap_disable(struct devlink *devlink, } /** - * devlink_traps_register - Register packet traps with devlink. + * devl_traps_register - Register packet traps with devlink. * @devlink: devlink. * @traps: Packet traps. * @traps_count: Count of provided packet traps. @@ -11548,16 +11668,16 @@ static void devlink_trap_disable(struct devlink *devlink, * * Return: Non-zero value on failure. */ -int devlink_traps_register(struct devlink *devlink, - const struct devlink_trap *traps, - size_t traps_count, void *priv) +int devl_traps_register(struct devlink *devlink, + const struct devlink_trap *traps, + size_t traps_count, void *priv) { int i, err; if (!devlink->ops->trap_init || !devlink->ops->trap_action_set) return -EINVAL; - devl_lock(devlink); + devl_assert_locked(devlink); for (i = 0; i < traps_count; i++) { const struct devlink_trap *trap = &traps[i]; @@ -11569,7 +11689,6 @@ int devlink_traps_register(struct devlink *devlink, if (err) goto err_trap_register; } - devl_unlock(devlink); return 0; @@ -11577,24 +11696,47 @@ err_trap_register: err_trap_verify: for (i--; i >= 0; i--) devlink_trap_unregister(devlink, &traps[i]); + return err; +} +EXPORT_SYMBOL_GPL(devl_traps_register); + +/** + * devlink_traps_register - Register packet traps with devlink. + * @devlink: devlink. + * @traps: Packet traps. + * @traps_count: Count of provided packet traps. + * @priv: Driver private information. + * + * Context: Takes and release devlink->lock <mutex>. + * + * Return: Non-zero value on failure. + */ +int devlink_traps_register(struct devlink *devlink, + const struct devlink_trap *traps, + size_t traps_count, void *priv) +{ + int err; + + devl_lock(devlink); + err = devl_traps_register(devlink, traps, traps_count, priv); devl_unlock(devlink); return err; } EXPORT_SYMBOL_GPL(devlink_traps_register); /** - * devlink_traps_unregister - Unregister packet traps from devlink. + * devl_traps_unregister - Unregister packet traps from devlink. * @devlink: devlink. * @traps: Packet traps. * @traps_count: Count of provided packet traps. */ -void devlink_traps_unregister(struct devlink *devlink, - const struct devlink_trap *traps, - size_t traps_count) +void devl_traps_unregister(struct devlink *devlink, + const struct devlink_trap *traps, + size_t traps_count) { int i; - devl_lock(devlink); + devl_assert_locked(devlink); /* Make sure we do not have any packets in-flight while unregistering * traps by disabling all of them and waiting for a grace period. */ @@ -11603,6 +11745,23 @@ void devlink_traps_unregister(struct devlink *devlink, synchronize_rcu(); for (i = traps_count - 1; i >= 0; i--) devlink_trap_unregister(devlink, &traps[i]); +} +EXPORT_SYMBOL_GPL(devl_traps_unregister); + +/** + * devlink_traps_unregister - Unregister packet traps from devlink. + * @devlink: devlink. + * @traps: Packet traps. + * @traps_count: Count of provided packet traps. + * + * Context: Takes and release devlink->lock <mutex>. + */ +void devlink_traps_unregister(struct devlink *devlink, + const struct devlink_trap *traps, + size_t traps_count) +{ + devl_lock(devlink); + devl_traps_unregister(devlink, traps, traps_count); devl_unlock(devlink); } EXPORT_SYMBOL_GPL(devlink_traps_unregister); @@ -11762,20 +11921,20 @@ devlink_trap_group_unregister(struct devlink *devlink, } /** - * devlink_trap_groups_register - Register packet trap groups with devlink. + * devl_trap_groups_register - Register packet trap groups with devlink. * @devlink: devlink. * @groups: Packet trap groups. * @groups_count: Count of provided packet trap groups. * * Return: Non-zero value on failure. */ -int devlink_trap_groups_register(struct devlink *devlink, - const struct devlink_trap_group *groups, - size_t groups_count) +int devl_trap_groups_register(struct devlink *devlink, + const struct devlink_trap_group *groups, + size_t groups_count) { int i, err; - devl_lock(devlink); + devl_assert_locked(devlink); for (i = 0; i < groups_count; i++) { const struct devlink_trap_group *group = &groups[i]; @@ -11787,7 +11946,6 @@ int devlink_trap_groups_register(struct devlink *devlink, if (err) goto err_trap_group_register; } - devl_unlock(devlink); return 0; @@ -11795,26 +11953,65 @@ err_trap_group_register: err_trap_group_verify: for (i--; i >= 0; i--) devlink_trap_group_unregister(devlink, &groups[i]); + return err; +} +EXPORT_SYMBOL_GPL(devl_trap_groups_register); + +/** + * devlink_trap_groups_register - Register packet trap groups with devlink. + * @devlink: devlink. + * @groups: Packet trap groups. + * @groups_count: Count of provided packet trap groups. + * + * Context: Takes and release devlink->lock <mutex>. + * + * Return: Non-zero value on failure. + */ +int devlink_trap_groups_register(struct devlink *devlink, + const struct devlink_trap_group *groups, + size_t groups_count) +{ + int err; + + devl_lock(devlink); + err = devl_trap_groups_register(devlink, groups, groups_count); devl_unlock(devlink); return err; } EXPORT_SYMBOL_GPL(devlink_trap_groups_register); /** + * devl_trap_groups_unregister - Unregister packet trap groups from devlink. + * @devlink: devlink. + * @groups: Packet trap groups. + * @groups_count: Count of provided packet trap groups. + */ +void devl_trap_groups_unregister(struct devlink *devlink, + const struct devlink_trap_group *groups, + size_t groups_count) +{ + int i; + + devl_assert_locked(devlink); + for (i = groups_count - 1; i >= 0; i--) + devlink_trap_group_unregister(devlink, &groups[i]); +} +EXPORT_SYMBOL_GPL(devl_trap_groups_unregister); + +/** * devlink_trap_groups_unregister - Unregister packet trap groups from devlink. * @devlink: devlink. * @groups: Packet trap groups. * @groups_count: Count of provided packet trap groups. + * + * Context: Takes and release devlink->lock <mutex>. */ void devlink_trap_groups_unregister(struct devlink *devlink, const struct devlink_trap_group *groups, size_t groups_count) { - int i; - devl_lock(devlink); - for (i = groups_count - 1; i >= 0; i--) - devlink_trap_group_unregister(devlink, &groups[i]); + devl_trap_groups_unregister(devlink, groups, groups_count); devl_unlock(devlink); } EXPORT_SYMBOL_GPL(devlink_trap_groups_unregister); @@ -11901,7 +12098,7 @@ devlink_trap_policer_unregister(struct devlink *devlink, } /** - * devlink_trap_policers_register - Register packet trap policers with devlink. + * devl_trap_policers_register - Register packet trap policers with devlink. * @devlink: devlink. * @policers: Packet trap policers. * @policers_count: Count of provided packet trap policers. @@ -11909,13 +12106,13 @@ devlink_trap_policer_unregister(struct devlink *devlink, * Return: Non-zero value on failure. */ int -devlink_trap_policers_register(struct devlink *devlink, - const struct devlink_trap_policer *policers, - size_t policers_count) +devl_trap_policers_register(struct devlink *devlink, + const struct devlink_trap_policer *policers, + size_t policers_count) { int i, err; - devl_lock(devlink); + devl_assert_locked(devlink); for (i = 0; i < policers_count; i++) { const struct devlink_trap_policer *policer = &policers[i]; @@ -11930,38 +12127,34 @@ devlink_trap_policers_register(struct devlink *devlink, if (err) goto err_trap_policer_register; } - devl_unlock(devlink); - return 0; err_trap_policer_register: err_trap_policer_verify: for (i--; i >= 0; i--) devlink_trap_policer_unregister(devlink, &policers[i]); - devl_unlock(devlink); return err; } -EXPORT_SYMBOL_GPL(devlink_trap_policers_register); +EXPORT_SYMBOL_GPL(devl_trap_policers_register); /** - * devlink_trap_policers_unregister - Unregister packet trap policers from devlink. + * devl_trap_policers_unregister - Unregister packet trap policers from devlink. * @devlink: devlink. * @policers: Packet trap policers. * @policers_count: Count of provided packet trap policers. */ void -devlink_trap_policers_unregister(struct devlink *devlink, - const struct devlink_trap_policer *policers, - size_t policers_count) +devl_trap_policers_unregister(struct devlink *devlink, + const struct devlink_trap_policer *policers, + size_t policers_count) { int i; - devl_lock(devlink); + devl_assert_locked(devlink); for (i = policers_count - 1; i >= 0; i--) devlink_trap_policer_unregister(devlink, &policers[i]); - devl_unlock(devlink); } -EXPORT_SYMBOL_GPL(devlink_trap_policers_unregister); +EXPORT_SYMBOL_GPL(devl_trap_policers_unregister); static void __devlink_compat_running_version(struct devlink *devlink, char *buf, size_t len) diff --git a/net/core/filter.c b/net/core/filter.c index 59aada9c4e19..57c5e4c4efd2 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -237,7 +237,7 @@ BPF_CALL_2(bpf_skb_load_helper_8_no_cache, const struct sk_buff *, skb, BPF_CALL_4(bpf_skb_load_helper_16, const struct sk_buff *, skb, const void *, data, int, headlen, int, offset) { - u16 tmp, *ptr; + __be16 tmp, *ptr; const int len = sizeof(tmp); if (offset >= 0) { @@ -264,7 +264,7 @@ BPF_CALL_2(bpf_skb_load_helper_16_no_cache, const struct sk_buff *, skb, BPF_CALL_4(bpf_skb_load_helper_32, const struct sk_buff *, skb, const void *, data, int, headlen, int, offset) { - u32 tmp, *ptr; + __be32 tmp, *ptr; const int len = sizeof(tmp); if (likely(offset >= 0)) { @@ -7084,7 +7084,7 @@ BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN) return -EINVAL; - if (!sock_net(sk)->ipv4.sysctl_tcp_syncookies) + if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies)) return -EINVAL; if (!th->ack || th->rst || th->syn) @@ -7159,7 +7159,7 @@ BPF_CALL_5(bpf_tcp_gen_syncookie, struct sock *, sk, void *, iph, u32, iph_len, if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN) return -EINVAL; - if (!sock_net(sk)->ipv4.sysctl_tcp_syncookies) + if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies)) return -ENOENT; if (!th->syn || th->ack || th->fin || th->rst) diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c index 5f85e01d4093..b0ff6153be62 100644 --- a/net/core/secure_seq.c +++ b/net/core/secure_seq.c @@ -64,7 +64,7 @@ u32 secure_tcpv6_ts_off(const struct net *net, .daddr = *(struct in6_addr *)daddr, }; - if (net->ipv4.sysctl_tcp_timestamps != 1) + if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) return 0; ts_secret_init(); @@ -120,7 +120,7 @@ EXPORT_SYMBOL(secure_ipv6_port_ephemeral); #ifdef CONFIG_INET u32 secure_tcp_ts_off(const struct net *net, __be32 saddr, __be32 daddr) { - if (net->ipv4.sysctl_tcp_timestamps != 1) + if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) return 0; ts_secret_init(); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index c4a751781581..974bbbbe7138 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -669,11 +669,18 @@ static void skb_release_data(struct sk_buff *skb) &shinfo->dataref)) goto exit; - skb_zcopy_clear(skb, true); + if (skb_zcopy(skb)) { + bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS; + + skb_zcopy_clear(skb, true); + if (skip_unref) + goto free_head; + } for (i = 0; i < shinfo->nr_frags; i++) __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle); +free_head: if (shinfo->frag_list) kfree_skb_list(shinfo->frag_list); @@ -898,7 +905,10 @@ EXPORT_SYMBOL(skb_dump); */ void skb_tx_error(struct sk_buff *skb) { - skb_zcopy_clear(skb, true); + if (skb) { + skb_zcopy_downgrade_managed(skb); + skb_zcopy_clear(skb, true); + } } EXPORT_SYMBOL(skb_tx_error); @@ -1196,7 +1206,7 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) uarg->len = 1; uarg->bytelen = size; uarg->zerocopy = 1; - uarg->flags = SKBFL_ZEROCOPY_FRAG; + uarg->flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; refcount_set(&uarg->refcnt, 1); sock_hold(sk); @@ -1215,6 +1225,10 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, const u32 byte_limit = 1 << 19; /* limit to a few TSO */ u32 bytelen, next; + /* there might be non MSG_ZEROCOPY users */ + if (uarg->callback != msg_zerocopy_callback) + return NULL; + /* realloc only when socket is locked (TCP, UDP cork), * so uarg->len and sk_zckey access is serialized */ @@ -1357,7 +1371,7 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, if (orig_uarg && uarg != orig_uarg) return -EEXIST; - err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len); + err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len); if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { struct sock *save_sk = skb->sk; @@ -1374,6 +1388,16 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, } EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); +void __skb_zcopy_downgrade_managed(struct sk_buff *skb) +{ + int i; + + skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS; + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) + skb_frag_ref(skb, i); +} +EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed); + static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, gfp_t gfp_mask) { @@ -1692,6 +1716,8 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, BUG_ON(skb_shared(skb)); + skb_zcopy_downgrade_managed(skb); + size = SKB_DATA_ALIGN(size); if (skb_pfmemalloc(skb)) @@ -3486,6 +3512,8 @@ void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) int pos = skb_headlen(skb); const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY; + skb_zcopy_downgrade_managed(skb); + skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags; skb_zerocopy_clone(skb1, skb, 0); if (len < pos) /* Split line is inside header. */ @@ -3834,6 +3862,7 @@ int skb_append_pagefrags(struct sk_buff *skb, struct page *page, if (skb_can_coalesce(skb, i, page, offset)) { skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); } else if (i < MAX_SKB_FRAGS) { + skb_zcopy_downgrade_managed(skb); get_page(page); skb_fill_page_desc(skb, i, page, offset, size); } else { diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 266d3b7b7d0b..81627892bdd4 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -462,7 +462,7 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, if (copied == len) break; - } while (i != msg_rx->sg.end); + } while (!sg_is_last(sge)); if (unlikely(peek)) { msg_rx = sk_psock_next_msg(psock, msg_rx); @@ -472,7 +472,7 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, } msg_rx->sg.start = i; - if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) { + if (!sge->length && sg_is_last(sge)) { msg_rx = sk_psock_dequeue_msg(psock); kfree_sk_msg(msg_rx); } diff --git a/net/core/sock_reuseport.c b/net/core/sock_reuseport.c index 3f00a28fe762..5daa1fa54249 100644 --- a/net/core/sock_reuseport.c +++ b/net/core/sock_reuseport.c @@ -387,7 +387,7 @@ void reuseport_stop_listen_sock(struct sock *sk) prog = rcu_dereference_protected(reuse->prog, lockdep_is_held(&reuseport_lock)); - if (sock_net(sk)->ipv4.sysctl_tcp_migrate_req || + if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_migrate_req) || (prog && prog->expected_attach_type == BPF_SK_REUSEPORT_SELECT_OR_MIGRATE)) { /* Migration capable, move sk from the listening section * to the closed section. @@ -545,7 +545,7 @@ struct sock *reuseport_migrate_sock(struct sock *sk, hash = migrating_sk->sk_hash; prog = rcu_dereference(reuse->prog); if (!prog || prog->expected_attach_type != BPF_SK_REUSEPORT_SELECT_OR_MIGRATE) { - if (sock_net(sk)->ipv4.sysctl_tcp_migrate_req) + if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_migrate_req)) goto select_by_hash; goto failure; } diff --git a/net/dsa/port.c b/net/dsa/port.c index 3738f2d40a0b..2dd76eb1621c 100644 --- a/net/dsa/port.c +++ b/net/dsa/port.c @@ -248,6 +248,7 @@ static void dsa_port_reset_vlan_filtering(struct dsa_port *dp, struct netlink_ext_ack extack = {0}; bool change_vlan_filtering = false; struct dsa_switch *ds = dp->ds; + struct dsa_port *other_dp; bool vlan_filtering; int err; @@ -270,8 +271,8 @@ static void dsa_port_reset_vlan_filtering(struct dsa_port *dp, * VLAN-aware bridge. */ if (change_vlan_filtering && ds->vlan_filtering_is_global) { - dsa_switch_for_each_port(dp, ds) { - struct net_device *br = dsa_port_bridge_dev_get(dp); + dsa_switch_for_each_port(other_dp, ds) { + struct net_device *br = dsa_port_bridge_dev_get(other_dp); if (br && br_vlan_enabled(br)) { change_vlan_filtering = false; @@ -799,7 +800,7 @@ int dsa_port_vlan_filtering(struct dsa_port *dp, bool vlan_filtering, ds->vlan_filtering = vlan_filtering; dsa_switch_for_each_user_port(other_dp, ds) { - struct net_device *slave = dp->slave; + struct net_device *slave = other_dp->slave; /* We might be called in the unbind path, so not * all slave devices might still be registered. diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index fac552e98eb9..3ca0cc467886 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -217,7 +217,7 @@ int inet_listen(struct socket *sock, int backlog) * because the socket was in TCP_LISTEN state previously but * was shutdown() rather than close(). */ - tcp_fastopen = sock_net(sk)->ipv4.sysctl_tcp_fastopen; + tcp_fastopen = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen); if ((tcp_fastopen & TFO_SERVER_WO_SOCKOPT1) && (tcp_fastopen & TFO_SERVER_ENABLE) && !inet_csk(sk)->icsk_accept_queue.fastopenq.max_qlen) { @@ -335,7 +335,7 @@ lookup_protocol: inet->hdrincl = 1; } - if (net->ipv4.sysctl_ip_no_pmtu_disc) + if (READ_ONCE(net->ipv4.sysctl_ip_no_pmtu_disc)) inet->pmtudisc = IP_PMTUDISC_DONT; else inet->pmtudisc = IP_PMTUDISC_WANT; @@ -1711,24 +1711,14 @@ static const struct net_protocol igmp_protocol = { }; #endif -/* thinking of making this const? Don't. - * early_demux can change based on sysctl. - */ -static struct net_protocol tcp_protocol = { - .early_demux = tcp_v4_early_demux, - .early_demux_handler = tcp_v4_early_demux, +static const struct net_protocol tcp_protocol = { .handler = tcp_v4_rcv, .err_handler = tcp_v4_err, .no_policy = 1, .icmp_strict_tag_validation = 1, }; -/* thinking of making this const? Don't. - * early_demux can change based on sysctl. - */ -static struct net_protocol udp_protocol = { - .early_demux = udp_v4_early_demux, - .early_demux_handler = udp_v4_early_demux, +static const struct net_protocol udp_protocol = { .handler = udp_rcv, .err_handler = udp_err, .no_policy = 1, diff --git a/net/ipv4/ah4.c b/net/ipv4/ah4.c index 6eea1e9e998d..f8ad04470d3a 100644 --- a/net/ipv4/ah4.c +++ b/net/ipv4/ah4.c @@ -507,7 +507,7 @@ static int ah_init_state(struct xfrm_state *x) if (aalg_desc->uinfo.auth.icv_fullbits/8 != crypto_ahash_digestsize(ahash)) { - pr_info("%s: %s digestsize %u != %hu\n", + pr_info("%s: %s digestsize %u != %u\n", __func__, x->aalg->alg_name, crypto_ahash_digestsize(ahash), aalg_desc->uinfo.auth.icv_fullbits / 8); diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c index af2f12ffc9ca..87c7e3fc5197 100644 --- a/net/ipv4/arp.c +++ b/net/ipv4/arp.c @@ -429,6 +429,26 @@ static int arp_ignore(struct in_device *in_dev, __be32 sip, __be32 tip) return !inet_confirm_addr(net, in_dev, sip, tip, scope); } +static int arp_accept(struct in_device *in_dev, __be32 sip) +{ + struct net *net = dev_net(in_dev->dev); + int scope = RT_SCOPE_LINK; + + switch (IN_DEV_ARP_ACCEPT(in_dev)) { + case 0: /* Don't create new entries from garp */ + return 0; + case 1: /* Create new entries from garp */ + return 1; + case 2: /* Create a neighbor in the arp table only if sip + * is in the same subnet as an address configured + * on the interface that received the garp message + */ + return !!inet_confirm_addr(net, in_dev, sip, 0, scope); + default: + return 0; + } +} + static int arp_filter(__be32 sip, __be32 tip, struct net_device *dev) { struct rtable *rt; @@ -868,12 +888,12 @@ static int arp_process(struct net *net, struct sock *sk, struct sk_buff *skb) n = __neigh_lookup(&arp_tbl, &sip, dev, 0); addr_type = -1; - if (n || IN_DEV_ARP_ACCEPT(in_dev)) { + if (n || arp_accept(in_dev, sip)) { is_garp = arp_is_garp(net, dev, &addr_type, arp->ar_op, sip, tip, sha, tha); } - if (IN_DEV_ARP_ACCEPT(in_dev)) { + if (arp_accept(in_dev, sip)) { /* Unsolicited ARP is not accepted by default. It is possible, that this option should be enabled for some devices (strip is candidate) diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 7a181631b995..85a9e500c42d 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -197,17 +197,17 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id, } } -BTF_SET_START(bpf_tcp_ca_check_kfunc_ids) -BTF_ID(func, tcp_reno_ssthresh) -BTF_ID(func, tcp_reno_cong_avoid) -BTF_ID(func, tcp_reno_undo_cwnd) -BTF_ID(func, tcp_slow_start) -BTF_ID(func, tcp_cong_avoid_ai) -BTF_SET_END(bpf_tcp_ca_check_kfunc_ids) +BTF_SET8_START(bpf_tcp_ca_check_kfunc_ids) +BTF_ID_FLAGS(func, tcp_reno_ssthresh) +BTF_ID_FLAGS(func, tcp_reno_cong_avoid) +BTF_ID_FLAGS(func, tcp_reno_undo_cwnd) +BTF_ID_FLAGS(func, tcp_slow_start) +BTF_ID_FLAGS(func, tcp_cong_avoid_ai) +BTF_SET8_END(bpf_tcp_ca_check_kfunc_ids) static const struct btf_kfunc_id_set bpf_tcp_ca_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &bpf_tcp_ca_check_kfunc_ids, + .owner = THIS_MODULE, + .set = &bpf_tcp_ca_check_kfunc_ids, }; static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = { diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index 7eae8d686e20..5c03eba787e5 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -1106,7 +1106,7 @@ static int esp_init_authenc(struct xfrm_state *x) err = -EINVAL; if (aalg_desc->uinfo.auth.icv_fullbits / 8 != crypto_aead_authsize(aead)) { - pr_info("ESP: %s digestsize %u != %hu\n", + pr_info("ESP: %s digestsize %u != %u\n", x->aalg->alg_name, crypto_aead_authsize(aead), aalg_desc->uinfo.auth.icv_fullbits / 8); diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index 14f037e8ac55..2dc97583d279 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -2217,7 +2217,7 @@ void fib_select_multipath(struct fib_result *res, int hash) } change_nexthops(fi) { - if (net->ipv4.sysctl_fib_multipath_use_neigh) { + if (READ_ONCE(net->ipv4.sysctl_fib_multipath_use_neigh)) { if (!fib_good_nh(nexthop_nh)) continue; if (!first) { diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 57c4f0d87a7a..d5d745c3e345 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -881,7 +881,7 @@ static enum skb_drop_reason icmp_unreach(struct sk_buff *skb) * values please see * Documentation/networking/ip-sysctl.rst */ - switch (net->ipv4.sysctl_ip_no_pmtu_disc) { + switch (READ_ONCE(net->ipv4.sysctl_ip_no_pmtu_disc)) { default: net_dbg_ratelimited("%pI4: fragmentation needed and DF set\n", &iph->daddr); diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index b65d074d9620..e3ab0cb61624 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -467,7 +467,8 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ip_mc_list *pmc, if (pmc->multiaddr == IGMP_ALL_HOSTS) return skb; - if (ipv4_is_local_multicast(pmc->multiaddr) && !net->ipv4.sysctl_igmp_llm_reports) + if (ipv4_is_local_multicast(pmc->multiaddr) && + !READ_ONCE(net->ipv4.sysctl_igmp_llm_reports)) return skb; mtu = READ_ONCE(dev->mtu); @@ -593,7 +594,7 @@ static int igmpv3_send_report(struct in_device *in_dev, struct ip_mc_list *pmc) if (pmc->multiaddr == IGMP_ALL_HOSTS) continue; if (ipv4_is_local_multicast(pmc->multiaddr) && - !net->ipv4.sysctl_igmp_llm_reports) + !READ_ONCE(net->ipv4.sysctl_igmp_llm_reports)) continue; spin_lock_bh(&pmc->lock); if (pmc->sfcount[MCAST_EXCLUDE]) @@ -736,7 +737,8 @@ static int igmp_send_report(struct in_device *in_dev, struct ip_mc_list *pmc, if (type == IGMPV3_HOST_MEMBERSHIP_REPORT) return igmpv3_send_report(in_dev, pmc); - if (ipv4_is_local_multicast(group) && !net->ipv4.sysctl_igmp_llm_reports) + if (ipv4_is_local_multicast(group) && + !READ_ONCE(net->ipv4.sysctl_igmp_llm_reports)) return 0; if (type == IGMP_HOST_LEAVE_MESSAGE) @@ -825,7 +827,7 @@ static void igmp_ifc_event(struct in_device *in_dev) struct net *net = dev_net(in_dev->dev); if (IGMP_V1_SEEN(in_dev) || IGMP_V2_SEEN(in_dev)) return; - WRITE_ONCE(in_dev->mr_ifc_count, in_dev->mr_qrv ?: net->ipv4.sysctl_igmp_qrv); + WRITE_ONCE(in_dev->mr_ifc_count, in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv)); igmp_ifc_start_timer(in_dev, 1); } @@ -920,7 +922,8 @@ static bool igmp_heard_report(struct in_device *in_dev, __be32 group) if (group == IGMP_ALL_HOSTS) return false; - if (ipv4_is_local_multicast(group) && !net->ipv4.sysctl_igmp_llm_reports) + if (ipv4_is_local_multicast(group) && + !READ_ONCE(net->ipv4.sysctl_igmp_llm_reports)) return false; rcu_read_lock(); @@ -1006,7 +1009,7 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb, * received value was zero, use the default or statically * configured value. */ - in_dev->mr_qrv = ih3->qrv ?: net->ipv4.sysctl_igmp_qrv; + in_dev->mr_qrv = ih3->qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); in_dev->mr_qi = IGMPV3_QQIC(ih3->qqic)*HZ ?: IGMP_QUERY_INTERVAL; /* RFC3376, 8.3. Query Response Interval: @@ -1045,7 +1048,7 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb, if (im->multiaddr == IGMP_ALL_HOSTS) continue; if (ipv4_is_local_multicast(im->multiaddr) && - !net->ipv4.sysctl_igmp_llm_reports) + !READ_ONCE(net->ipv4.sysctl_igmp_llm_reports)) continue; spin_lock_bh(&im->lock); if (im->tm_running) @@ -1186,7 +1189,7 @@ static void igmpv3_add_delrec(struct in_device *in_dev, struct ip_mc_list *im, pmc->interface = im->interface; in_dev_hold(in_dev); pmc->multiaddr = im->multiaddr; - pmc->crcount = in_dev->mr_qrv ?: net->ipv4.sysctl_igmp_qrv; + pmc->crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); pmc->sfmode = im->sfmode; if (pmc->sfmode == MCAST_INCLUDE) { struct ip_sf_list *psf; @@ -1237,9 +1240,11 @@ static void igmpv3_del_delrec(struct in_device *in_dev, struct ip_mc_list *im) swap(im->tomb, pmc->tomb); swap(im->sources, pmc->sources); for (psf = im->sources; psf; psf = psf->sf_next) - psf->sf_crcount = in_dev->mr_qrv ?: net->ipv4.sysctl_igmp_qrv; + psf->sf_crcount = in_dev->mr_qrv ?: + READ_ONCE(net->ipv4.sysctl_igmp_qrv); } else { - im->crcount = in_dev->mr_qrv ?: net->ipv4.sysctl_igmp_qrv; + im->crcount = in_dev->mr_qrv ?: + READ_ONCE(net->ipv4.sysctl_igmp_qrv); } in_dev_put(pmc->interface); kfree_pmc(pmc); @@ -1296,7 +1301,8 @@ static void __igmp_group_dropped(struct ip_mc_list *im, gfp_t gfp) #ifdef CONFIG_IP_MULTICAST if (im->multiaddr == IGMP_ALL_HOSTS) return; - if (ipv4_is_local_multicast(im->multiaddr) && !net->ipv4.sysctl_igmp_llm_reports) + if (ipv4_is_local_multicast(im->multiaddr) && + !READ_ONCE(net->ipv4.sysctl_igmp_llm_reports)) return; reporter = im->reporter; @@ -1338,13 +1344,14 @@ static void igmp_group_added(struct ip_mc_list *im) #ifdef CONFIG_IP_MULTICAST if (im->multiaddr == IGMP_ALL_HOSTS) return; - if (ipv4_is_local_multicast(im->multiaddr) && !net->ipv4.sysctl_igmp_llm_reports) + if (ipv4_is_local_multicast(im->multiaddr) && + !READ_ONCE(net->ipv4.sysctl_igmp_llm_reports)) return; if (in_dev->dead) return; - im->unsolicit_count = net->ipv4.sysctl_igmp_qrv; + im->unsolicit_count = READ_ONCE(net->ipv4.sysctl_igmp_qrv); if (IGMP_V1_SEEN(in_dev) || IGMP_V2_SEEN(in_dev)) { spin_lock_bh(&im->lock); igmp_start_timer(im, IGMP_INITIAL_REPORT_DELAY); @@ -1358,7 +1365,7 @@ static void igmp_group_added(struct ip_mc_list *im) * IN() to IN(A). */ if (im->sfmode == MCAST_EXCLUDE) - im->crcount = in_dev->mr_qrv ?: net->ipv4.sysctl_igmp_qrv; + im->crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); igmp_ifc_event(in_dev); #endif @@ -1642,7 +1649,7 @@ static void ip_mc_rejoin_groups(struct in_device *in_dev) if (im->multiaddr == IGMP_ALL_HOSTS) continue; if (ipv4_is_local_multicast(im->multiaddr) && - !net->ipv4.sysctl_igmp_llm_reports) + !READ_ONCE(net->ipv4.sysctl_igmp_llm_reports)) continue; /* a failover is happening and switches @@ -1749,7 +1756,7 @@ static void ip_mc_reset(struct in_device *in_dev) in_dev->mr_qi = IGMP_QUERY_INTERVAL; in_dev->mr_qri = IGMP_QUERY_RESPONSE_INTERVAL; - in_dev->mr_qrv = net->ipv4.sysctl_igmp_qrv; + in_dev->mr_qrv = READ_ONCE(net->ipv4.sysctl_igmp_qrv); } #else static void ip_mc_reset(struct in_device *in_dev) @@ -1883,7 +1890,7 @@ static int ip_mc_del1_src(struct ip_mc_list *pmc, int sfmode, #ifdef CONFIG_IP_MULTICAST if (psf->sf_oldin && !IGMP_V1_SEEN(in_dev) && !IGMP_V2_SEEN(in_dev)) { - psf->sf_crcount = in_dev->mr_qrv ?: net->ipv4.sysctl_igmp_qrv; + psf->sf_crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); psf->sf_next = pmc->tomb; pmc->tomb = psf; rv = 1; @@ -1947,7 +1954,7 @@ static int ip_mc_del_src(struct in_device *in_dev, __be32 *pmca, int sfmode, /* filter mode change */ pmc->sfmode = MCAST_INCLUDE; #ifdef CONFIG_IP_MULTICAST - pmc->crcount = in_dev->mr_qrv ?: net->ipv4.sysctl_igmp_qrv; + pmc->crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); WRITE_ONCE(in_dev->mr_ifc_count, pmc->crcount); for (psf = pmc->sources; psf; psf = psf->sf_next) psf->sf_crcount = 0; @@ -2126,7 +2133,7 @@ static int ip_mc_add_src(struct in_device *in_dev, __be32 *pmca, int sfmode, #ifdef CONFIG_IP_MULTICAST /* else no filters; keep old mode for reports */ - pmc->crcount = in_dev->mr_qrv ?: net->ipv4.sysctl_igmp_qrv; + pmc->crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); WRITE_ONCE(in_dev->mr_ifc_count, pmc->crcount); for (psf = pmc->sources; psf; psf = psf->sf_next) psf->sf_crcount = 0; @@ -2192,7 +2199,7 @@ static int __ip_mc_join_group(struct sock *sk, struct ip_mreqn *imr, count++; } err = -ENOBUFS; - if (count >= net->ipv4.sysctl_igmp_max_memberships) + if (count >= READ_ONCE(net->ipv4.sysctl_igmp_max_memberships)) goto done; iml = sock_kmalloc(sk, sizeof(*iml), GFP_KERNEL); if (!iml) @@ -2379,7 +2386,7 @@ int ip_mc_source(int add, int omode, struct sock *sk, struct } /* else, add a new source to the filter */ - if (psl && psl->sl_count >= net->ipv4.sysctl_igmp_max_msf) { + if (psl && psl->sl_count >= READ_ONCE(net->ipv4.sysctl_igmp_max_msf)) { err = -ENOBUFS; goto done; } diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 53f5f956d948..eb31c7158b39 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -263,7 +263,7 @@ next_port: goto other_half_scan; } - if (net->ipv4.sysctl_ip_autobind_reuse && !relax) { + if (READ_ONCE(net->ipv4.sysctl_ip_autobind_reuse) && !relax) { /* We still have a chance to connect to different destinations */ relax = true; goto ports_exhausted; @@ -833,7 +833,8 @@ static void reqsk_timer_handler(struct timer_list *t) icsk = inet_csk(sk_listener); net = sock_net(sk_listener); - max_syn_ack_retries = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_synack_retries; + max_syn_ack_retries = icsk->icsk_syn_retries ? : + READ_ONCE(net->ipv4.sysctl_tcp_synack_retries); /* Normally all the openreqs are young and become mature * (i.e. converted to established socket) for first timeout. * If synack was not acknowledged for 1 second, it means diff --git a/net/ipv4/ip_forward.c b/net/ipv4/ip_forward.c index e3aa436a1bdf..e18931a6d153 100644 --- a/net/ipv4/ip_forward.c +++ b/net/ipv4/ip_forward.c @@ -157,7 +157,7 @@ int ip_forward(struct sk_buff *skb) !skb_sec_path(skb)) ip_rt_send_redirect(skb); - if (net->ipv4.sysctl_ip_fwd_update_priority) + if (READ_ONCE(net->ipv4.sysctl_ip_fwd_update_priority)) skb->priority = rt_tos2priority(iph->tos); return NF_HOOK(NFPROTO_IPV4, NF_INET_FORWARD, diff --git a/net/ipv4/ip_input.c b/net/ipv4/ip_input.c index b1165f717cd1..1b512390b3cf 100644 --- a/net/ipv4/ip_input.c +++ b/net/ipv4/ip_input.c @@ -312,14 +312,13 @@ static bool ip_can_use_hint(const struct sk_buff *skb, const struct iphdr *iph, ip_hdr(hint)->tos == iph->tos; } -INDIRECT_CALLABLE_DECLARE(int udp_v4_early_demux(struct sk_buff *)); -INDIRECT_CALLABLE_DECLARE(int tcp_v4_early_demux(struct sk_buff *)); +int tcp_v4_early_demux(struct sk_buff *skb); +int udp_v4_early_demux(struct sk_buff *skb); static int ip_rcv_finish_core(struct net *net, struct sock *sk, struct sk_buff *skb, struct net_device *dev, const struct sk_buff *hint) { const struct iphdr *iph = ip_hdr(skb); - int (*edemux)(struct sk_buff *skb); int err, drop_reason; struct rtable *rt; @@ -332,21 +331,29 @@ static int ip_rcv_finish_core(struct net *net, struct sock *sk, goto drop_error; } - if (net->ipv4.sysctl_ip_early_demux && + if (READ_ONCE(net->ipv4.sysctl_ip_early_demux) && !skb_dst(skb) && !skb->sk && !ip_is_fragment(iph)) { - const struct net_protocol *ipprot; - int protocol = iph->protocol; - - ipprot = rcu_dereference(inet_protos[protocol]); - if (ipprot && (edemux = READ_ONCE(ipprot->early_demux))) { - err = INDIRECT_CALL_2(edemux, tcp_v4_early_demux, - udp_v4_early_demux, skb); - if (unlikely(err)) - goto drop_error; - /* must reload iph, skb->head might have changed */ - iph = ip_hdr(skb); + switch (iph->protocol) { + case IPPROTO_TCP: + if (READ_ONCE(net->ipv4.sysctl_tcp_early_demux)) { + tcp_v4_early_demux(skb); + + /* must reload iph, skb->head might have changed */ + iph = ip_hdr(skb); + } + break; + case IPPROTO_UDP: + if (READ_ONCE(net->ipv4.sysctl_udp_early_demux)) { + err = udp_v4_early_demux(skb); + if (unlikely(err)) + goto drop_error; + + /* must reload iph, skb->head might have changed */ + iph = ip_hdr(skb); + } + break; } } diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index f7156845ddf7..d7bd1daf022b 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -969,7 +969,6 @@ static int __ip_append_data(struct sock *sk, struct inet_sock *inet = inet_sk(sk); struct ubuf_info *uarg = NULL; struct sk_buff *skb; - struct ip_options *opt = cork->opt; int hh_len; int exthdrlen; @@ -977,6 +976,7 @@ static int __ip_append_data(struct sock *sk, int copy; int err; int offset = 0; + bool zc = false; unsigned int maxfraglen, fragheaderlen, maxnonfragsize; int csummode = CHECKSUM_NONE; struct rtable *rt = (struct rtable *)cork->dst; @@ -1017,17 +1017,35 @@ static int __ip_append_data(struct sock *sk, (!exthdrlen || (rt->dst.dev->features & NETIF_F_HW_ESP_TX_CSUM))) csummode = CHECKSUM_PARTIAL; - if (flags & MSG_ZEROCOPY && length && sock_flag(sk, SOCK_ZEROCOPY)) { - uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb)); - if (!uarg) - return -ENOBUFS; - extra_uref = !skb_zcopy(skb); /* only ref on new uarg */ - if (rt->dst.dev->features & NETIF_F_SG && - csummode == CHECKSUM_PARTIAL) { - paged = true; - } else { - uarg->zerocopy = 0; - skb_zcopy_set(skb, uarg, &extra_uref); + if ((flags & MSG_ZEROCOPY) && length) { + struct msghdr *msg = from; + + if (getfrag == ip_generic_getfrag && msg->msg_ubuf) { + if (skb_zcopy(skb) && msg->msg_ubuf != skb_zcopy(skb)) + return -EINVAL; + + /* Leave uarg NULL if can't zerocopy, callers should + * be able to handle it. + */ + if ((rt->dst.dev->features & NETIF_F_SG) && + csummode == CHECKSUM_PARTIAL) { + paged = true; + zc = true; + uarg = msg->msg_ubuf; + } + } else if (sock_flag(sk, SOCK_ZEROCOPY)) { + uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb)); + if (!uarg) + return -ENOBUFS; + extra_uref = !skb_zcopy(skb); /* only ref on new uarg */ + if (rt->dst.dev->features & NETIF_F_SG && + csummode == CHECKSUM_PARTIAL) { + paged = true; + zc = true; + } else { + uarg->zerocopy = 0; + skb_zcopy_set(skb, uarg, &extra_uref); + } } } @@ -1091,9 +1109,12 @@ alloc_new_skb: (fraglen + alloc_extra < SKB_MAX_ALLOC || !(rt->dst.dev->features & NETIF_F_SG))) alloclen = fraglen; - else { + else if (!zc) { alloclen = min_t(int, fraglen, MAX_HEADER); pagedlen = fraglen - alloclen; + } else { + alloclen = fragheaderlen + transhdrlen; + pagedlen = datalen - transhdrlen; } alloclen += alloc_extra; @@ -1188,13 +1209,14 @@ alloc_new_skb: err = -EFAULT; goto error; } - } else if (!uarg || !uarg->zerocopy) { + } else if (!zc) { int i = skb_shinfo(skb)->nr_frags; err = -ENOMEM; if (!sk_page_frag_refill(sk, pfrag)) goto error; + skb_zcopy_downgrade_managed(skb); if (!skb_can_coalesce(skb, i, pfrag->page, pfrag->offset)) { err = -EMSGSIZE; diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index 445a9ecaefa1..a8a323ecbb54 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -782,7 +782,7 @@ static int ip_set_mcast_msfilter(struct sock *sk, sockptr_t optval, int optlen) /* numsrc >= (4G-140)/128 overflow in 32 bits */ err = -ENOBUFS; if (gsf->gf_numsrc >= 0x1ffffff || - gsf->gf_numsrc > sock_net(sk)->ipv4.sysctl_igmp_max_msf) + gsf->gf_numsrc > READ_ONCE(sock_net(sk)->ipv4.sysctl_igmp_max_msf)) goto out_free_gsf; err = -EINVAL; @@ -832,7 +832,7 @@ static int compat_ip_set_mcast_msfilter(struct sock *sk, sockptr_t optval, /* numsrc >= (4G-140)/128 overflow in 32 bits */ err = -ENOBUFS; - if (n > sock_net(sk)->ipv4.sysctl_igmp_max_msf) + if (n > READ_ONCE(sock_net(sk)->ipv4.sysctl_igmp_max_msf)) goto out_free_gsf; err = set_mcast_msfilter(sk, gf32->gf_interface, n, gf32->gf_fmode, &gf32->gf_group, gf32->gf_slist_flex); @@ -1244,7 +1244,7 @@ static int do_ip_setsockopt(struct sock *sk, int level, int optname, } /* numsrc >= (1G-4) overflow in 32 bits */ if (msf->imsf_numsrc >= 0x3ffffffcU || - msf->imsf_numsrc > net->ipv4.sysctl_igmp_max_msf) { + msf->imsf_numsrc > READ_ONCE(net->ipv4.sysctl_igmp_max_msf)) { kfree(msf); err = -ENOBUFS; break; @@ -1606,7 +1606,7 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname, { struct net *net = sock_net(sk); val = (inet->uc_ttl == -1 ? - net->ipv4.sysctl_ip_default_ttl : + READ_ONCE(net->ipv4.sysctl_ip_default_ttl) : inet->uc_ttl); break; } diff --git a/net/ipv4/netfilter/nf_nat_h323.c b/net/ipv4/netfilter/nf_nat_h323.c index 76a411ae9fe6..a334f0dcc2d0 100644 --- a/net/ipv4/netfilter/nf_nat_h323.c +++ b/net/ipv4/netfilter/nf_nat_h323.c @@ -579,28 +579,22 @@ static struct nf_ct_helper_expectfn callforwarding_nat = { .expectfn = ip_nat_callforwarding_expect, }; +static const struct nfct_h323_nat_hooks nathooks = { + .set_h245_addr = set_h245_addr, + .set_h225_addr = set_h225_addr, + .set_sig_addr = set_sig_addr, + .set_ras_addr = set_ras_addr, + .nat_rtp_rtcp = nat_rtp_rtcp, + .nat_t120 = nat_t120, + .nat_h245 = nat_h245, + .nat_callforwarding = nat_callforwarding, + .nat_q931 = nat_q931, +}; + /****************************************************************************/ static int __init nf_nat_h323_init(void) { - BUG_ON(set_h245_addr_hook != NULL); - BUG_ON(set_h225_addr_hook != NULL); - BUG_ON(set_sig_addr_hook != NULL); - BUG_ON(set_ras_addr_hook != NULL); - BUG_ON(nat_rtp_rtcp_hook != NULL); - BUG_ON(nat_t120_hook != NULL); - BUG_ON(nat_h245_hook != NULL); - BUG_ON(nat_callforwarding_hook != NULL); - BUG_ON(nat_q931_hook != NULL); - - RCU_INIT_POINTER(set_h245_addr_hook, set_h245_addr); - RCU_INIT_POINTER(set_h225_addr_hook, set_h225_addr); - RCU_INIT_POINTER(set_sig_addr_hook, set_sig_addr); - RCU_INIT_POINTER(set_ras_addr_hook, set_ras_addr); - RCU_INIT_POINTER(nat_rtp_rtcp_hook, nat_rtp_rtcp); - RCU_INIT_POINTER(nat_t120_hook, nat_t120); - RCU_INIT_POINTER(nat_h245_hook, nat_h245); - RCU_INIT_POINTER(nat_callforwarding_hook, nat_callforwarding); - RCU_INIT_POINTER(nat_q931_hook, nat_q931); + RCU_INIT_POINTER(nfct_h323_nat_hook, &nathooks); nf_ct_helper_expectfn_register(&q931_nat); nf_ct_helper_expectfn_register(&callforwarding_nat); return 0; @@ -609,15 +603,7 @@ static int __init nf_nat_h323_init(void) /****************************************************************************/ static void __exit nf_nat_h323_fini(void) { - RCU_INIT_POINTER(set_h245_addr_hook, NULL); - RCU_INIT_POINTER(set_h225_addr_hook, NULL); - RCU_INIT_POINTER(set_sig_addr_hook, NULL); - RCU_INIT_POINTER(set_ras_addr_hook, NULL); - RCU_INIT_POINTER(nat_rtp_rtcp_hook, NULL); - RCU_INIT_POINTER(nat_t120_hook, NULL); - RCU_INIT_POINTER(nat_h245_hook, NULL); - RCU_INIT_POINTER(nat_callforwarding_hook, NULL); - RCU_INIT_POINTER(nat_q931_hook, NULL); + RCU_INIT_POINTER(nfct_h323_nat_hook, NULL); nf_ct_helper_expectfn_unregister(&q931_nat); nf_ct_helper_expectfn_unregister(&callforwarding_nat); synchronize_rcu(); diff --git a/net/ipv4/netfilter/nf_reject_ipv4.c b/net/ipv4/netfilter/nf_reject_ipv4.c index 918c61fda0f3..d640adcaf1b1 100644 --- a/net/ipv4/netfilter/nf_reject_ipv4.c +++ b/net/ipv4/netfilter/nf_reject_ipv4.c @@ -62,7 +62,7 @@ struct sk_buff *nf_reject_skb_v4_tcp_reset(struct net *net, skb_reserve(nskb, LL_MAX_HEADER); niph = nf_reject_iphdr_put(nskb, oldskb, IPPROTO_TCP, - net->ipv4.sysctl_ip_default_ttl); + READ_ONCE(net->ipv4.sysctl_ip_default_ttl)); nf_reject_ip_tcphdr_put(nskb, oldskb, oth); niph->tot_len = htons(nskb->len); ip_send_check(niph); @@ -117,7 +117,7 @@ struct sk_buff *nf_reject_skb_v4_unreach(struct net *net, skb_reserve(nskb, LL_MAX_HEADER); niph = nf_reject_iphdr_put(nskb, oldskb, IPPROTO_ICMP, - net->ipv4.sysctl_ip_default_ttl); + READ_ONCE(net->ipv4.sysctl_ip_default_ttl)); skb_reset_transport_header(nskb); icmph = skb_put_zero(nskb, sizeof(struct icmphdr)); diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c index 28836071f0a6..0088a4c64d77 100644 --- a/net/ipv4/proc.c +++ b/net/ipv4/proc.c @@ -387,7 +387,7 @@ static int snmp_seq_show_ipstats(struct seq_file *seq, void *v) seq_printf(seq, "\nIp: %d %d", IPV4_DEVCONF_ALL(net, FORWARDING) ? 1 : 2, - net->ipv4.sysctl_ip_default_ttl); + READ_ONCE(net->ipv4.sysctl_ip_default_ttl)); BUILD_BUG_ON(offsetof(struct ipstats_mib, mibs) != 0); snmp_get_cpu_field64_batch(buff64, snmp4_ipstats_list, diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 8f4a8f66d604..795cbe1de912 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -1398,7 +1398,7 @@ u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr) struct fib_info *fi = res->fi; u32 mtu = 0; - if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu || + if (READ_ONCE(dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu) || fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU)) mtu = fi->fib_mtu; @@ -1626,12 +1626,11 @@ static void rt_set_nexthop(struct rtable *rt, __be32 daddr, struct rtable *rt_dst_alloc(struct net_device *dev, unsigned int flags, u16 type, - bool nopolicy, bool noxfrm) + bool noxfrm) { struct rtable *rt; rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK, - (nopolicy ? DST_NOPOLICY : 0) | (noxfrm ? DST_NOXFRM : 0)); if (rt) { @@ -1726,7 +1725,6 @@ static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr, struct in_device *in_dev = __in_dev_get_rcu(dev); unsigned int flags = RTCF_MULTICAST; struct rtable *rth; - bool no_policy; u32 itag = 0; int err; @@ -1737,12 +1735,11 @@ static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr, if (our) flags |= RTCF_LOCAL; - no_policy = IN_DEV_ORCONF(in_dev, NOPOLICY); - if (no_policy) + if (IN_DEV_ORCONF(in_dev, NOPOLICY)) IPCB(skb)->flags |= IPSKB_NOPOLICY; rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST, - no_policy, false); + false); if (!rth) return -ENOBUFS; @@ -1801,7 +1798,7 @@ static int __mkroute_input(struct sk_buff *skb, struct rtable *rth; int err; struct in_device *out_dev; - bool do_cache, no_policy; + bool do_cache; u32 itag = 0; /* get a working reference to the output device */ @@ -1846,8 +1843,7 @@ static int __mkroute_input(struct sk_buff *skb, } } - no_policy = IN_DEV_ORCONF(in_dev, NOPOLICY); - if (no_policy) + if (IN_DEV_ORCONF(in_dev, NOPOLICY)) IPCB(skb)->flags |= IPSKB_NOPOLICY; fnhe = find_exception(nhc, daddr); @@ -1862,7 +1858,7 @@ static int __mkroute_input(struct sk_buff *skb, } } - rth = rt_dst_alloc(out_dev->dev, 0, res->type, no_policy, + rth = rt_dst_alloc(out_dev->dev, 0, res->type, IN_DEV_ORCONF(out_dev, NOXFRM)); if (!rth) { err = -ENOBUFS; @@ -1928,7 +1924,7 @@ static u32 fib_multipath_custom_hash_outer(const struct net *net, const struct sk_buff *skb, bool *p_has_inner) { - u32 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields; + u32 hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields); struct flow_keys keys, hash_keys; if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK)) @@ -1957,7 +1953,7 @@ static u32 fib_multipath_custom_hash_inner(const struct net *net, const struct sk_buff *skb, bool has_inner) { - u32 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields; + u32 hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields); struct flow_keys keys, hash_keys; /* We assume the packet carries an encapsulation, but if none was @@ -2017,7 +2013,7 @@ static u32 fib_multipath_custom_hash_skb(const struct net *net, static u32 fib_multipath_custom_hash_fl4(const struct net *net, const struct flowi4 *fl4) { - u32 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields; + u32 hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields); struct flow_keys hash_keys; if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK)) @@ -2047,7 +2043,7 @@ int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4, struct flow_keys hash_keys; u32 mhash = 0; - switch (net->ipv4.sysctl_fib_multipath_hash_policy) { + switch (READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_policy)) { case 0: memset(&hash_keys, 0, sizeof(hash_keys)); hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; @@ -2237,7 +2233,6 @@ static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr, struct rtable *rth; struct flowi4 fl4; bool do_cache = true; - bool no_policy; /* IP on this device is disabled. */ @@ -2356,8 +2351,7 @@ brd_input: RT_CACHE_STAT_INC(in_brd); local_input: - no_policy = IN_DEV_ORCONF(in_dev, NOPOLICY); - if (no_policy) + if (IN_DEV_ORCONF(in_dev, NOPOLICY)) IPCB(skb)->flags |= IPSKB_NOPOLICY; do_cache &= res->fi && !itag; @@ -2373,8 +2367,7 @@ local_input: } rth = rt_dst_alloc(ip_rt_get_dev(net, res), - flags | RTCF_LOCAL, res->type, - no_policy, false); + flags | RTCF_LOCAL, res->type, false); if (!rth) goto e_nobufs; @@ -2597,7 +2590,6 @@ static struct rtable *__mkroute_output(const struct fib_result *res, add: rth = rt_dst_alloc(dev_out, flags, type, - IN_DEV_ORCONF(in_dev, NOPOLICY), IN_DEV_ORCONF(in_dev, NOXFRM)); if (!rth) return ERR_PTR(-ENOBUFS); diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index b387c4835155..942d2dfa1115 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -247,12 +247,12 @@ bool cookie_timestamp_decode(const struct net *net, return true; } - if (!net->ipv4.sysctl_tcp_timestamps) + if (!READ_ONCE(net->ipv4.sysctl_tcp_timestamps)) return false; tcp_opt->sack_ok = (options & TS_OPT_SACK) ? TCP_SACK_SEEN : 0; - if (tcp_opt->sack_ok && !net->ipv4.sysctl_tcp_sack) + if (tcp_opt->sack_ok && !READ_ONCE(net->ipv4.sysctl_tcp_sack)) return false; if ((options & TS_OPT_WSCALE_MASK) == TS_OPT_WSCALE_MASK) @@ -261,7 +261,7 @@ bool cookie_timestamp_decode(const struct net *net, tcp_opt->wscale_ok = 1; tcp_opt->snd_wscale = options & TS_OPT_WSCALE_MASK; - return net->ipv4.sysctl_tcp_window_scaling != 0; + return READ_ONCE(net->ipv4.sysctl_tcp_window_scaling) != 0; } EXPORT_SYMBOL(cookie_timestamp_decode); @@ -340,7 +340,8 @@ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb) struct flowi4 fl4; u32 tsoff = 0; - if (!sock_net(sk)->ipv4.sysctl_tcp_syncookies || !th->ack || th->rst) + if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies) || + !th->ack || th->rst) goto out; if (tcp_synq_no_recent_overflow(sk)) diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index 108fd86f2718..5490c285668b 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -84,7 +84,7 @@ static int ipv4_local_port_range(struct ctl_table *table, int write, * port limit. */ if ((range[1] < range[0]) || - (range[0] < net->ipv4.sysctl_ip_prot_sock)) + (range[0] < READ_ONCE(net->ipv4.sysctl_ip_prot_sock))) ret = -EINVAL; else set_local_port_range(net, range); @@ -110,7 +110,7 @@ static int ipv4_privileged_ports(struct ctl_table *table, int write, .extra2 = &ip_privileged_port_max, }; - pports = net->ipv4.sysctl_ip_prot_sock; + pports = READ_ONCE(net->ipv4.sysctl_ip_prot_sock); ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); @@ -122,7 +122,7 @@ static int ipv4_privileged_ports(struct ctl_table *table, int write, if (range[0] < pports) ret = -EINVAL; else - net->ipv4.sysctl_ip_prot_sock = pports; + WRITE_ONCE(net->ipv4.sysctl_ip_prot_sock, pports); } return ret; @@ -350,61 +350,6 @@ bad_key: return ret; } -static void proc_configure_early_demux(int enabled, int protocol) -{ - struct net_protocol *ipprot; -#if IS_ENABLED(CONFIG_IPV6) - struct inet6_protocol *ip6prot; -#endif - - rcu_read_lock(); - - ipprot = rcu_dereference(inet_protos[protocol]); - if (ipprot) - ipprot->early_demux = enabled ? ipprot->early_demux_handler : - NULL; - -#if IS_ENABLED(CONFIG_IPV6) - ip6prot = rcu_dereference(inet6_protos[protocol]); - if (ip6prot) - ip6prot->early_demux = enabled ? ip6prot->early_demux_handler : - NULL; -#endif - rcu_read_unlock(); -} - -static int proc_tcp_early_demux(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - int ret = 0; - - ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos); - - if (write && !ret) { - int enabled = init_net.ipv4.sysctl_tcp_early_demux; - - proc_configure_early_demux(enabled, IPPROTO_TCP); - } - - return ret; -} - -static int proc_udp_early_demux(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - int ret = 0; - - ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos); - - if (write && !ret) { - int enabled = init_net.ipv4.sysctl_udp_early_demux; - - proc_configure_early_demux(enabled, IPPROTO_UDP); - } - - return ret; -} - static int proc_tfo_blackhole_detect_timeout(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) @@ -707,14 +652,14 @@ static struct ctl_table ipv4_net_table[] = { .data = &init_net.ipv4.sysctl_udp_early_demux, .maxlen = sizeof(u8), .mode = 0644, - .proc_handler = proc_udp_early_demux + .proc_handler = proc_dou8vec_minmax, }, { .procname = "tcp_early_demux", .data = &init_net.ipv4.sysctl_tcp_early_demux, .maxlen = sizeof(u8), .mode = 0644, - .proc_handler = proc_tcp_early_demux + .proc_handler = proc_dou8vec_minmax, }, { .procname = "nexthop_compat_mode", diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 96b6e9c22068..ba2bdc811374 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -443,7 +443,7 @@ void tcp_init_sock(struct sock *sk) tp->snd_cwnd_clamp = ~0; tp->mss_cache = TCP_MSS_DEFAULT; - tp->reordering = sock_net(sk)->ipv4.sysctl_tcp_reordering; + tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering); tcp_assign_congestion_control(sk); tp->tsoffset = 0; @@ -1171,7 +1171,8 @@ static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, struct sockaddr *uaddr = msg->msg_name; int err, flags; - if (!(sock_net(sk)->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) || + if (!(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) & + TFO_CLIENT_ENABLE) || (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) && uaddr->sa_family == AF_UNSPEC)) return -EOPNOTSUPP; @@ -1223,17 +1224,23 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) flags = msg->msg_flags; - if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) { + if ((flags & MSG_ZEROCOPY) && size) { skb = tcp_write_queue_tail(sk); - uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb)); - if (!uarg) { - err = -ENOBUFS; - goto out_err; - } - zc = sk->sk_route_caps & NETIF_F_SG; - if (!zc) - uarg->zerocopy = 0; + if (msg->msg_ubuf) { + uarg = msg->msg_ubuf; + net_zcopy_get(uarg); + zc = sk->sk_route_caps & NETIF_F_SG; + } else if (sock_flag(sk, SOCK_ZEROCOPY)) { + uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb)); + if (!uarg) { + err = -ENOBUFS; + goto out_err; + } + zc = sk->sk_route_caps & NETIF_F_SG; + if (!zc) + uarg->zerocopy = 0; + } } if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) && @@ -1356,8 +1363,11 @@ new_segment: copy = min_t(int, copy, pfrag->size - pfrag->offset); - if (tcp_downgrade_zcopy_pure(sk, skb)) - goto wait_for_space; + if (unlikely(skb_zcopy_pure(skb) || skb_zcopy_managed(skb))) { + if (tcp_downgrade_zcopy_pure(sk, skb)) + goto wait_for_space; + skb_zcopy_downgrade_managed(skb); + } copy = tcp_wmem_schedule(sk, copy); if (!copy) @@ -3682,7 +3692,8 @@ static int do_tcp_setsockopt(struct sock *sk, int level, int optname, case TCP_FASTOPEN_CONNECT: if (val > 1 || val < 0) { err = -EINVAL; - } else if (net->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) { + } else if (READ_ONCE(net->ipv4.sysctl_tcp_fastopen) & + TFO_CLIENT_ENABLE) { if (sk->sk_state == TCP_CLOSE) tp->fastopen_connect = val; else @@ -4032,12 +4043,13 @@ static int do_tcp_getsockopt(struct sock *sk, int level, val = keepalive_probes(tp); break; case TCP_SYNCNT: - val = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_syn_retries; + val = icsk->icsk_syn_retries ? : + READ_ONCE(net->ipv4.sysctl_tcp_syn_retries); break; case TCP_LINGER2: val = tp->linger2; if (val >= 0) - val = (val ? : net->ipv4.sysctl_tcp_fin_timeout) / HZ; + val = (val ? : READ_ONCE(net->ipv4.sysctl_tcp_fin_timeout)) / HZ; break; case TCP_DEFER_ACCEPT: val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept, diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c index 075e744bfb48..54eec33c6e1c 100644 --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c @@ -1154,24 +1154,24 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { .set_state = bbr_set_state, }; -BTF_SET_START(tcp_bbr_check_kfunc_ids) +BTF_SET8_START(tcp_bbr_check_kfunc_ids) #ifdef CONFIG_X86 #ifdef CONFIG_DYNAMIC_FTRACE -BTF_ID(func, bbr_init) -BTF_ID(func, bbr_main) -BTF_ID(func, bbr_sndbuf_expand) -BTF_ID(func, bbr_undo_cwnd) -BTF_ID(func, bbr_cwnd_event) -BTF_ID(func, bbr_ssthresh) -BTF_ID(func, bbr_min_tso_segs) -BTF_ID(func, bbr_set_state) +BTF_ID_FLAGS(func, bbr_init) +BTF_ID_FLAGS(func, bbr_main) +BTF_ID_FLAGS(func, bbr_sndbuf_expand) +BTF_ID_FLAGS(func, bbr_undo_cwnd) +BTF_ID_FLAGS(func, bbr_cwnd_event) +BTF_ID_FLAGS(func, bbr_ssthresh) +BTF_ID_FLAGS(func, bbr_min_tso_segs) +BTF_ID_FLAGS(func, bbr_set_state) #endif #endif -BTF_SET_END(tcp_bbr_check_kfunc_ids) +BTF_SET8_END(tcp_bbr_check_kfunc_ids) static const struct btf_kfunc_id_set tcp_bbr_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &tcp_bbr_check_kfunc_ids, + .owner = THIS_MODULE, + .set = &tcp_bbr_check_kfunc_ids, }; static int __init bbr_register(void) diff --git a/net/ipv4/tcp_cubic.c b/net/ipv4/tcp_cubic.c index 68178e7280ce..768c10c1f649 100644 --- a/net/ipv4/tcp_cubic.c +++ b/net/ipv4/tcp_cubic.c @@ -485,22 +485,22 @@ static struct tcp_congestion_ops cubictcp __read_mostly = { .name = "cubic", }; -BTF_SET_START(tcp_cubic_check_kfunc_ids) +BTF_SET8_START(tcp_cubic_check_kfunc_ids) #ifdef CONFIG_X86 #ifdef CONFIG_DYNAMIC_FTRACE -BTF_ID(func, cubictcp_init) -BTF_ID(func, cubictcp_recalc_ssthresh) -BTF_ID(func, cubictcp_cong_avoid) -BTF_ID(func, cubictcp_state) -BTF_ID(func, cubictcp_cwnd_event) -BTF_ID(func, cubictcp_acked) +BTF_ID_FLAGS(func, cubictcp_init) +BTF_ID_FLAGS(func, cubictcp_recalc_ssthresh) +BTF_ID_FLAGS(func, cubictcp_cong_avoid) +BTF_ID_FLAGS(func, cubictcp_state) +BTF_ID_FLAGS(func, cubictcp_cwnd_event) +BTF_ID_FLAGS(func, cubictcp_acked) #endif #endif -BTF_SET_END(tcp_cubic_check_kfunc_ids) +BTF_SET8_END(tcp_cubic_check_kfunc_ids) static const struct btf_kfunc_id_set tcp_cubic_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &tcp_cubic_check_kfunc_ids, + .owner = THIS_MODULE, + .set = &tcp_cubic_check_kfunc_ids, }; static int __init cubictcp_register(void) diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c index ab034a4e9324..2a6c0dd665a4 100644 --- a/net/ipv4/tcp_dctcp.c +++ b/net/ipv4/tcp_dctcp.c @@ -239,22 +239,22 @@ static struct tcp_congestion_ops dctcp_reno __read_mostly = { .name = "dctcp-reno", }; -BTF_SET_START(tcp_dctcp_check_kfunc_ids) +BTF_SET8_START(tcp_dctcp_check_kfunc_ids) #ifdef CONFIG_X86 #ifdef CONFIG_DYNAMIC_FTRACE -BTF_ID(func, dctcp_init) -BTF_ID(func, dctcp_update_alpha) -BTF_ID(func, dctcp_cwnd_event) -BTF_ID(func, dctcp_ssthresh) -BTF_ID(func, dctcp_cwnd_undo) -BTF_ID(func, dctcp_state) +BTF_ID_FLAGS(func, dctcp_init) +BTF_ID_FLAGS(func, dctcp_update_alpha) +BTF_ID_FLAGS(func, dctcp_cwnd_event) +BTF_ID_FLAGS(func, dctcp_ssthresh) +BTF_ID_FLAGS(func, dctcp_cwnd_undo) +BTF_ID_FLAGS(func, dctcp_state) #endif #endif -BTF_SET_END(tcp_dctcp_check_kfunc_ids) +BTF_SET8_END(tcp_dctcp_check_kfunc_ids) static const struct btf_kfunc_id_set tcp_dctcp_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &tcp_dctcp_check_kfunc_ids, + .owner = THIS_MODULE, + .set = &tcp_dctcp_check_kfunc_ids, }; static int __init dctcp_register(void) diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index fdbcf2a6d08e..825b216d11f5 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c @@ -332,7 +332,7 @@ static bool tcp_fastopen_no_cookie(const struct sock *sk, const struct dst_entry *dst, int flag) { - return (sock_net(sk)->ipv4.sysctl_tcp_fastopen & flag) || + return (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) & flag) || tcp_sk(sk)->fastopen_no_cookie || (dst && dst_metric(dst, RTAX_FASTOPEN_NO_COOKIE)); } @@ -347,7 +347,7 @@ struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, const struct dst_entry *dst) { bool syn_data = TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq + 1; - int tcp_fastopen = sock_net(sk)->ipv4.sysctl_tcp_fastopen; + int tcp_fastopen = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen); struct tcp_fastopen_cookie valid_foc = { .len = -1 }; struct sock *child; int ret = 0; @@ -489,7 +489,7 @@ void tcp_fastopen_active_disable(struct sock *sk) { struct net *net = sock_net(sk); - if (!sock_net(sk)->ipv4.sysctl_tcp_fastopen_blackhole_timeout) + if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen_blackhole_timeout)) return; /* Paired with READ_ONCE() in tcp_fastopen_active_should_disable() */ @@ -510,7 +510,8 @@ void tcp_fastopen_active_disable(struct sock *sk) */ bool tcp_fastopen_active_should_disable(struct sock *sk) { - unsigned int tfo_bh_timeout = sock_net(sk)->ipv4.sysctl_tcp_fastopen_blackhole_timeout; + unsigned int tfo_bh_timeout = + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen_blackhole_timeout); unsigned long timeout; int tfo_da_times; int multiplier; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index acee998c14b2..ae73b34d32e9 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -1050,7 +1050,7 @@ static void tcp_check_sack_reordering(struct sock *sk, const u32 low_seq, tp->undo_marker ? tp->undo_retrans : 0); #endif tp->reordering = min_t(u32, (metric + mss - 1) / mss, - sock_net(sk)->ipv4.sysctl_tcp_max_reordering); + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering)); } /* This exciting event is worth to be remembered. 8) */ @@ -2029,7 +2029,7 @@ static void tcp_check_reno_reordering(struct sock *sk, const int addend) return; tp->reordering = min_t(u32, tp->packets_out + addend, - sock_net(sk)->ipv4.sysctl_tcp_max_reordering); + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering)); tp->reord_seen++; NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRENOREORDER); } @@ -2094,7 +2094,8 @@ static inline void tcp_init_undo(struct tcp_sock *tp) static bool tcp_is_rack(const struct sock *sk) { - return sock_net(sk)->ipv4.sysctl_tcp_recovery & TCP_RACK_LOSS_DETECTION; + return READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_recovery) & + TCP_RACK_LOSS_DETECTION; } /* If we detect SACK reneging, forget all SACK information @@ -2138,6 +2139,7 @@ void tcp_enter_loss(struct sock *sk) struct tcp_sock *tp = tcp_sk(sk); struct net *net = sock_net(sk); bool new_recovery = icsk->icsk_ca_state < TCP_CA_Recovery; + u8 reordering; tcp_timeout_mark_lost(sk); @@ -2158,10 +2160,12 @@ void tcp_enter_loss(struct sock *sk) /* Timeout in disordered state after receiving substantial DUPACKs * suggests that the degree of reordering is over-estimated. */ + reordering = READ_ONCE(net->ipv4.sysctl_tcp_reordering); if (icsk->icsk_ca_state <= TCP_CA_Disorder && - tp->sacked_out >= net->ipv4.sysctl_tcp_reordering) + tp->sacked_out >= reordering) tp->reordering = min_t(unsigned int, tp->reordering, - net->ipv4.sysctl_tcp_reordering); + reordering); + tcp_set_ca_state(sk, TCP_CA_Loss); tp->high_seq = tp->snd_nxt; tcp_ecn_queue_cwr(tp); @@ -3463,7 +3467,8 @@ static inline bool tcp_may_raise_cwnd(const struct sock *sk, const int flag) * new SACK or ECE mark may first advance cwnd here and later reduce * cwnd in tcp_fastretrans_alert() based on more states. */ - if (tcp_sk(sk)->reordering > sock_net(sk)->ipv4.sysctl_tcp_reordering) + if (tcp_sk(sk)->reordering > + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering)) return flag & FLAG_FORWARD_PROGRESS; return flag & FLAG_DATA_ACKED; @@ -4056,7 +4061,7 @@ void tcp_parse_options(const struct net *net, break; case TCPOPT_WINDOW: if (opsize == TCPOLEN_WINDOW && th->syn && - !estab && net->ipv4.sysctl_tcp_window_scaling) { + !estab && READ_ONCE(net->ipv4.sysctl_tcp_window_scaling)) { __u8 snd_wscale = *(__u8 *)ptr; opt_rx->wscale_ok = 1; if (snd_wscale > TCP_MAX_WSCALE) { @@ -4072,7 +4077,7 @@ void tcp_parse_options(const struct net *net, case TCPOPT_TIMESTAMP: if ((opsize == TCPOLEN_TIMESTAMP) && ((estab && opt_rx->tstamp_ok) || - (!estab && net->ipv4.sysctl_tcp_timestamps))) { + (!estab && READ_ONCE(net->ipv4.sysctl_tcp_timestamps)))) { opt_rx->saw_tstamp = 1; opt_rx->rcv_tsval = get_unaligned_be32(ptr); opt_rx->rcv_tsecr = get_unaligned_be32(ptr + 4); @@ -4080,7 +4085,7 @@ void tcp_parse_options(const struct net *net, break; case TCPOPT_SACK_PERM: if (opsize == TCPOLEN_SACK_PERM && th->syn && - !estab && net->ipv4.sysctl_tcp_sack) { + !estab && READ_ONCE(net->ipv4.sysctl_tcp_sack)) { opt_rx->sack_ok = TCP_SACK_SEEN; tcp_sack_reset(opt_rx); } @@ -5564,7 +5569,7 @@ static void tcp_check_urg(struct sock *sk, const struct tcphdr *th) struct tcp_sock *tp = tcp_sk(sk); u32 ptr = ntohs(th->urg_ptr); - if (ptr && !sock_net(sk)->ipv4.sysctl_tcp_stdurg) + if (ptr && !READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_stdurg)) ptr--; ptr += ntohl(th->seq); @@ -6794,11 +6799,14 @@ static bool tcp_syn_flood_action(const struct sock *sk, const char *proto) { struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue; const char *msg = "Dropping request"; - bool want_cookie = false; struct net *net = sock_net(sk); + bool want_cookie = false; + u8 syncookies; + + syncookies = READ_ONCE(net->ipv4.sysctl_tcp_syncookies); #ifdef CONFIG_SYN_COOKIES - if (net->ipv4.sysctl_tcp_syncookies) { + if (syncookies) { msg = "Sending cookies"; want_cookie = true; __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDOCOOKIES); @@ -6806,8 +6814,7 @@ static bool tcp_syn_flood_action(const struct sock *sk, const char *proto) #endif __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDROP); - if (!queue->synflood_warned && - net->ipv4.sysctl_tcp_syncookies != 2 && + if (!queue->synflood_warned && syncookies != 2 && xchg(&queue->synflood_warned, 1) == 0) net_info_ratelimited("%s: Possible SYN flooding on port %d. %s. Check SNMP counters.\n", proto, sk->sk_num, msg); @@ -6856,7 +6863,7 @@ u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops, struct tcp_sock *tp = tcp_sk(sk); u16 mss; - if (sock_net(sk)->ipv4.sysctl_tcp_syncookies != 2 && + if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies) != 2 && !inet_csk_reqsk_queue_is_full(sk)) return 0; @@ -6890,13 +6897,15 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, bool want_cookie = false; struct dst_entry *dst; struct flowi fl; + u8 syncookies; + + syncookies = READ_ONCE(net->ipv4.sysctl_tcp_syncookies); /* TW buckets are converted to open requests without * limitations, they conserve resources and peer is * evidently real one. */ - if ((net->ipv4.sysctl_tcp_syncookies == 2 || - inet_csk_reqsk_queue_is_full(sk)) && !isn) { + if ((syncookies == 2 || inet_csk_reqsk_queue_is_full(sk)) && !isn) { want_cookie = tcp_syn_flood_action(sk, rsk_ops->slab_name); if (!want_cookie) goto drop; @@ -6945,10 +6954,12 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, tcp_rsk(req)->ts_off = af_ops->init_ts_off(net, skb); if (!want_cookie && !isn) { + int max_syn_backlog = READ_ONCE(net->ipv4.sysctl_max_syn_backlog); + /* Kill the following clause, if you dislike this way. */ - if (!net->ipv4.sysctl_tcp_syncookies && - (net->ipv4.sysctl_max_syn_backlog - inet_csk_reqsk_queue_len(sk) < - (net->ipv4.sysctl_max_syn_backlog >> 2)) && + if (!syncookies && + (max_syn_backlog - inet_csk_reqsk_queue_len(sk) < + (max_syn_backlog >> 2)) && !tcp_peer_is_proven(req, dst)) { /* Without syncookies last quarter of * backlog is filled with destinations, diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 228d36692d08..c7e7101647dc 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -108,10 +108,10 @@ static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb) int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp) { + int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse); const struct inet_timewait_sock *tw = inet_twsk(sktw); const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw); struct tcp_sock *tp = tcp_sk(sk); - int reuse = sock_net(sk)->ipv4.sysctl_tcp_tw_reuse; if (reuse == 2) { /* Still does not detect *everything* that goes through diff --git a/net/ipv4/tcp_metrics.c b/net/ipv4/tcp_metrics.c index 7029b0e98edb..a501150deaa3 100644 --- a/net/ipv4/tcp_metrics.c +++ b/net/ipv4/tcp_metrics.c @@ -428,7 +428,8 @@ void tcp_update_metrics(struct sock *sk) if (!tcp_metric_locked(tm, TCP_METRIC_REORDERING)) { val = tcp_metric_get(tm, TCP_METRIC_REORDERING); if (val < tp->reordering && - tp->reordering != net->ipv4.sysctl_tcp_reordering) + tp->reordering != + READ_ONCE(net->ipv4.sysctl_tcp_reordering)) tcp_metric_set(tm, TCP_METRIC_REORDERING, tp->reordering); } diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index 6854bb1fb32b..cb95d88497ae 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -173,7 +173,7 @@ tcp_timewait_state_process(struct inet_timewait_sock *tw, struct sk_buff *skb, * Oh well... nobody has a sufficient solution to this * protocol bug yet. */ - if (twsk_net(tw)->ipv4.sysctl_tcp_rfc1337 == 0) { + if (!READ_ONCE(twsk_net(tw)->ipv4.sysctl_tcp_rfc1337)) { kill: inet_twsk_deschedule_put(tw); return TCP_TW_SUCCESS; @@ -781,7 +781,7 @@ listen_overflow: if (sk != req->rsk_listener) __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE); - if (!sock_net(sk)->ipv4.sysctl_tcp_abort_on_overflow) { + if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_abort_on_overflow)) { inet_rsk(req)->acked = 1; return NULL; } diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 858a15cc2cc9..2b72ccd2e651 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -791,18 +791,18 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, opts->mss = tcp_advertise_mss(sk); remaining -= TCPOLEN_MSS_ALIGNED; - if (likely(sock_net(sk)->ipv4.sysctl_tcp_timestamps && !*md5)) { + if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps) && !*md5)) { opts->options |= OPTION_TS; opts->tsval = tcp_skb_timestamp(skb) + tp->tsoffset; opts->tsecr = tp->rx_opt.ts_recent; remaining -= TCPOLEN_TSTAMP_ALIGNED; } - if (likely(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) { + if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling))) { opts->ws = tp->rx_opt.rcv_wscale; opts->options |= OPTION_WSCALE; remaining -= TCPOLEN_WSCALE_ALIGNED; } - if (likely(sock_net(sk)->ipv4.sysctl_tcp_sack)) { + if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_sack))) { opts->options |= OPTION_SACK_ADVERTISE; if (unlikely(!(OPTION_TS & opts->options))) remaining -= TCPOLEN_SACKPERM_ALIGNED; @@ -1719,7 +1719,8 @@ static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu) mss_now -= icsk->icsk_ext_hdr_len; /* Then reserve room for full set of TCP options and 8 bytes of data */ - mss_now = max(mss_now, sock_net(sk)->ipv4.sysctl_tcp_min_snd_mss); + mss_now = max(mss_now, + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_snd_mss)); return mss_now; } @@ -1762,10 +1763,10 @@ void tcp_mtup_init(struct sock *sk) struct inet_connection_sock *icsk = inet_csk(sk); struct net *net = sock_net(sk); - icsk->icsk_mtup.enabled = net->ipv4.sysctl_tcp_mtu_probing > 1; + icsk->icsk_mtup.enabled = READ_ONCE(net->ipv4.sysctl_tcp_mtu_probing) > 1; icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) + icsk->icsk_af_ops->net_header_len; - icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, net->ipv4.sysctl_tcp_base_mss); + icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, READ_ONCE(net->ipv4.sysctl_tcp_base_mss)); icsk->icsk_mtup.probe_size = 0; if (icsk->icsk_mtup.enabled) icsk->icsk_mtup.probe_timestamp = tcp_jiffies32; @@ -1897,7 +1898,7 @@ static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited) if (tp->packets_out > tp->snd_cwnd_used) tp->snd_cwnd_used = tp->packets_out; - if (sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle && + if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) && (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto && !ca_ops->cong_control) tcp_cwnd_application_limited(sk); @@ -2282,7 +2283,7 @@ static inline void tcp_mtu_check_reprobe(struct sock *sk) u32 interval; s32 delta; - interval = net->ipv4.sysctl_tcp_probe_interval; + interval = READ_ONCE(net->ipv4.sysctl_tcp_probe_interval); delta = tcp_jiffies32 - icsk->icsk_mtup.probe_timestamp; if (unlikely(delta >= interval * HZ)) { int mss = tcp_current_mss(sk); @@ -2366,7 +2367,7 @@ static int tcp_mtu_probe(struct sock *sk) * probing process by not resetting search range to its orignal. */ if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high) || - interval < net->ipv4.sysctl_tcp_probe_threshold) { + interval < READ_ONCE(net->ipv4.sysctl_tcp_probe_threshold)) { /* Check whether enough time has elaplased for * another round of probing. */ @@ -2740,7 +2741,7 @@ bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto) if (rcu_access_pointer(tp->fastopen_rsk)) return false; - early_retrans = sock_net(sk)->ipv4.sysctl_tcp_early_retrans; + early_retrans = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_early_retrans); /* Schedule a loss probe in 2*RTT for SACK capable connections * not in loss recovery, that are either limited by cwnd or application. */ @@ -3104,7 +3105,7 @@ static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to, struct sk_buff *skb = to, *tmp; bool first = true; - if (!sock_net(sk)->ipv4.sysctl_tcp_retrans_collapse) + if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_retrans_collapse)) return; if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN) return; @@ -3656,7 +3657,7 @@ static void tcp_connect_init(struct sock *sk) * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT. */ tp->tcp_header_len = sizeof(struct tcphdr); - if (sock_net(sk)->ipv4.sysctl_tcp_timestamps) + if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps)) tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED; #ifdef CONFIG_TCP_MD5SIG @@ -3692,7 +3693,7 @@ static void tcp_connect_init(struct sock *sk) tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0), &tp->rcv_wnd, &tp->window_clamp, - sock_net(sk)->ipv4.sysctl_tcp_window_scaling, + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling), &rcv_wscale, rcv_wnd); @@ -4099,7 +4100,7 @@ void tcp_send_probe0(struct sock *sk) icsk->icsk_probes_out++; if (err <= 0) { - if (icsk->icsk_backoff < net->ipv4.sysctl_tcp_retries2) + if (icsk->icsk_backoff < READ_ONCE(net->ipv4.sysctl_tcp_retries2)) icsk->icsk_backoff++; timeout = tcp_probe0_when(sk, TCP_RTO_MAX); } else { diff --git a/net/ipv4/tcp_recovery.c b/net/ipv4/tcp_recovery.c index 48f30e7209f2..50abaa941387 100644 --- a/net/ipv4/tcp_recovery.c +++ b/net/ipv4/tcp_recovery.c @@ -14,7 +14,8 @@ static u32 tcp_rack_reo_wnd(const struct sock *sk) return 0; if (tp->sacked_out >= tp->reordering && - !(sock_net(sk)->ipv4.sysctl_tcp_recovery & TCP_RACK_NO_DUPTHRESH)) + !(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_recovery) & + TCP_RACK_NO_DUPTHRESH)) return 0; } @@ -187,7 +188,8 @@ void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs) { struct tcp_sock *tp = tcp_sk(sk); - if (sock_net(sk)->ipv4.sysctl_tcp_recovery & TCP_RACK_STATIC_REO_WND || + if ((READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_recovery) & + TCP_RACK_STATIC_REO_WND) || !rs->prior_delivered) return; diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index 2208755e8efc..b4dfb82d6ecb 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -143,7 +143,7 @@ static int tcp_out_of_resources(struct sock *sk, bool do_reset) */ static int tcp_orphan_retries(struct sock *sk, bool alive) { - int retries = sock_net(sk)->ipv4.sysctl_tcp_orphan_retries; /* May be zero. */ + int retries = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_orphan_retries); /* May be zero. */ /* We know from an ICMP that something is wrong. */ if (sk->sk_err_soft && !alive) @@ -163,7 +163,7 @@ static void tcp_mtu_probing(struct inet_connection_sock *icsk, struct sock *sk) int mss; /* Black hole detection */ - if (!net->ipv4.sysctl_tcp_mtu_probing) + if (!READ_ONCE(net->ipv4.sysctl_tcp_mtu_probing)) return; if (!icsk->icsk_mtup.enabled) { @@ -171,9 +171,9 @@ static void tcp_mtu_probing(struct inet_connection_sock *icsk, struct sock *sk) icsk->icsk_mtup.probe_timestamp = tcp_jiffies32; } else { mss = tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low) >> 1; - mss = min(net->ipv4.sysctl_tcp_base_mss, mss); - mss = max(mss, net->ipv4.sysctl_tcp_mtu_probe_floor); - mss = max(mss, net->ipv4.sysctl_tcp_min_snd_mss); + mss = min(READ_ONCE(net->ipv4.sysctl_tcp_base_mss), mss); + mss = max(mss, READ_ONCE(net->ipv4.sysctl_tcp_mtu_probe_floor)); + mss = max(mss, READ_ONCE(net->ipv4.sysctl_tcp_min_snd_mss)); icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, mss); } tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); @@ -239,17 +239,18 @@ static int tcp_write_timeout(struct sock *sk) if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { if (icsk->icsk_retransmits) __dst_negative_advice(sk); - retry_until = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_syn_retries; + retry_until = icsk->icsk_syn_retries ? : + READ_ONCE(net->ipv4.sysctl_tcp_syn_retries); expired = icsk->icsk_retransmits >= retry_until; } else { - if (retransmits_timed_out(sk, net->ipv4.sysctl_tcp_retries1, 0)) { + if (retransmits_timed_out(sk, READ_ONCE(net->ipv4.sysctl_tcp_retries1), 0)) { /* Black hole detection */ tcp_mtu_probing(icsk, sk); __dst_negative_advice(sk); } - retry_until = net->ipv4.sysctl_tcp_retries2; + retry_until = READ_ONCE(net->ipv4.sysctl_tcp_retries2); if (sock_flag(sk, SOCK_DEAD)) { const bool alive = icsk->icsk_rto < TCP_RTO_MAX; @@ -374,7 +375,7 @@ static void tcp_probe_timer(struct sock *sk) msecs_to_jiffies(icsk->icsk_user_timeout)) goto abort; - max_probes = sock_net(sk)->ipv4.sysctl_tcp_retries2; + max_probes = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_retries2); if (sock_flag(sk, SOCK_DEAD)) { const bool alive = inet_csk_rto_backoff(icsk, TCP_RTO_MAX) < TCP_RTO_MAX; @@ -400,12 +401,15 @@ abort: tcp_write_err(sk); static void tcp_fastopen_synack_timer(struct sock *sk, struct request_sock *req) { struct inet_connection_sock *icsk = inet_csk(sk); - int max_retries = icsk->icsk_syn_retries ? : - sock_net(sk)->ipv4.sysctl_tcp_synack_retries + 1; /* add one more retry for fastopen */ struct tcp_sock *tp = tcp_sk(sk); + int max_retries; req->rsk_ops->syn_ack_timeout(req); + /* add one more retry for fastopen */ + max_retries = icsk->icsk_syn_retries ? : + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_synack_retries) + 1; + if (req->num_timeout >= max_retries) { tcp_write_err(sk); return; @@ -568,7 +572,7 @@ out_reset_timer: * linear-timeout retransmissions into a black hole */ if (sk->sk_state == TCP_ESTABLISHED && - (tp->thin_lto || net->ipv4.sysctl_tcp_thin_linear_timeouts) && + (tp->thin_lto || READ_ONCE(net->ipv4.sysctl_tcp_thin_linear_timeouts)) && tcp_stream_is_thin(tp) && icsk->icsk_retransmits <= TCP_THIN_LINEAR_RETRIES) { icsk->icsk_backoff = 0; @@ -579,7 +583,7 @@ out_reset_timer: } inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, tcp_clamp_rto_to_user_timeout(sk), TCP_RTO_MAX); - if (retransmits_timed_out(sk, net->ipv4.sysctl_tcp_retries1 + 1, 0)) + if (retransmits_timed_out(sk, READ_ONCE(net->ipv4.sysctl_tcp_retries1) + 1, 0)) __sk_dst_reset(sk); out:; diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 88becb037eb6..b624e3d8c5f0 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -7043,8 +7043,8 @@ static const struct ctl_table addrconf_sysctl[] = { .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec_minmax, - .extra1 = (void *)SYSCTL_ZERO, - .extra2 = (void *)SYSCTL_ONE, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, }, { /* sentinel */ diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 0ee0770e79aa..2ce0c44d0081 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -227,7 +227,7 @@ lookup_protocol: RCU_INIT_POINTER(inet->mc_list, NULL); inet->rcv_tos = 0; - if (net->ipv4.sysctl_ip_no_pmtu_disc) + if (READ_ONCE(net->ipv4.sysctl_ip_no_pmtu_disc)) inet->pmtudisc = IP_PMTUDISC_DONT; else inet->pmtudisc = IP_PMTUDISC_WANT; diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index 36e1d0f8dd06..8220923a12f7 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -343,7 +343,7 @@ static struct ip_esp_hdr *esp_output_set_esn(struct sk_buff *skb, struct esp_output_extra *extra) { /* For ESN we move the header forward by 4 bytes to - * accomodate the high bits. We will move it back after + * accommodate the high bits. We will move it back after * encryption. */ if ((x->props.flags & XFRM_STATE_ESN)) { @@ -896,7 +896,7 @@ static void esp_input_set_header(struct sk_buff *skb, __be32 *seqhi) struct xfrm_state *x = xfrm_input_state(skb); /* For ESN we move the header forward by 4 bytes to - * accomodate the high bits. We will move it back after + * accommodate the high bits. We will move it back after * decryption. */ if ((x->props.flags & XFRM_STATE_ESN)) { diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c index 0322cc86b84e..e1ebf5e42ebe 100644 --- a/net/ipv6/ip6_input.c +++ b/net/ipv6/ip6_input.c @@ -45,20 +45,23 @@ #include <net/inet_ecn.h> #include <net/dst_metadata.h> -INDIRECT_CALLABLE_DECLARE(void tcp_v6_early_demux(struct sk_buff *)); static void ip6_rcv_finish_core(struct net *net, struct sock *sk, struct sk_buff *skb) { - void (*edemux)(struct sk_buff *skb); - - if (net->ipv4.sysctl_ip_early_demux && !skb_dst(skb) && skb->sk == NULL) { - const struct inet6_protocol *ipprot; - - ipprot = rcu_dereference(inet6_protos[ipv6_hdr(skb)->nexthdr]); - if (ipprot && (edemux = READ_ONCE(ipprot->early_demux))) - INDIRECT_CALL_2(edemux, tcp_v6_early_demux, - udp_v6_early_demux, skb); + if (READ_ONCE(net->ipv4.sysctl_ip_early_demux) && + !skb_dst(skb) && !skb->sk) { + switch (ipv6_hdr(skb)->nexthdr) { + case IPPROTO_TCP: + if (READ_ONCE(net->ipv4.sysctl_tcp_early_demux)) + tcp_v6_early_demux(skb); + break; + case IPPROTO_UDP: + if (READ_ONCE(net->ipv4.sysctl_udp_early_demux)) + udp_v6_early_demux(skb); + break; + } } + if (!skb_valid_dst(skb)) ip6_route_input(skb); } diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 77e3f5970ce4..897ca4f9b791 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1464,6 +1464,7 @@ static int __ip6_append_data(struct sock *sk, int copy; int err; int offset = 0; + bool zc = false; u32 tskey = 0; struct rt6_info *rt = (struct rt6_info *)cork->dst; struct ipv6_txoptions *opt = v6_cork->opt; @@ -1541,17 +1542,35 @@ emsgsize: rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) csummode = CHECKSUM_PARTIAL; - if (flags & MSG_ZEROCOPY && length && sock_flag(sk, SOCK_ZEROCOPY)) { - uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb)); - if (!uarg) - return -ENOBUFS; - extra_uref = !skb_zcopy(skb); /* only ref on new uarg */ - if (rt->dst.dev->features & NETIF_F_SG && - csummode == CHECKSUM_PARTIAL) { - paged = true; - } else { - uarg->zerocopy = 0; - skb_zcopy_set(skb, uarg, &extra_uref); + if ((flags & MSG_ZEROCOPY) && length) { + struct msghdr *msg = from; + + if (getfrag == ip_generic_getfrag && msg->msg_ubuf) { + if (skb_zcopy(skb) && msg->msg_ubuf != skb_zcopy(skb)) + return -EINVAL; + + /* Leave uarg NULL if can't zerocopy, callers should + * be able to handle it. + */ + if ((rt->dst.dev->features & NETIF_F_SG) && + csummode == CHECKSUM_PARTIAL) { + paged = true; + zc = true; + uarg = msg->msg_ubuf; + } + } else if (sock_flag(sk, SOCK_ZEROCOPY)) { + uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb)); + if (!uarg) + return -ENOBUFS; + extra_uref = !skb_zcopy(skb); /* only ref on new uarg */ + if (rt->dst.dev->features & NETIF_F_SG && + csummode == CHECKSUM_PARTIAL) { + paged = true; + zc = true; + } else { + uarg->zerocopy = 0; + skb_zcopy_set(skb, uarg, &extra_uref); + } } } @@ -1630,9 +1649,12 @@ alloc_new_skb: (fraglen + alloc_extra < SKB_MAX_ALLOC || !(rt->dst.dev->features & NETIF_F_SG))) alloclen = fraglen; - else { + else if (!zc) { alloclen = min_t(int, fraglen, MAX_HEADER); pagedlen = fraglen - alloclen; + } else { + alloclen = fragheaderlen + transhdrlen; + pagedlen = datalen - transhdrlen; } alloclen += alloc_extra; @@ -1742,13 +1764,14 @@ alloc_new_skb: err = -EFAULT; goto error; } - } else if (!uarg || !uarg->zerocopy) { + } else if (!zc) { int i = skb_shinfo(skb)->nr_frags; err = -ENOMEM; if (!sk_page_frag_refill(sk, pfrag)) goto error; + skb_zcopy_downgrade_managed(skb); if (!skb_can_coalesce(skb, i, pfrag->page, pfrag->offset)) { err = -EMSGSIZE; diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index cd84cbdac0a2..98453693e400 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -967,6 +967,25 @@ out: in6_dev_put(idev); } +static int accept_untracked_na(struct net_device *dev, struct in6_addr *saddr) +{ + struct inet6_dev *idev = __in6_dev_get(dev); + + switch (idev->cnf.accept_untracked_na) { + case 0: /* Don't accept untracked na (absent in neighbor cache) */ + return 0; + case 1: /* Create new entries from na if currently untracked */ + return 1; + case 2: /* Create new entries from untracked na only if saddr is in the + * same subnet as an address configured on the interface that + * received the na + */ + return !!ipv6_chk_prefix(saddr, dev); + default: + return 0; + } +} + static void ndisc_recv_na(struct sk_buff *skb) { struct nd_msg *msg = (struct nd_msg *)skb_transport_header(skb); @@ -1061,11 +1080,11 @@ static void ndisc_recv_na(struct sk_buff *skb) * Note that we don't do a (daddr == all-routers-mcast) check. */ new_state = msg->icmph.icmp6_solicited ? NUD_REACHABLE : NUD_STALE; - if (!neigh && lladdr && - idev && idev->cnf.forwarding && - idev->cnf.accept_untracked_na) { - neigh = neigh_create(&nd_tbl, &msg->target, dev); - new_state = NUD_STALE; + if (!neigh && lladdr && idev && idev->cnf.forwarding) { + if (accept_untracked_na(dev, saddr)) { + neigh = neigh_create(&nd_tbl, &msg->target, dev); + new_state = NUD_STALE; + } } if (neigh && !IS_ERR(neigh)) { diff --git a/net/ipv6/ping.c b/net/ipv6/ping.c index ecf3a553a0dc..b1179f62bd23 100644 --- a/net/ipv6/ping.c +++ b/net/ipv6/ping.c @@ -64,6 +64,8 @@ static int ping_v6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) if (err) return err; + memset(&fl6, 0, sizeof(fl6)); + if (msg->msg_name) { DECLARE_SOCKADDR(struct sockaddr_in6 *, u, msg->msg_name); if (msg->msg_namelen < sizeof(*u)) @@ -72,12 +74,15 @@ static int ping_v6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) return -EAFNOSUPPORT; } daddr = &(u->sin6_addr); + if (np->sndflow) + fl6.flowlabel = u->sin6_flowinfo & IPV6_FLOWINFO_MASK; if (__ipv6_addr_needs_scope_id(ipv6_addr_type(daddr))) oif = u->sin6_scope_id; } else { if (sk->sk_state != TCP_ESTABLISHED) return -EDESTADDRREQ; daddr = &sk->sk_v6_daddr; + fl6.flowlabel = np->flow_label; } if (!oif) @@ -101,7 +106,6 @@ static int ping_v6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) ipc6.sockc.tsflags = sk->sk_tsflags; ipc6.sockc.mark = sk->sk_mark; - memset(&fl6, 0, sizeof(fl6)); fl6.flowi6_oif = oif; if (msg->msg_controllen) { diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c index 9cc123f000fb..5014aa663452 100644 --- a/net/ipv6/syncookies.c +++ b/net/ipv6/syncookies.c @@ -141,7 +141,8 @@ struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb) __u8 rcv_wscale; u32 tsoff = 0; - if (!sock_net(sk)->ipv4.sysctl_tcp_syncookies || !th->ack || th->rst) + if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies) || + !th->ack || th->rst) goto out; if (tcp_synq_no_recent_overflow(sk)) diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 70d4890d8d2f..85b8b765dcb1 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -1825,7 +1825,7 @@ do_time_wait: goto discard_it; } -INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb) +void tcp_v6_early_demux(struct sk_buff *skb) { const struct ipv6hdr *hdr; const struct tcphdr *th; @@ -2182,12 +2182,7 @@ struct proto tcpv6_prot = { }; EXPORT_SYMBOL_GPL(tcpv6_prot); -/* thinking of making this const? Don't. - * early_demux can change based on sysctl. - */ -static struct inet6_protocol tcpv6_protocol = { - .early_demux = tcp_v6_early_demux, - .early_demux_handler = tcp_v6_early_demux, +static const struct inet6_protocol tcpv6_protocol = { .handler = tcp_v6_rcv, .err_handler = tcp_v6_err, .flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL, diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index be074f07073a..16c176e7c69a 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -1052,7 +1052,7 @@ static struct sock *__udp6_lib_demux_lookup(struct net *net, return NULL; } -INDIRECT_CALLABLE_SCOPE void udp_v6_early_demux(struct sk_buff *skb) +void udp_v6_early_demux(struct sk_buff *skb) { struct net *net = dev_net(skb->dev); const struct udphdr *uh; @@ -1660,12 +1660,7 @@ int udpv6_getsockopt(struct sock *sk, int level, int optname, return ipv6_getsockopt(sk, level, optname, optval, optlen); } -/* thinking of making this const? Don't. - * early_demux can change based on sysctl. - */ -static struct inet6_protocol udpv6_protocol = { - .early_demux = udp_v6_early_demux, - .early_demux_handler = udp_v6_early_demux, +static const struct inet6_protocol udpv6_protocol = { .handler = udpv6_rcv, .err_handler = udpv6_err, .flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL, diff --git a/net/key/af_key.c b/net/key/af_key.c index fb16d7c4e1b8..fda2dcc8a383 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c @@ -3894,14 +3894,10 @@ static int __init ipsec_pfkey_init(void) err = sock_register(&pfkey_family_ops); if (err != 0) goto out_unregister_pernet; - err = xfrm_register_km(&pfkeyv2_mgr); - if (err != 0) - goto out_sock_unregister; + xfrm_register_km(&pfkeyv2_mgr); out: return err; -out_sock_unregister: - sock_unregister(PF_KEY); out_unregister_pernet: unregister_pernet_subsys(&pfkey_net_ops); out_unregister_key_proto: diff --git a/net/netfilter/Kconfig b/net/netfilter/Kconfig index ddc54b6d18ee..df6abbfe0079 100644 --- a/net/netfilter/Kconfig +++ b/net/netfilter/Kconfig @@ -734,6 +734,15 @@ config NF_FLOW_TABLE To compile it as a module, choose M here. +config NF_FLOW_TABLE_PROCFS + bool "Supply flow table statistics in procfs" + default y + depends on PROC_FS + depends on SYSCTL + help + This option enables for the flow table offload statistics + to be shown in procfs under net/netfilter/nf_flowtable. + config NETFILTER_XTABLES tristate "Netfilter Xtables support (required for ip_tables)" default m if NETFILTER_ADVANCED=n diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile index 238b6a620e88..06df49ea6329 100644 --- a/net/netfilter/Makefile +++ b/net/netfilter/Makefile @@ -128,6 +128,7 @@ obj-$(CONFIG_NFT_FWD_NETDEV) += nft_fwd_netdev.o obj-$(CONFIG_NF_FLOW_TABLE) += nf_flow_table.o nf_flow_table-objs := nf_flow_table_core.o nf_flow_table_ip.o \ nf_flow_table_offload.o +nf_flow_table-$(CONFIG_NF_FLOW_TABLE_PROCFS) += nf_flow_table_procfs.o obj-$(CONFIG_NF_FLOW_TABLE_INET) += nf_flow_table_inet.o diff --git a/net/netfilter/ipvs/ip_vs_mh.c b/net/netfilter/ipvs/ip_vs_mh.c index da0280cec506..e3d7f5c879ce 100644 --- a/net/netfilter/ipvs/ip_vs_mh.c +++ b/net/netfilter/ipvs/ip_vs_mh.c @@ -174,8 +174,7 @@ static int ip_vs_mh_populate(struct ip_vs_mh_state *s, return 0; } - table = kcalloc(BITS_TO_LONGS(IP_VS_MH_TAB_SIZE), - sizeof(unsigned long), GFP_KERNEL); + table = bitmap_zalloc(IP_VS_MH_TAB_SIZE, GFP_KERNEL); if (!table) return -ENOMEM; @@ -227,7 +226,7 @@ static int ip_vs_mh_populate(struct ip_vs_mh_state *s, } out: - kfree(table); + bitmap_free(table); return 0; } diff --git a/net/netfilter/nf_conntrack_bpf.c b/net/netfilter/nf_conntrack_bpf.c index bc4d5cd63a94..1cd87b28c9b0 100644 --- a/net/netfilter/nf_conntrack_bpf.c +++ b/net/netfilter/nf_conntrack_bpf.c @@ -55,57 +55,131 @@ enum { NF_BPF_CT_OPTS_SZ = 12, }; -static struct nf_conn *__bpf_nf_ct_lookup(struct net *net, - struct bpf_sock_tuple *bpf_tuple, - u32 tuple_len, u8 protonum, - s32 netns_id, u8 *dir) +static int bpf_nf_ct_tuple_parse(struct bpf_sock_tuple *bpf_tuple, + u32 tuple_len, u8 protonum, u8 dir, + struct nf_conntrack_tuple *tuple) { - struct nf_conntrack_tuple_hash *hash; - struct nf_conntrack_tuple tuple; - struct nf_conn *ct; + union nf_inet_addr *src = dir ? &tuple->dst.u3 : &tuple->src.u3; + union nf_inet_addr *dst = dir ? &tuple->src.u3 : &tuple->dst.u3; + union nf_conntrack_man_proto *sport = dir ? (void *)&tuple->dst.u + : &tuple->src.u; + union nf_conntrack_man_proto *dport = dir ? &tuple->src.u + : (void *)&tuple->dst.u; if (unlikely(protonum != IPPROTO_TCP && protonum != IPPROTO_UDP)) - return ERR_PTR(-EPROTO); - if (unlikely(netns_id < BPF_F_CURRENT_NETNS)) - return ERR_PTR(-EINVAL); + return -EPROTO; + + memset(tuple, 0, sizeof(*tuple)); - memset(&tuple, 0, sizeof(tuple)); switch (tuple_len) { case sizeof(bpf_tuple->ipv4): - tuple.src.l3num = AF_INET; - tuple.src.u3.ip = bpf_tuple->ipv4.saddr; - tuple.src.u.tcp.port = bpf_tuple->ipv4.sport; - tuple.dst.u3.ip = bpf_tuple->ipv4.daddr; - tuple.dst.u.tcp.port = bpf_tuple->ipv4.dport; + tuple->src.l3num = AF_INET; + src->ip = bpf_tuple->ipv4.saddr; + sport->tcp.port = bpf_tuple->ipv4.sport; + dst->ip = bpf_tuple->ipv4.daddr; + dport->tcp.port = bpf_tuple->ipv4.dport; break; case sizeof(bpf_tuple->ipv6): - tuple.src.l3num = AF_INET6; - memcpy(tuple.src.u3.ip6, bpf_tuple->ipv6.saddr, sizeof(bpf_tuple->ipv6.saddr)); - tuple.src.u.tcp.port = bpf_tuple->ipv6.sport; - memcpy(tuple.dst.u3.ip6, bpf_tuple->ipv6.daddr, sizeof(bpf_tuple->ipv6.daddr)); - tuple.dst.u.tcp.port = bpf_tuple->ipv6.dport; + tuple->src.l3num = AF_INET6; + memcpy(src->ip6, bpf_tuple->ipv6.saddr, sizeof(bpf_tuple->ipv6.saddr)); + sport->tcp.port = bpf_tuple->ipv6.sport; + memcpy(dst->ip6, bpf_tuple->ipv6.daddr, sizeof(bpf_tuple->ipv6.daddr)); + dport->tcp.port = bpf_tuple->ipv6.dport; break; default: - return ERR_PTR(-EAFNOSUPPORT); + return -EAFNOSUPPORT; + } + tuple->dst.protonum = protonum; + tuple->dst.dir = dir; + + return 0; +} + +static struct nf_conn * +__bpf_nf_ct_alloc_entry(struct net *net, struct bpf_sock_tuple *bpf_tuple, + u32 tuple_len, struct bpf_ct_opts *opts, u32 opts_len, + u32 timeout) +{ + struct nf_conntrack_tuple otuple, rtuple; + struct nf_conn *ct; + int err; + + if (!opts || !bpf_tuple || opts->reserved[0] || opts->reserved[1] || + opts_len != NF_BPF_CT_OPTS_SZ) + return ERR_PTR(-EINVAL); + + if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS)) + return ERR_PTR(-EINVAL); + + err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto, + IP_CT_DIR_ORIGINAL, &otuple); + if (err < 0) + return ERR_PTR(err); + + err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto, + IP_CT_DIR_REPLY, &rtuple); + if (err < 0) + return ERR_PTR(err); + + if (opts->netns_id >= 0) { + net = get_net_ns_by_id(net, opts->netns_id); + if (unlikely(!net)) + return ERR_PTR(-ENONET); } - tuple.dst.protonum = protonum; + ct = nf_conntrack_alloc(net, &nf_ct_zone_dflt, &otuple, &rtuple, + GFP_ATOMIC); + if (IS_ERR(ct)) + goto out; + + memset(&ct->proto, 0, sizeof(ct->proto)); + __nf_ct_set_timeout(ct, timeout * HZ); + ct->status |= IPS_CONFIRMED; + +out: + if (opts->netns_id >= 0) + put_net(net); + + return ct; +} + +static struct nf_conn *__bpf_nf_ct_lookup(struct net *net, + struct bpf_sock_tuple *bpf_tuple, + u32 tuple_len, struct bpf_ct_opts *opts, + u32 opts_len) +{ + struct nf_conntrack_tuple_hash *hash; + struct nf_conntrack_tuple tuple; + struct nf_conn *ct; + int err; + + if (!opts || !bpf_tuple || opts->reserved[0] || opts->reserved[1] || + opts_len != NF_BPF_CT_OPTS_SZ) + return ERR_PTR(-EINVAL); + if (unlikely(opts->l4proto != IPPROTO_TCP && opts->l4proto != IPPROTO_UDP)) + return ERR_PTR(-EPROTO); + if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS)) + return ERR_PTR(-EINVAL); + + err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto, + IP_CT_DIR_ORIGINAL, &tuple); + if (err < 0) + return ERR_PTR(err); - if (netns_id >= 0) { - net = get_net_ns_by_id(net, netns_id); + if (opts->netns_id >= 0) { + net = get_net_ns_by_id(net, opts->netns_id); if (unlikely(!net)) return ERR_PTR(-ENONET); } hash = nf_conntrack_find_get(net, &nf_ct_zone_dflt, &tuple); - if (netns_id >= 0) + if (opts->netns_id >= 0) put_net(net); if (!hash) return ERR_PTR(-ENOENT); ct = nf_ct_tuplehash_to_ctrack(hash); - if (dir) - *dir = NF_CT_DIRECTION(hash); + opts->dir = NF_CT_DIRECTION(hash); return ct; } @@ -114,6 +188,43 @@ __diag_push(); __diag_ignore_all("-Wmissing-prototypes", "Global functions as their definitions will be in nf_conntrack BTF"); +struct nf_conn___init { + struct nf_conn ct; +}; + +/* bpf_xdp_ct_alloc - Allocate a new CT entry + * + * Parameters: + * @xdp_ctx - Pointer to ctx (xdp_md) in XDP program + * Cannot be NULL + * @bpf_tuple - Pointer to memory representing the tuple to look up + * Cannot be NULL + * @tuple__sz - Length of the tuple structure + * Must be one of sizeof(bpf_tuple->ipv4) or + * sizeof(bpf_tuple->ipv6) + * @opts - Additional options for allocation (documented above) + * Cannot be NULL + * @opts__sz - Length of the bpf_ct_opts structure + * Must be NF_BPF_CT_OPTS_SZ (12) + */ +struct nf_conn___init * +bpf_xdp_ct_alloc(struct xdp_md *xdp_ctx, struct bpf_sock_tuple *bpf_tuple, + u32 tuple__sz, struct bpf_ct_opts *opts, u32 opts__sz) +{ + struct xdp_buff *ctx = (struct xdp_buff *)xdp_ctx; + struct nf_conn *nfct; + + nfct = __bpf_nf_ct_alloc_entry(dev_net(ctx->rxq->dev), bpf_tuple, tuple__sz, + opts, opts__sz, 10); + if (IS_ERR(nfct)) { + if (opts) + opts->error = PTR_ERR(nfct); + return NULL; + } + + return (struct nf_conn___init *)nfct; +} + /* bpf_xdp_ct_lookup - Lookup CT entry for the given tuple, and acquire a * reference to it * @@ -138,25 +249,50 @@ bpf_xdp_ct_lookup(struct xdp_md *xdp_ctx, struct bpf_sock_tuple *bpf_tuple, struct net *caller_net; struct nf_conn *nfct; - BUILD_BUG_ON(sizeof(struct bpf_ct_opts) != NF_BPF_CT_OPTS_SZ); - - if (!opts) - return NULL; - if (!bpf_tuple || opts->reserved[0] || opts->reserved[1] || - opts__sz != NF_BPF_CT_OPTS_SZ) { - opts->error = -EINVAL; - return NULL; - } caller_net = dev_net(ctx->rxq->dev); - nfct = __bpf_nf_ct_lookup(caller_net, bpf_tuple, tuple__sz, opts->l4proto, - opts->netns_id, &opts->dir); + nfct = __bpf_nf_ct_lookup(caller_net, bpf_tuple, tuple__sz, opts, opts__sz); if (IS_ERR(nfct)) { - opts->error = PTR_ERR(nfct); + if (opts) + opts->error = PTR_ERR(nfct); return NULL; } return nfct; } +/* bpf_skb_ct_alloc - Allocate a new CT entry + * + * Parameters: + * @skb_ctx - Pointer to ctx (__sk_buff) in TC program + * Cannot be NULL + * @bpf_tuple - Pointer to memory representing the tuple to look up + * Cannot be NULL + * @tuple__sz - Length of the tuple structure + * Must be one of sizeof(bpf_tuple->ipv4) or + * sizeof(bpf_tuple->ipv6) + * @opts - Additional options for allocation (documented above) + * Cannot be NULL + * @opts__sz - Length of the bpf_ct_opts structure + * Must be NF_BPF_CT_OPTS_SZ (12) + */ +struct nf_conn___init * +bpf_skb_ct_alloc(struct __sk_buff *skb_ctx, struct bpf_sock_tuple *bpf_tuple, + u32 tuple__sz, struct bpf_ct_opts *opts, u32 opts__sz) +{ + struct sk_buff *skb = (struct sk_buff *)skb_ctx; + struct nf_conn *nfct; + struct net *net; + + net = skb->dev ? dev_net(skb->dev) : sock_net(skb->sk); + nfct = __bpf_nf_ct_alloc_entry(net, bpf_tuple, tuple__sz, opts, opts__sz, 10); + if (IS_ERR(nfct)) { + if (opts) + opts->error = PTR_ERR(nfct); + return NULL; + } + + return (struct nf_conn___init *)nfct; +} + /* bpf_skb_ct_lookup - Lookup CT entry for the given tuple, and acquire a * reference to it * @@ -181,20 +317,31 @@ bpf_skb_ct_lookup(struct __sk_buff *skb_ctx, struct bpf_sock_tuple *bpf_tuple, struct net *caller_net; struct nf_conn *nfct; - BUILD_BUG_ON(sizeof(struct bpf_ct_opts) != NF_BPF_CT_OPTS_SZ); - - if (!opts) - return NULL; - if (!bpf_tuple || opts->reserved[0] || opts->reserved[1] || - opts__sz != NF_BPF_CT_OPTS_SZ) { - opts->error = -EINVAL; - return NULL; - } caller_net = skb->dev ? dev_net(skb->dev) : sock_net(skb->sk); - nfct = __bpf_nf_ct_lookup(caller_net, bpf_tuple, tuple__sz, opts->l4proto, - opts->netns_id, &opts->dir); + nfct = __bpf_nf_ct_lookup(caller_net, bpf_tuple, tuple__sz, opts, opts__sz); if (IS_ERR(nfct)) { - opts->error = PTR_ERR(nfct); + if (opts) + opts->error = PTR_ERR(nfct); + return NULL; + } + return nfct; +} + +/* bpf_ct_insert_entry - Add the provided entry into a CT map + * + * This must be invoked for referenced PTR_TO_BTF_ID. + * + * @nfct - Pointer to referenced nf_conn___init object, obtained + * using bpf_xdp_ct_alloc or bpf_skb_ct_alloc. + */ +struct nf_conn *bpf_ct_insert_entry(struct nf_conn___init *nfct_i) +{ + struct nf_conn *nfct = (struct nf_conn *)nfct_i; + int err; + + err = nf_conntrack_hash_check_insert(nfct); + if (err < 0) { + nf_conntrack_free(nfct); return NULL; } return nfct; @@ -217,50 +364,90 @@ void bpf_ct_release(struct nf_conn *nfct) nf_ct_put(nfct); } +/* bpf_ct_set_timeout - Set timeout of allocated nf_conn + * + * Sets the default timeout of newly allocated nf_conn before insertion. + * This helper must be invoked for refcounted pointer to nf_conn___init. + * + * Parameters: + * @nfct - Pointer to referenced nf_conn object, obtained using + * bpf_xdp_ct_alloc or bpf_skb_ct_alloc. + * @timeout - Timeout in msecs. + */ +void bpf_ct_set_timeout(struct nf_conn___init *nfct, u32 timeout) +{ + __nf_ct_set_timeout((struct nf_conn *)nfct, msecs_to_jiffies(timeout)); +} + +/* bpf_ct_change_timeout - Change timeout of inserted nf_conn + * + * Change timeout associated of the inserted or looked up nf_conn. + * This helper must be invoked for refcounted pointer to nf_conn. + * + * Parameters: + * @nfct - Pointer to referenced nf_conn object, obtained using + * bpf_ct_insert_entry, bpf_xdp_ct_lookup, or bpf_skb_ct_lookup. + * @timeout - New timeout in msecs. + */ +int bpf_ct_change_timeout(struct nf_conn *nfct, u32 timeout) +{ + return __nf_ct_change_timeout(nfct, msecs_to_jiffies(timeout)); +} + +/* bpf_ct_set_status - Set status field of allocated nf_conn + * + * Set the status field of the newly allocated nf_conn before insertion. + * This must be invoked for referenced PTR_TO_BTF_ID to nf_conn___init. + * + * Parameters: + * @nfct - Pointer to referenced nf_conn object, obtained using + * bpf_xdp_ct_alloc or bpf_skb_ct_alloc. + * @status - New status value. + */ +int bpf_ct_set_status(const struct nf_conn___init *nfct, u32 status) +{ + return nf_ct_change_status_common((struct nf_conn *)nfct, status); +} + +/* bpf_ct_change_status - Change status of inserted nf_conn + * + * Change the status field of the provided connection tracking entry. + * This must be invoked for referenced PTR_TO_BTF_ID to nf_conn. + * + * Parameters: + * @nfct - Pointer to referenced nf_conn object, obtained using + * bpf_ct_insert_entry, bpf_xdp_ct_lookup or bpf_skb_ct_lookup. + * @status - New status value. + */ +int bpf_ct_change_status(struct nf_conn *nfct, u32 status) +{ + return nf_ct_change_status_common(nfct, status); +} + __diag_pop() -BTF_SET_START(nf_ct_xdp_check_kfunc_ids) -BTF_ID(func, bpf_xdp_ct_lookup) -BTF_ID(func, bpf_ct_release) -BTF_SET_END(nf_ct_xdp_check_kfunc_ids) - -BTF_SET_START(nf_ct_tc_check_kfunc_ids) -BTF_ID(func, bpf_skb_ct_lookup) -BTF_ID(func, bpf_ct_release) -BTF_SET_END(nf_ct_tc_check_kfunc_ids) - -BTF_SET_START(nf_ct_acquire_kfunc_ids) -BTF_ID(func, bpf_xdp_ct_lookup) -BTF_ID(func, bpf_skb_ct_lookup) -BTF_SET_END(nf_ct_acquire_kfunc_ids) - -BTF_SET_START(nf_ct_release_kfunc_ids) -BTF_ID(func, bpf_ct_release) -BTF_SET_END(nf_ct_release_kfunc_ids) - -/* Both sets are identical */ -#define nf_ct_ret_null_kfunc_ids nf_ct_acquire_kfunc_ids - -static const struct btf_kfunc_id_set nf_conntrack_xdp_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &nf_ct_xdp_check_kfunc_ids, - .acquire_set = &nf_ct_acquire_kfunc_ids, - .release_set = &nf_ct_release_kfunc_ids, - .ret_null_set = &nf_ct_ret_null_kfunc_ids, -}; +BTF_SET8_START(nf_ct_kfunc_set) +BTF_ID_FLAGS(func, bpf_xdp_ct_alloc, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_xdp_ct_lookup, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_skb_ct_alloc, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_skb_ct_lookup, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_ct_insert_entry, KF_ACQUIRE | KF_RET_NULL | KF_RELEASE) +BTF_ID_FLAGS(func, bpf_ct_release, KF_RELEASE) +BTF_ID_FLAGS(func, bpf_ct_set_timeout, KF_TRUSTED_ARGS) +BTF_ID_FLAGS(func, bpf_ct_change_timeout, KF_TRUSTED_ARGS) +BTF_ID_FLAGS(func, bpf_ct_set_status, KF_TRUSTED_ARGS) +BTF_ID_FLAGS(func, bpf_ct_change_status, KF_TRUSTED_ARGS) +BTF_SET8_END(nf_ct_kfunc_set) -static const struct btf_kfunc_id_set nf_conntrack_tc_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &nf_ct_tc_check_kfunc_ids, - .acquire_set = &nf_ct_acquire_kfunc_ids, - .release_set = &nf_ct_release_kfunc_ids, - .ret_null_set = &nf_ct_ret_null_kfunc_ids, +static const struct btf_kfunc_id_set nf_conntrack_kfunc_set = { + .owner = THIS_MODULE, + .set = &nf_ct_kfunc_set, }; int register_nf_conntrack_bpf(void) { int ret; - ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &nf_conntrack_xdp_kfunc_set); - return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &nf_conntrack_tc_kfunc_set); + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &nf_conntrack_kfunc_set); + return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &nf_conntrack_kfunc_set); } diff --git a/net/netfilter/nf_conntrack_broadcast.c b/net/netfilter/nf_conntrack_broadcast.c index 1ba6becc3079..9fb9b8031298 100644 --- a/net/netfilter/nf_conntrack_broadcast.c +++ b/net/netfilter/nf_conntrack_broadcast.c @@ -20,6 +20,7 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, enum ip_conntrack_info ctinfo, unsigned int timeout) { + const struct nf_conntrack_helper *helper; struct nf_conntrack_expect *exp; struct iphdr *iph = ip_hdr(skb); struct rtable *rt = skb_rtable(skb); @@ -58,7 +59,10 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, goto out; exp->tuple = ct->tuplehash[IP_CT_DIR_REPLY].tuple; - exp->tuple.src.u.udp.port = help->helper->tuple.src.u.udp.port; + + helper = rcu_dereference(help->helper); + if (helper) + exp->tuple.src.u.udp.port = helper->tuple.src.u.udp.port; exp->mask.src.u3.ip = mask; exp->mask.src.u.udp.port = htons(0xFFFF); diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index 369aeabb94fe..71c2f4f95d36 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -329,20 +329,18 @@ nf_ct_get_tuple(const struct sk_buff *skb, return gre_pkt_to_tuple(skb, dataoff, net, tuple); #endif case IPPROTO_TCP: - case IPPROTO_UDP: /* fallthrough */ - return nf_ct_get_tuple_ports(skb, dataoff, tuple); + case IPPROTO_UDP: #ifdef CONFIG_NF_CT_PROTO_UDPLITE case IPPROTO_UDPLITE: - return nf_ct_get_tuple_ports(skb, dataoff, tuple); #endif #ifdef CONFIG_NF_CT_PROTO_SCTP case IPPROTO_SCTP: - return nf_ct_get_tuple_ports(skb, dataoff, tuple); #endif #ifdef CONFIG_NF_CT_PROTO_DCCP case IPPROTO_DCCP: - return nf_ct_get_tuple_ports(skb, dataoff, tuple); #endif + /* fallthrough */ + return nf_ct_get_tuple_ports(skb, dataoff, tuple); default: break; } @@ -2808,3 +2806,65 @@ err_expect: free_percpu(net->ct.stat); return ret; } + +#if (IS_BUILTIN(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \ + (IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES) || \ + IS_ENABLED(CONFIG_NF_CT_NETLINK)) + +/* ctnetlink code shared by both ctnetlink and nf_conntrack_bpf */ + +int __nf_ct_change_timeout(struct nf_conn *ct, u64 timeout) +{ + if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status)) + return -EPERM; + + __nf_ct_set_timeout(ct, timeout); + + if (test_bit(IPS_DYING_BIT, &ct->status)) + return -ETIME; + + return 0; +} +EXPORT_SYMBOL_GPL(__nf_ct_change_timeout); + +void __nf_ct_change_status(struct nf_conn *ct, unsigned long on, unsigned long off) +{ + unsigned int bit; + + /* Ignore these unchangable bits */ + on &= ~IPS_UNCHANGEABLE_MASK; + off &= ~IPS_UNCHANGEABLE_MASK; + + for (bit = 0; bit < __IPS_MAX_BIT; bit++) { + if (on & (1 << bit)) + set_bit(bit, &ct->status); + else if (off & (1 << bit)) + clear_bit(bit, &ct->status); + } +} +EXPORT_SYMBOL_GPL(__nf_ct_change_status); + +int nf_ct_change_status_common(struct nf_conn *ct, unsigned int status) +{ + unsigned long d; + + d = ct->status ^ status; + + if (d & (IPS_EXPECTED|IPS_CONFIRMED|IPS_DYING)) + /* unchangeable */ + return -EBUSY; + + if (d & IPS_SEEN_REPLY && !(status & IPS_SEEN_REPLY)) + /* SEEN_REPLY bit can only be set */ + return -EBUSY; + + if (d & IPS_ASSURED && !(status & IPS_ASSURED)) + /* ASSURED bit can only be set */ + return -EBUSY; + + __nf_ct_change_status(ct, status, 0); + return 0; +} +EXPORT_SYMBOL_GPL(nf_ct_change_status_common); + +#endif diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c index 2eb31ffb3d14..bb76305bb7ff 100644 --- a/net/netfilter/nf_conntrack_h323_main.c +++ b/net/netfilter/nf_conntrack_h323_main.c @@ -49,64 +49,8 @@ MODULE_PARM_DESC(callforward_filter, "only create call forwarding expectations " "if both endpoints are on different sides " "(determined by routing information)"); -/* Hooks for NAT */ -int (*set_h245_addr_hook) (struct sk_buff *skb, unsigned int protoff, - unsigned char **data, int dataoff, - H245_TransportAddress *taddr, - union nf_inet_addr *addr, __be16 port) - __read_mostly; -int (*set_h225_addr_hook) (struct sk_buff *skb, unsigned int protoff, - unsigned char **data, int dataoff, - TransportAddress *taddr, - union nf_inet_addr *addr, __be16 port) - __read_mostly; -int (*set_sig_addr_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, unsigned char **data, - TransportAddress *taddr, int count) __read_mostly; -int (*set_ras_addr_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, unsigned char **data, - TransportAddress *taddr, int count) __read_mostly; -int (*nat_rtp_rtcp_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, int dataoff, - H245_TransportAddress *taddr, - __be16 port, __be16 rtp_port, - struct nf_conntrack_expect *rtp_exp, - struct nf_conntrack_expect *rtcp_exp) __read_mostly; -int (*nat_t120_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, int dataoff, - H245_TransportAddress *taddr, __be16 port, - struct nf_conntrack_expect *exp) __read_mostly; -int (*nat_h245_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, int dataoff, - TransportAddress *taddr, __be16 port, - struct nf_conntrack_expect *exp) __read_mostly; -int (*nat_callforwarding_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, int dataoff, - TransportAddress *taddr, __be16 port, - struct nf_conntrack_expect *exp) __read_mostly; -int (*nat_q931_hook) (struct sk_buff *skb, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned char **data, TransportAddress *taddr, int idx, - __be16 port, struct nf_conntrack_expect *exp) - __read_mostly; +const struct nfct_h323_nat_hooks __rcu *nfct_h323_nat_hook __read_mostly; +EXPORT_SYMBOL_GPL(nfct_h323_nat_hook); static DEFINE_SPINLOCK(nf_h323_lock); static char *h323_buffer; @@ -259,6 +203,7 @@ static int expect_rtp_rtcp(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, int dataoff, H245_TransportAddress *taddr) { + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret = 0; __be16 port; @@ -266,7 +211,6 @@ static int expect_rtp_rtcp(struct sk_buff *skb, struct nf_conn *ct, union nf_inet_addr addr; struct nf_conntrack_expect *rtp_exp; struct nf_conntrack_expect *rtcp_exp; - typeof(nat_rtp_rtcp_hook) nat_rtp_rtcp; /* Read RTP or RTCP address */ if (!get_h245_addr(ct, *data, taddr, &addr, &port) || @@ -296,15 +240,16 @@ static int expect_rtp_rtcp(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.dst.u3, IPPROTO_UDP, NULL, &rtcp_port); + nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, sizeof(ct->tuplehash[dir].tuple.src.u3)) && - (nat_rtp_rtcp = rcu_dereference(nat_rtp_rtcp_hook)) && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* NAT needed */ - ret = nat_rtp_rtcp(skb, ct, ctinfo, protoff, data, dataoff, - taddr, port, rtp_port, rtp_exp, rtcp_exp); + ret = nathook->nat_rtp_rtcp(skb, ct, ctinfo, protoff, data, dataoff, + taddr, port, rtp_port, rtp_exp, rtcp_exp); } else { /* Conntrack only */ if (nf_ct_expect_related(rtp_exp, 0) == 0) { if (nf_ct_expect_related(rtcp_exp, 0) == 0) { @@ -333,12 +278,12 @@ static int expect_t120(struct sk_buff *skb, unsigned char **data, int dataoff, H245_TransportAddress *taddr) { + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret = 0; __be16 port; union nf_inet_addr addr; struct nf_conntrack_expect *exp; - typeof(nat_t120_hook) nat_t120; /* Read T.120 address */ if (!get_h245_addr(ct, *data, taddr, &addr, &port) || @@ -355,15 +300,16 @@ static int expect_t120(struct sk_buff *skb, IPPROTO_TCP, NULL, &port); exp->flags = NF_CT_EXPECT_PERMANENT; /* Accept multiple channels */ + nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, sizeof(ct->tuplehash[dir].tuple.src.u3)) && - (nat_t120 = rcu_dereference(nat_t120_hook)) && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* NAT needed */ - ret = nat_t120(skb, ct, ctinfo, protoff, data, dataoff, taddr, - port, exp); + ret = nathook->nat_t120(skb, ct, ctinfo, protoff, data, + dataoff, taddr, port, exp); } else { /* Conntrack only */ if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_h323: expect T.120 "); @@ -664,18 +610,19 @@ int get_h225_addr(struct nf_conn *ct, unsigned char *data, return 1; } +EXPORT_SYMBOL_GPL(get_h225_addr); static int expect_h245(struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info ctinfo, unsigned int protoff, unsigned char **data, int dataoff, TransportAddress *taddr) { + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret = 0; __be16 port; union nf_inet_addr addr; struct nf_conntrack_expect *exp; - typeof(nat_h245_hook) nat_h245; /* Read h245Address */ if (!get_h225_addr(ct, *data, taddr, &addr, &port) || @@ -692,15 +639,16 @@ static int expect_h245(struct sk_buff *skb, struct nf_conn *ct, IPPROTO_TCP, NULL, &port); exp->helper = &nf_conntrack_helper_h245; + nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, sizeof(ct->tuplehash[dir].tuple.src.u3)) && - (nat_h245 = rcu_dereference(nat_h245_hook)) && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* NAT needed */ - ret = nat_h245(skb, ct, ctinfo, protoff, data, dataoff, taddr, - port, exp); + ret = nathook->nat_h245(skb, ct, ctinfo, protoff, data, + dataoff, taddr, port, exp); } else { /* Conntrack only */ if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_q931: expect H.245 "); @@ -785,13 +733,13 @@ static int expect_callforwarding(struct sk_buff *skb, unsigned char **data, int dataoff, TransportAddress *taddr) { + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret = 0; __be16 port; union nf_inet_addr addr; struct nf_conntrack_expect *exp; struct net *net = nf_ct_net(ct); - typeof(nat_callforwarding_hook) nat_callforwarding; /* Read alternativeAddress */ if (!get_h225_addr(ct, *data, taddr, &addr, &port) || port == 0) @@ -815,16 +763,17 @@ static int expect_callforwarding(struct sk_buff *skb, IPPROTO_TCP, NULL, &port); exp->helper = nf_conntrack_helper_q931; + nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, sizeof(ct->tuplehash[dir].tuple.src.u3)) && - (nat_callforwarding = rcu_dereference(nat_callforwarding_hook)) && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* Need NAT */ - ret = nat_callforwarding(skb, ct, ctinfo, - protoff, data, dataoff, - taddr, port, exp); + ret = nathook->nat_callforwarding(skb, ct, ctinfo, + protoff, data, dataoff, + taddr, port, exp); } else { /* Conntrack only */ if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_q931: expect Call Forwarding "); @@ -844,12 +793,12 @@ static int process_setup(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, int dataoff, Setup_UUIE *setup) { + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret; int i; __be16 port; union nf_inet_addr addr; - typeof(set_h225_addr_hook) set_h225_addr; pr_debug("nf_ct_q931: Setup\n"); @@ -860,9 +809,9 @@ static int process_setup(struct sk_buff *skb, struct nf_conn *ct, return -1; } - set_h225_addr = rcu_dereference(set_h225_addr_hook); + nathook = rcu_dereference(nfct_h323_nat_hook); if ((setup->options & eSetup_UUIE_destCallSignalAddress) && - (set_h225_addr) && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK && get_h225_addr(ct, *data, &setup->destCallSignalAddress, &addr, &port) && @@ -870,16 +819,16 @@ static int process_setup(struct sk_buff *skb, struct nf_conn *ct, pr_debug("nf_ct_q931: set destCallSignalAddress %pI6:%hu->%pI6:%hu\n", &addr, ntohs(port), &ct->tuplehash[!dir].tuple.src.u3, ntohs(ct->tuplehash[!dir].tuple.src.u.tcp.port)); - ret = set_h225_addr(skb, protoff, data, dataoff, - &setup->destCallSignalAddress, - &ct->tuplehash[!dir].tuple.src.u3, - ct->tuplehash[!dir].tuple.src.u.tcp.port); + ret = nathook->set_h225_addr(skb, protoff, data, dataoff, + &setup->destCallSignalAddress, + &ct->tuplehash[!dir].tuple.src.u3, + ct->tuplehash[!dir].tuple.src.u.tcp.port); if (ret < 0) return -1; } if ((setup->options & eSetup_UUIE_sourceCallSignalAddress) && - (set_h225_addr) && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK && get_h225_addr(ct, *data, &setup->sourceCallSignalAddress, &addr, &port) && @@ -887,10 +836,10 @@ static int process_setup(struct sk_buff *skb, struct nf_conn *ct, pr_debug("nf_ct_q931: set sourceCallSignalAddress %pI6:%hu->%pI6:%hu\n", &addr, ntohs(port), &ct->tuplehash[!dir].tuple.dst.u3, ntohs(ct->tuplehash[!dir].tuple.dst.u.tcp.port)); - ret = set_h225_addr(skb, protoff, data, dataoff, - &setup->sourceCallSignalAddress, - &ct->tuplehash[!dir].tuple.dst.u3, - ct->tuplehash[!dir].tuple.dst.u.tcp.port); + ret = nathook->set_h225_addr(skb, protoff, data, dataoff, + &setup->sourceCallSignalAddress, + &ct->tuplehash[!dir].tuple.dst.u3, + ct->tuplehash[!dir].tuple.dst.u.tcp.port); if (ret < 0) return -1; } @@ -1249,13 +1198,13 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct, TransportAddress *taddr, int count) { struct nf_ct_h323_master *info = nfct_help_data(ct); + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret = 0; int i; __be16 port; union nf_inet_addr addr; struct nf_conntrack_expect *exp; - typeof(nat_q931_hook) nat_q931; /* Look for the first related address */ for (i = 0; i < count; i++) { @@ -1279,11 +1228,11 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct, exp->helper = nf_conntrack_helper_q931; exp->flags = NF_CT_EXPECT_PERMANENT; /* Accept multiple calls */ - nat_q931 = rcu_dereference(nat_q931_hook); - if (nat_q931 && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* Need NAT */ - ret = nat_q931(skb, ct, ctinfo, protoff, data, - taddr, i, port, exp); + ret = nathook->nat_q931(skb, ct, ctinfo, protoff, data, + taddr, i, port, exp); } else { /* Conntrack only */ if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect Q.931 "); @@ -1305,15 +1254,15 @@ static int process_grq(struct sk_buff *skb, struct nf_conn *ct, unsigned int protoff, unsigned char **data, GatekeeperRequest *grq) { - typeof(set_ras_addr_hook) set_ras_addr; + const struct nfct_h323_nat_hooks *nathook; pr_debug("nf_ct_ras: GRQ\n"); - set_ras_addr = rcu_dereference(set_ras_addr_hook); - if (set_ras_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) /* NATed */ - return set_ras_addr(skb, ct, ctinfo, protoff, data, - &grq->rasAddress, 1); + return nathook->set_ras_addr(skb, ct, ctinfo, protoff, data, + &grq->rasAddress, 1); return 0; } @@ -1367,8 +1316,8 @@ static int process_rrq(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, RegistrationRequest *rrq) { struct nf_ct_h323_master *info = nfct_help_data(ct); + const struct nfct_h323_nat_hooks *nathook; int ret; - typeof(set_ras_addr_hook) set_ras_addr; pr_debug("nf_ct_ras: RRQ\n"); @@ -1378,12 +1327,12 @@ static int process_rrq(struct sk_buff *skb, struct nf_conn *ct, if (ret < 0) return -1; - set_ras_addr = rcu_dereference(set_ras_addr_hook); - if (set_ras_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { - ret = set_ras_addr(skb, ct, ctinfo, protoff, data, - rrq->rasAddress.item, - rrq->rasAddress.count); + ret = nathook->set_ras_addr(skb, ct, ctinfo, protoff, data, + rrq->rasAddress.item, + rrq->rasAddress.count); if (ret < 0) return -1; } @@ -1403,19 +1352,19 @@ static int process_rcf(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, RegistrationConfirm *rcf) { struct nf_ct_h323_master *info = nfct_help_data(ct); + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret; struct nf_conntrack_expect *exp; - typeof(set_sig_addr_hook) set_sig_addr; pr_debug("nf_ct_ras: RCF\n"); - set_sig_addr = rcu_dereference(set_sig_addr_hook); - if (set_sig_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { - ret = set_sig_addr(skb, ct, ctinfo, protoff, data, - rcf->callSignalAddress.item, - rcf->callSignalAddress.count); + ret = nathook->set_sig_addr(skb, ct, ctinfo, protoff, data, + rcf->callSignalAddress.item, + rcf->callSignalAddress.count); if (ret < 0) return -1; } @@ -1454,18 +1403,18 @@ static int process_urq(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, UnregistrationRequest *urq) { struct nf_ct_h323_master *info = nfct_help_data(ct); + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); int ret; - typeof(set_sig_addr_hook) set_sig_addr; pr_debug("nf_ct_ras: URQ\n"); - set_sig_addr = rcu_dereference(set_sig_addr_hook); - if (set_sig_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { - ret = set_sig_addr(skb, ct, ctinfo, protoff, data, - urq->callSignalAddress.item, - urq->callSignalAddress.count); + ret = nathook->set_sig_addr(skb, ct, ctinfo, protoff, data, + urq->callSignalAddress.item, + urq->callSignalAddress.count); if (ret < 0) return -1; } @@ -1487,39 +1436,42 @@ static int process_arq(struct sk_buff *skb, struct nf_conn *ct, unsigned char **data, AdmissionRequest *arq) { const struct nf_ct_h323_master *info = nfct_help_data(ct); + const struct nfct_h323_nat_hooks *nathook; int dir = CTINFO2DIR(ctinfo); __be16 port; union nf_inet_addr addr; - typeof(set_h225_addr_hook) set_h225_addr; pr_debug("nf_ct_ras: ARQ\n"); - set_h225_addr = rcu_dereference(set_h225_addr_hook); + nathook = rcu_dereference(nfct_h323_nat_hook); + if (!nathook) + return 0; + if ((arq->options & eAdmissionRequest_destCallSignalAddress) && get_h225_addr(ct, *data, &arq->destCallSignalAddress, &addr, &port) && !memcmp(&addr, &ct->tuplehash[dir].tuple.src.u3, sizeof(addr)) && port == info->sig_port[dir] && nf_ct_l3num(ct) == NFPROTO_IPV4 && - set_h225_addr && ct->status & IPS_NAT_MASK) { + ct->status & IPS_NAT_MASK) { /* Answering ARQ */ - return set_h225_addr(skb, protoff, data, 0, - &arq->destCallSignalAddress, - &ct->tuplehash[!dir].tuple.dst.u3, - info->sig_port[!dir]); + return nathook->set_h225_addr(skb, protoff, data, 0, + &arq->destCallSignalAddress, + &ct->tuplehash[!dir].tuple.dst.u3, + info->sig_port[!dir]); } if ((arq->options & eAdmissionRequest_srcCallSignalAddress) && get_h225_addr(ct, *data, &arq->srcCallSignalAddress, &addr, &port) && !memcmp(&addr, &ct->tuplehash[dir].tuple.src.u3, sizeof(addr)) && - set_h225_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { /* Calling ARQ */ - return set_h225_addr(skb, protoff, data, 0, - &arq->srcCallSignalAddress, - &ct->tuplehash[!dir].tuple.dst.u3, - port); + return nathook->set_h225_addr(skb, protoff, data, 0, + &arq->srcCallSignalAddress, + &ct->tuplehash[!dir].tuple.dst.u3, + port); } return 0; @@ -1535,7 +1487,6 @@ static int process_acf(struct sk_buff *skb, struct nf_conn *ct, __be16 port; union nf_inet_addr addr; struct nf_conntrack_expect *exp; - typeof(set_sig_addr_hook) set_sig_addr; pr_debug("nf_ct_ras: ACF\n"); @@ -1544,12 +1495,15 @@ static int process_acf(struct sk_buff *skb, struct nf_conn *ct, return 0; if (!memcmp(&addr, &ct->tuplehash[dir].tuple.dst.u3, sizeof(addr))) { + const struct nfct_h323_nat_hooks *nathook; + /* Answering ACF */ - set_sig_addr = rcu_dereference(set_sig_addr_hook); - if (set_sig_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) - return set_sig_addr(skb, ct, ctinfo, protoff, data, - &acf->destCallSignalAddress, 1); + return nathook->set_sig_addr(skb, ct, ctinfo, protoff, + data, + &acf->destCallSignalAddress, 1); return 0; } @@ -1578,15 +1532,15 @@ static int process_lrq(struct sk_buff *skb, struct nf_conn *ct, unsigned int protoff, unsigned char **data, LocationRequest *lrq) { - typeof(set_ras_addr_hook) set_ras_addr; + const struct nfct_h323_nat_hooks *nathook; pr_debug("nf_ct_ras: LRQ\n"); - set_ras_addr = rcu_dereference(set_ras_addr_hook); - if (set_ras_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) - return set_ras_addr(skb, ct, ctinfo, protoff, data, - &lrq->replyAddress, 1); + return nathook->set_ras_addr(skb, ct, ctinfo, protoff, data, + &lrq->replyAddress, 1); return 0; } @@ -1634,27 +1588,22 @@ static int process_irr(struct sk_buff *skb, struct nf_conn *ct, unsigned int protoff, unsigned char **data, InfoRequestResponse *irr) { + const struct nfct_h323_nat_hooks *nathook; int ret; - typeof(set_ras_addr_hook) set_ras_addr; - typeof(set_sig_addr_hook) set_sig_addr; pr_debug("nf_ct_ras: IRR\n"); - set_ras_addr = rcu_dereference(set_ras_addr_hook); - if (set_ras_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && + nathook = rcu_dereference(nfct_h323_nat_hook); + if (nathook && nf_ct_l3num(ct) == NFPROTO_IPV4 && ct->status & IPS_NAT_MASK) { - ret = set_ras_addr(skb, ct, ctinfo, protoff, data, - &irr->rasAddress, 1); + ret = nathook->set_ras_addr(skb, ct, ctinfo, protoff, data, + &irr->rasAddress, 1); if (ret < 0) return -1; - } - set_sig_addr = rcu_dereference(set_sig_addr_hook); - if (set_sig_addr && nf_ct_l3num(ct) == NFPROTO_IPV4 && - ct->status & IPS_NAT_MASK) { - ret = set_sig_addr(skb, ct, ctinfo, protoff, data, - irr->callSignalAddress.item, - irr->callSignalAddress.count); + ret = nathook->set_sig_addr(skb, ct, ctinfo, protoff, data, + irr->callSignalAddress.item, + irr->callSignalAddress.count); if (ret < 0) return -1; } @@ -1837,17 +1786,6 @@ err1: module_init(nf_conntrack_h323_init); module_exit(nf_conntrack_h323_fini); -EXPORT_SYMBOL_GPL(get_h225_addr); -EXPORT_SYMBOL_GPL(set_h245_addr_hook); -EXPORT_SYMBOL_GPL(set_h225_addr_hook); -EXPORT_SYMBOL_GPL(set_sig_addr_hook); -EXPORT_SYMBOL_GPL(set_ras_addr_hook); -EXPORT_SYMBOL_GPL(nat_rtp_rtcp_hook); -EXPORT_SYMBOL_GPL(nat_t120_hook); -EXPORT_SYMBOL_GPL(nat_h245_hook); -EXPORT_SYMBOL_GPL(nat_callforwarding_hook); -EXPORT_SYMBOL_GPL(nat_q931_hook); - MODULE_AUTHOR("Jing Min Zhao <zhaojingmin@users.sourceforge.net>"); MODULE_DESCRIPTION("H.323 connection tracking helper"); MODULE_LICENSE("GPL"); diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c index c12a87ebc3ee..e96b32221444 100644 --- a/net/netfilter/nf_conntrack_helper.c +++ b/net/netfilter/nf_conntrack_helper.c @@ -165,7 +165,7 @@ nf_nat_helper_try_module_get(const char *name, u16 l3num, u8 protonum) if (!nat) { snprintf(mod_name, sizeof(mod_name), "%s", h->nat_mod_name); rcu_read_unlock(); - request_module(mod_name); + request_module("%s", mod_name); rcu_read_lock(); nat = nf_conntrack_nat_helper_find(mod_name); @@ -249,7 +249,7 @@ int __nf_ct_try_assign_helper(struct nf_conn *ct, struct nf_conn *tmpl, if (tmpl != NULL) { help = nfct_help(tmpl); if (help != NULL) { - helper = help->helper; + helper = rcu_dereference(help->helper); set_bit(IPS_HELPER_BIT, &ct->status); } } diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index f5905b5201a7..04169b54f2a2 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -1891,45 +1891,10 @@ ctnetlink_parse_nat_setup(struct nf_conn *ct, } #endif -static void -__ctnetlink_change_status(struct nf_conn *ct, unsigned long on, - unsigned long off) -{ - unsigned int bit; - - /* Ignore these unchangable bits */ - on &= ~IPS_UNCHANGEABLE_MASK; - off &= ~IPS_UNCHANGEABLE_MASK; - - for (bit = 0; bit < __IPS_MAX_BIT; bit++) { - if (on & (1 << bit)) - set_bit(bit, &ct->status); - else if (off & (1 << bit)) - clear_bit(bit, &ct->status); - } -} - static int ctnetlink_change_status(struct nf_conn *ct, const struct nlattr * const cda[]) { - unsigned long d; - unsigned int status = ntohl(nla_get_be32(cda[CTA_STATUS])); - d = ct->status ^ status; - - if (d & (IPS_EXPECTED|IPS_CONFIRMED|IPS_DYING)) - /* unchangeable */ - return -EBUSY; - - if (d & IPS_SEEN_REPLY && !(status & IPS_SEEN_REPLY)) - /* SEEN_REPLY bit can only be set */ - return -EBUSY; - - if (d & IPS_ASSURED && !(status & IPS_ASSURED)) - /* ASSURED bit can only be set */ - return -EBUSY; - - __ctnetlink_change_status(ct, status, 0); - return 0; + return nf_ct_change_status_common(ct, ntohl(nla_get_be32(cda[CTA_STATUS]))); } static int @@ -2005,7 +1970,7 @@ static int ctnetlink_change_helper(struct nf_conn *ct, } if (help) { - if (help->helper == helper) { + if (rcu_access_pointer(help->helper) == helper) { /* update private helper data if allowed. */ if (helper->from_nlattr) helper->from_nlattr(helpinfo, ct); @@ -2024,16 +1989,7 @@ static int ctnetlink_change_helper(struct nf_conn *ct, static int ctnetlink_change_timeout(struct nf_conn *ct, const struct nlattr * const cda[]) { - u64 timeout = (u64)ntohl(nla_get_be32(cda[CTA_TIMEOUT])) * HZ; - - if (timeout > INT_MAX) - timeout = INT_MAX; - WRITE_ONCE(ct->timeout, nfct_time_stamp + (u32)timeout); - - if (test_bit(IPS_DYING_BIT, &ct->status)) - return -ETIME; - - return 0; + return __nf_ct_change_timeout(ct, (u64)ntohl(nla_get_be32(cda[CTA_TIMEOUT])) * HZ); } #if defined(CONFIG_NF_CONNTRACK_MARK) @@ -2293,9 +2249,7 @@ ctnetlink_create_conntrack(struct net *net, goto err1; timeout = (u64)ntohl(nla_get_be32(cda[CTA_TIMEOUT])) * HZ; - if (timeout > INT_MAX) - timeout = INT_MAX; - ct->timeout = (u32)timeout + nfct_time_stamp; + __nf_ct_set_timeout(ct, timeout); rcu_read_lock(); if (cda[CTA_HELP]) { @@ -2837,7 +2791,7 @@ ctnetlink_update_status(struct nf_conn *ct, const struct nlattr * const cda[]) * unchangeable bits but do not error out. Also user programs * are allowed to clear the bits that they are allowed to change. */ - __ctnetlink_change_status(ct, status, ~status); + __nf_ct_change_status(ct, status, ~status); return 0; } @@ -3413,12 +3367,17 @@ static int ctnetlink_get_expect(struct sk_buff *skb, static bool expect_iter_name(struct nf_conntrack_expect *exp, void *data) { + struct nf_conntrack_helper *helper; const struct nf_conn_help *m_help; const char *name = data; m_help = nfct_help(exp->master); - return strcmp(m_help->helper->name, name) == 0; + helper = rcu_dereference(m_help->helper); + if (!helper) + return false; + + return strcmp(helper->name, name) == 0; } static bool expect_iter_all(struct nf_conntrack_expect *exp, void *data) diff --git a/net/netfilter/nf_conntrack_pptp.c b/net/netfilter/nf_conntrack_pptp.c index f3fa367b455f..4c679638df06 100644 --- a/net/netfilter/nf_conntrack_pptp.c +++ b/net/netfilter/nf_conntrack_pptp.c @@ -45,7 +45,7 @@ MODULE_ALIAS_NFCT_HELPER("pptp"); static DEFINE_SPINLOCK(nf_pptp_lock); -const struct nf_nat_pptp_hook *nf_nat_pptp_hook; +const struct nf_nat_pptp_hook __rcu *nf_nat_pptp_hook; EXPORT_SYMBOL_GPL(nf_nat_pptp_hook); #if defined(DEBUG) || defined(CONFIG_DYNAMIC_DEBUG) diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index b83dc9bf0a5d..daf06f71d31c 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -60,7 +60,7 @@ module_param(sip_external_media, int, 0600); MODULE_PARM_DESC(sip_external_media, "Expect Media streams between external " "endpoints (default 0)"); -const struct nf_nat_sip_hooks *nf_nat_sip_hooks; +const struct nf_nat_sip_hooks __rcu *nf_nat_sip_hooks; EXPORT_SYMBOL_GPL(nf_nat_sip_hooks); static int string_len(const struct nf_conn *ct, const char *dptr, @@ -1229,6 +1229,7 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, struct nf_conntrack_expect *exp; union nf_inet_addr *saddr, daddr; const struct nf_nat_sip_hooks *hooks; + struct nf_conntrack_helper *helper; __be16 port; u8 proto; unsigned int expires = 0; @@ -1289,10 +1290,14 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, if (sip_direct_signalling) saddr = &ct->tuplehash[!dir].tuple.src.u3; + helper = rcu_dereference(nfct_help(ct)->helper); + if (!helper) + return NF_DROP; + nf_ct_expect_init(exp, SIP_EXPECT_SIGNALLING, nf_ct_l3num(ct), saddr, &daddr, proto, NULL, &port); exp->timeout.expires = sip_timeout * HZ; - exp->helper = nfct_help(ct)->helper; + exp->helper = helper; exp->flags = NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE; hooks = rcu_dereference(nf_nat_sip_hooks); diff --git a/net/netfilter/nf_conntrack_timeout.c b/net/netfilter/nf_conntrack_timeout.c index 0f828d05ea60..0cc584d3dbb1 100644 --- a/net/netfilter/nf_conntrack_timeout.c +++ b/net/netfilter/nf_conntrack_timeout.c @@ -22,15 +22,21 @@ #include <net/netfilter/nf_conntrack_l4proto.h> #include <net/netfilter/nf_conntrack_timeout.h> -const struct nf_ct_timeout_hooks *nf_ct_timeout_hook __read_mostly; +const struct nf_ct_timeout_hooks __rcu *nf_ct_timeout_hook __read_mostly; EXPORT_SYMBOL_GPL(nf_ct_timeout_hook); static int untimeout(struct nf_conn *ct, void *timeout) { struct nf_conn_timeout *timeout_ext = nf_ct_timeout_find(ct); - if (timeout_ext && (!timeout || timeout_ext->timeout == timeout)) - RCU_INIT_POINTER(timeout_ext->timeout, NULL); + if (timeout_ext) { + const struct nf_ct_timeout *t; + + t = rcu_access_pointer(timeout_ext->timeout); + + if (!timeout || t == timeout) + RCU_INIT_POINTER(timeout_ext->timeout, NULL); + } /* We are not intended to delete this conntrack. */ return 0; @@ -127,7 +133,11 @@ void nf_ct_destroy_timeout(struct nf_conn *ct) if (h) { timeout_ext = nf_ct_timeout_find(ct); if (timeout_ext) { - h->timeout_put(timeout_ext->timeout); + struct nf_ct_timeout *t; + + t = rcu_dereference(timeout_ext->timeout); + if (t) + h->timeout_put(t); RCU_INIT_POINTER(timeout_ext->timeout, NULL); } } diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c index f2def06d1070..765ac779bfc8 100644 --- a/net/netfilter/nf_flow_table_core.c +++ b/net/netfilter/nf_flow_table_core.c @@ -53,14 +53,14 @@ struct flow_offload *flow_offload_alloc(struct nf_conn *ct) { struct flow_offload *flow; - if (unlikely(nf_ct_is_dying(ct) || - !refcount_inc_not_zero(&ct->ct_general.use))) + if (unlikely(nf_ct_is_dying(ct))) return NULL; flow = kzalloc(sizeof(*flow), GFP_ATOMIC); if (!flow) - goto err_ct_refcnt; + return NULL; + refcount_inc(&ct->ct_general.use); flow->ct = ct; flow_offload_fill_dir(flow, FLOW_OFFLOAD_DIR_ORIGINAL); @@ -72,11 +72,6 @@ struct flow_offload *flow_offload_alloc(struct nf_conn *ct) __set_bit(NF_FLOW_DNAT, &flow->flags); return flow; - -err_ct_refcnt: - nf_ct_put(ct); - - return NULL; } EXPORT_SYMBOL_GPL(flow_offload_alloc); @@ -614,14 +609,74 @@ void nf_flow_table_free(struct nf_flowtable *flow_table) } EXPORT_SYMBOL_GPL(nf_flow_table_free); +static int nf_flow_table_init_net(struct net *net) +{ + net->ft.stat = alloc_percpu(struct nf_flow_table_stat); + return net->ft.stat ? 0 : -ENOMEM; +} + +static void nf_flow_table_fini_net(struct net *net) +{ + free_percpu(net->ft.stat); +} + +static int nf_flow_table_pernet_init(struct net *net) +{ + int ret; + + ret = nf_flow_table_init_net(net); + if (ret < 0) + return ret; + + ret = nf_flow_table_init_proc(net); + if (ret < 0) + goto out_proc; + + return 0; + +out_proc: + nf_flow_table_fini_net(net); + return ret; +} + +static void nf_flow_table_pernet_exit(struct list_head *net_exit_list) +{ + struct net *net; + + list_for_each_entry(net, net_exit_list, exit_list) { + nf_flow_table_fini_proc(net); + nf_flow_table_fini_net(net); + } +} + +static struct pernet_operations nf_flow_table_net_ops = { + .init = nf_flow_table_pernet_init, + .exit_batch = nf_flow_table_pernet_exit, +}; + static int __init nf_flow_table_module_init(void) { - return nf_flow_table_offload_init(); + int ret; + + ret = register_pernet_subsys(&nf_flow_table_net_ops); + if (ret < 0) + return ret; + + ret = nf_flow_table_offload_init(); + if (ret) + goto out_offload; + + return 0; + +out_offload: + unregister_pernet_subsys(&nf_flow_table_net_ops); + return ret; } static void __exit nf_flow_table_module_exit(void) { nf_flow_table_offload_exit(); + unregister_pernet_subsys(&nf_flow_table_net_ops); } module_init(nf_flow_table_module_init); diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c index 11b6e1942092..103b6cbf257f 100644 --- a/net/netfilter/nf_flow_table_offload.c +++ b/net/netfilter/nf_flow_table_offload.c @@ -967,17 +967,22 @@ static void flow_offload_work_stats(struct flow_offload_work *offload) static void flow_offload_work_handler(struct work_struct *work) { struct flow_offload_work *offload; + struct net *net; offload = container_of(work, struct flow_offload_work, work); + net = read_pnet(&offload->flowtable->net); switch (offload->cmd) { case FLOW_CLS_REPLACE: flow_offload_work_add(offload); + NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_add); break; case FLOW_CLS_DESTROY: flow_offload_work_del(offload); + NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_del); break; case FLOW_CLS_STATS: flow_offload_work_stats(offload); + NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_stats); break; default: WARN_ON_ONCE(1); @@ -989,12 +994,18 @@ static void flow_offload_work_handler(struct work_struct *work) static void flow_offload_queue_work(struct flow_offload_work *offload) { - if (offload->cmd == FLOW_CLS_REPLACE) + struct net *net = read_pnet(&offload->flowtable->net); + + if (offload->cmd == FLOW_CLS_REPLACE) { + NF_FLOW_TABLE_STAT_INC(net, count_wq_add); queue_work(nf_flow_offload_add_wq, &offload->work); - else if (offload->cmd == FLOW_CLS_DESTROY) + } else if (offload->cmd == FLOW_CLS_DESTROY) { + NF_FLOW_TABLE_STAT_INC(net, count_wq_del); queue_work(nf_flow_offload_del_wq, &offload->work); - else + } else { + NF_FLOW_TABLE_STAT_INC(net, count_wq_stats); queue_work(nf_flow_offload_stats_wq, &offload->work); + } } static struct flow_offload_work * diff --git a/net/netfilter/nf_flow_table_procfs.c b/net/netfilter/nf_flow_table_procfs.c new file mode 100644 index 000000000000..159b033a43e6 --- /dev/null +++ b/net/netfilter/nf_flow_table_procfs.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/kernel.h> +#include <linux/proc_fs.h> +#include <net/netfilter/nf_flow_table.h> + +static void *nf_flow_table_cpu_seq_start(struct seq_file *seq, loff_t *pos) +{ + struct net *net = seq_file_net(seq); + int cpu; + + if (*pos == 0) + return SEQ_START_TOKEN; + + for (cpu = *pos - 1; cpu < nr_cpu_ids; ++cpu) { + if (!cpu_possible(cpu)) + continue; + *pos = cpu + 1; + return per_cpu_ptr(net->ft.stat, cpu); + } + + return NULL; +} + +static void *nf_flow_table_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct net *net = seq_file_net(seq); + int cpu; + + for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) { + if (!cpu_possible(cpu)) + continue; + *pos = cpu + 1; + return per_cpu_ptr(net->ft.stat, cpu); + } + (*pos)++; + return NULL; +} + +static void nf_flow_table_cpu_seq_stop(struct seq_file *seq, void *v) +{ +} + +static int nf_flow_table_cpu_seq_show(struct seq_file *seq, void *v) +{ + const struct nf_flow_table_stat *st = v; + + if (v == SEQ_START_TOKEN) { + seq_puts(seq, "wq_add wq_del wq_stats\n"); + return 0; + } + + seq_printf(seq, "%8d %8d %8d\n", + st->count_wq_add, + st->count_wq_del, + st->count_wq_stats + ); + return 0; +} + +static const struct seq_operations nf_flow_table_cpu_seq_ops = { + .start = nf_flow_table_cpu_seq_start, + .next = nf_flow_table_cpu_seq_next, + .stop = nf_flow_table_cpu_seq_stop, + .show = nf_flow_table_cpu_seq_show, +}; + +int nf_flow_table_init_proc(struct net *net) +{ + struct proc_dir_entry *pde; + + pde = proc_create_net("nf_flowtable", 0444, net->proc_net_stat, + &nf_flow_table_cpu_seq_ops, + sizeof(struct seq_net_private)); + return pde ? 0 : -ENOMEM; +} + +void nf_flow_table_fini_proc(struct net *net) +{ + remove_proc_entry("nf_flowtable", net->proc_net_stat); +} diff --git a/net/netfilter/nf_synproxy_core.c b/net/netfilter/nf_synproxy_core.c index e479dd0561c5..16915f8eef2b 100644 --- a/net/netfilter/nf_synproxy_core.c +++ b/net/netfilter/nf_synproxy_core.c @@ -405,7 +405,7 @@ synproxy_build_ip(struct net *net, struct sk_buff *skb, __be32 saddr, iph->tos = 0; iph->id = 0; iph->frag_off = htons(IP_DF); - iph->ttl = net->ipv4.sysctl_ip_default_ttl; + iph->ttl = READ_ONCE(net->ipv4.sysctl_ip_default_ttl); iph->protocol = IPPROTO_TCP; iph->check = 0; iph->saddr = saddr; diff --git a/net/netfilter/nfnetlink.c b/net/netfilter/nfnetlink.c index 2f7c477fc9e7..c24b1240908f 100644 --- a/net/netfilter/nfnetlink.c +++ b/net/netfilter/nfnetlink.c @@ -626,7 +626,7 @@ static void nfnetlink_rcv_skb_batch(struct sk_buff *skb, struct nlmsghdr *nlh) nfgenmsg = nlmsg_data(nlh); skb_pull(skb, msglen); /* Work around old nft using host byte order */ - if (nfgenmsg->res_id == NFNL_SUBSYS_NFTABLES) + if (nfgenmsg->res_id == (__force __be16)NFNL_SUBSYS_NFTABLES) res_id = NFNL_SUBSYS_NFTABLES; else res_id = ntohs(nfgenmsg->res_id); diff --git a/net/netfilter/nfnetlink_cthelper.c b/net/netfilter/nfnetlink_cthelper.c index 5c622f55c9d6..97248963a7d3 100644 --- a/net/netfilter/nfnetlink_cthelper.c +++ b/net/netfilter/nfnetlink_cthelper.c @@ -96,11 +96,13 @@ static int nfnl_cthelper_from_nlattr(struct nlattr *attr, struct nf_conn *ct) { struct nf_conn_help *help = nfct_help(ct); + const struct nf_conntrack_helper *helper; if (attr == NULL) return -EINVAL; - if (help->helper->data_len == 0) + helper = rcu_dereference(help->helper); + if (!helper || helper->data_len == 0) return -EINVAL; nla_memcpy(help->data, attr, sizeof(help->data)); @@ -111,9 +113,11 @@ static int nfnl_cthelper_to_nlattr(struct sk_buff *skb, const struct nf_conn *ct) { const struct nf_conn_help *help = nfct_help(ct); + const struct nf_conntrack_helper *helper; - if (help->helper->data_len && - nla_put(skb, CTA_HELP_INFO, help->helper->data_len, &help->data)) + helper = rcu_dereference(help->helper); + if (helper && helper->data_len && + nla_put(skb, CTA_HELP_INFO, helper->data_len, &help->data)) goto nla_put_failure; return 0; diff --git a/net/netfilter/nft_byteorder.c b/net/netfilter/nft_byteorder.c index d77609144b26..f952a80275a8 100644 --- a/net/netfilter/nft_byteorder.c +++ b/net/netfilter/nft_byteorder.c @@ -44,7 +44,8 @@ void nft_byteorder_eval(const struct nft_expr *expr, case NFT_BYTEORDER_NTOH: for (i = 0; i < priv->len / 8; i++) { src64 = nft_reg_load64(&src[i]); - nft_reg_store64(&dst[i], be64_to_cpu(src64)); + nft_reg_store64(&dst[i], + be64_to_cpu((__force __be64)src64)); } break; case NFT_BYTEORDER_HTON: diff --git a/net/netfilter/nft_cmp.c b/net/netfilter/nft_cmp.c index 6528f76ca29e..777f09e4dc60 100644 --- a/net/netfilter/nft_cmp.c +++ b/net/netfilter/nft_cmp.c @@ -125,13 +125,13 @@ static void nft_payload_n2h(union nft_cmp_offload_data *data, { switch (len) { case 2: - data->val16 = ntohs(*((u16 *)val)); + data->val16 = ntohs(*((__be16 *)val)); break; case 4: - data->val32 = ntohl(*((u32 *)val)); + data->val32 = ntohl(*((__be32 *)val)); break; case 8: - data->val64 = be64_to_cpu(*((u64 *)val)); + data->val64 = be64_to_cpu(*((__be64 *)val)); break; default: WARN_ON_ONCE(1); @@ -197,6 +197,18 @@ static const struct nft_expr_ops nft_cmp_ops = { .offload = nft_cmp_offload, }; +/* Calculate the mask for the nft_cmp_fast expression. On big endian the + * mask needs to include the *upper* bytes when interpreting that data as + * something smaller than the full u32, therefore a cpu_to_le32 is done. + */ +static u32 nft_cmp_fast_mask(unsigned int len) +{ + __le32 mask = cpu_to_le32(~0U >> (sizeof_field(struct nft_cmp_fast_expr, + data) * BITS_PER_BYTE - len)); + + return (__force u32)mask; +} + static int nft_cmp_fast_init(const struct nft_ctx *ctx, const struct nft_expr *expr, const struct nlattr * const tb[]) diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index d8e1614918a1..b04995c3e17f 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -204,12 +204,12 @@ static void nft_ct_get_eval(const struct nft_expr *expr, case NFT_CT_SRC_IP: if (nf_ct_l3num(ct) != NFPROTO_IPV4) goto err; - *dest = tuple->src.u3.ip; + *dest = (__force __u32)tuple->src.u3.ip; return; case NFT_CT_DST_IP: if (nf_ct_l3num(ct) != NFPROTO_IPV4) goto err; - *dest = tuple->dst.u3.ip; + *dest = (__force __u32)tuple->dst.u3.ip; return; case NFT_CT_SRC_IP6: if (nf_ct_l3num(ct) != NFPROTO_IPV6) diff --git a/net/netfilter/nft_exthdr.c b/net/netfilter/nft_exthdr.c index 22c3e05b52db..a67ea9c3ae57 100644 --- a/net/netfilter/nft_exthdr.c +++ b/net/netfilter/nft_exthdr.c @@ -266,7 +266,7 @@ static void nft_exthdr_tcp_set_eval(const struct nft_expr *expr, switch (priv->len) { case 2: - old.v16 = get_unaligned((u16 *)(opt + offset)); + old.v16 = (__force __be16)get_unaligned((u16 *)(opt + offset)); new.v16 = (__force __be16)nft_reg_load16( ®s->data[priv->sreg]); @@ -281,18 +281,18 @@ static void nft_exthdr_tcp_set_eval(const struct nft_expr *expr, if (old.v16 == new.v16) return; - put_unaligned(new.v16, (u16*)(opt + offset)); + put_unaligned(new.v16, (__be16*)(opt + offset)); inet_proto_csum_replace2(&tcph->check, pkt->skb, old.v16, new.v16, false); break; case 4: - new.v32 = regs->data[priv->sreg]; - old.v32 = get_unaligned((u32 *)(opt + offset)); + new.v32 = nft_reg_load_be32(®s->data[priv->sreg]); + old.v32 = (__force __be32)get_unaligned((u32 *)(opt + offset)); if (old.v32 == new.v32) return; - put_unaligned(new.v32, (u32*)(opt + offset)); + put_unaligned(new.v32, (__be32*)(opt + offset)); inet_proto_csum_replace4(&tcph->check, pkt->skb, old.v32, new.v32, false); break; diff --git a/net/netfilter/nft_osf.c b/net/netfilter/nft_osf.c index 5eed18f90b02..0053a697c931 100644 --- a/net/netfilter/nft_osf.c +++ b/net/netfilter/nft_osf.c @@ -99,7 +99,7 @@ static int nft_osf_dump(struct sk_buff *skb, const struct nft_expr *expr) if (nla_put_u8(skb, NFTA_OSF_TTL, priv->ttl)) goto nla_put_failure; - if (nla_put_be32(skb, NFTA_OSF_FLAGS, ntohl(priv->flags))) + if (nla_put_u32(skb, NFTA_OSF_FLAGS, ntohl((__force __be32)priv->flags))) goto nla_put_failure; if (nft_dump_register(skb, NFTA_OSF_DREG, priv->dreg)) diff --git a/net/netfilter/nft_set_bitmap.c b/net/netfilter/nft_set_bitmap.c index e7ae5914971e..96081ac8d2b4 100644 --- a/net/netfilter/nft_set_bitmap.c +++ b/net/netfilter/nft_set_bitmap.c @@ -21,7 +21,7 @@ struct nft_bitmap_elem { * the element state in the current and the future generation. * * An element can be in three states. The generation cursor is represented using - * the ^ character, note that this cursor shifts on every succesful transaction. + * the ^ character, note that this cursor shifts on every successful transaction. * If no transaction is going on, we observe all elements are in the following * state: * @@ -39,7 +39,7 @@ struct nft_bitmap_elem { * 10 = this element is active in the current generation and it becomes inactive * ^ in the next one. This happens when the element is deactivated but commit * path has not yet been executed yet, so removal is still pending. On - * transation abortion, the next generation bit is reset to go back to + * transaction abortion, the next generation bit is reset to go back to * restore its previous state. */ struct nft_bitmap { diff --git a/net/netfilter/nft_socket.c b/net/netfilter/nft_socket.c index 05ae5a338b6f..a7de29137618 100644 --- a/net/netfilter/nft_socket.c +++ b/net/netfilter/nft_socket.c @@ -163,7 +163,7 @@ static int nft_socket_init(const struct nft_ctx *ctx, return -EOPNOTSUPP; } - priv->key = ntohl(nla_get_u32(tb[NFTA_SOCKET_KEY])); + priv->key = ntohl(nla_get_be32(tb[NFTA_SOCKET_KEY])); switch(priv->key) { case NFT_SOCKET_TRANSPARENT: case NFT_SOCKET_WILDCARD: @@ -179,7 +179,7 @@ static int nft_socket_init(const struct nft_ctx *ctx, if (!tb[NFTA_SOCKET_LEVEL]) return -EINVAL; - level = ntohl(nla_get_u32(tb[NFTA_SOCKET_LEVEL])); + level = ntohl(nla_get_be32(tb[NFTA_SOCKET_LEVEL])); if (level > 255) return -EOPNOTSUPP; @@ -202,12 +202,12 @@ static int nft_socket_dump(struct sk_buff *skb, { const struct nft_socket *priv = nft_expr_priv(expr); - if (nla_put_u32(skb, NFTA_SOCKET_KEY, htonl(priv->key))) + if (nla_put_be32(skb, NFTA_SOCKET_KEY, htonl(priv->key))) return -1; if (nft_dump_register(skb, NFTA_SOCKET_DREG, priv->dreg)) return -1; if (priv->key == NFT_SOCKET_CGROUPV2 && - nla_put_u32(skb, NFTA_SOCKET_LEVEL, htonl(priv->level))) + nla_put_be32(skb, NFTA_SOCKET_LEVEL, htonl(priv->level))) return -1; return 0; } diff --git a/net/netfilter/nft_tproxy.c b/net/netfilter/nft_tproxy.c index 801f013971df..68b2eed742df 100644 --- a/net/netfilter/nft_tproxy.c +++ b/net/netfilter/nft_tproxy.c @@ -52,11 +52,11 @@ static void nft_tproxy_eval_v4(const struct nft_expr *expr, skb->dev, NF_TPROXY_LOOKUP_ESTABLISHED); if (priv->sreg_addr) - taddr = regs->data[priv->sreg_addr]; + taddr = nft_reg_load_be32(®s->data[priv->sreg_addr]); taddr = nf_tproxy_laddr4(skb, taddr, iph->daddr); if (priv->sreg_port) - tport = nft_reg_load16(®s->data[priv->sreg_port]); + tport = nft_reg_load_be16(®s->data[priv->sreg_port]); if (!tport) tport = hp->dest; @@ -124,7 +124,7 @@ static void nft_tproxy_eval_v6(const struct nft_expr *expr, taddr = *nf_tproxy_laddr6(skb, &taddr, &iph->daddr); if (priv->sreg_port) - tport = nft_reg_load16(®s->data[priv->sreg_port]); + tport = nft_reg_load_be16(®s->data[priv->sreg_port]); if (!tport) tport = hp->dest; diff --git a/net/netfilter/nft_tunnel.c b/net/netfilter/nft_tunnel.c index d0f9b1d51b0e..5edaaded706d 100644 --- a/net/netfilter/nft_tunnel.c +++ b/net/netfilter/nft_tunnel.c @@ -383,8 +383,9 @@ static int nft_tunnel_obj_opts_init(const struct nft_ctx *ctx, struct ip_tunnel_info *info, struct nft_tunnel_opts *opts) { - int err, rem, type = 0; struct nlattr *nla; + __be16 type = 0; + int err, rem; err = nla_validate_nested_deprecated(attr, NFTA_TUNNEL_KEY_OPTS_MAX, nft_tunnel_opts_policy, NULL); diff --git a/net/netfilter/nft_xfrm.c b/net/netfilter/nft_xfrm.c index becb88fa4e9b..1c5343c936a8 100644 --- a/net/netfilter/nft_xfrm.c +++ b/net/netfilter/nft_xfrm.c @@ -51,7 +51,7 @@ static int nft_xfrm_get_init(const struct nft_ctx *ctx, return -EOPNOTSUPP; } - priv->key = ntohl(nla_get_u32(tb[NFTA_XFRM_KEY])); + priv->key = ntohl(nla_get_be32(tb[NFTA_XFRM_KEY])); switch (priv->key) { case NFT_XFRM_KEY_REQID: case NFT_XFRM_KEY_SPI: @@ -134,13 +134,13 @@ static void nft_xfrm_state_get_key(const struct nft_xfrm *priv, WARN_ON_ONCE(1); break; case NFT_XFRM_KEY_DADDR_IP4: - *dest = state->id.daddr.a4; + *dest = (__force __u32)state->id.daddr.a4; return; case NFT_XFRM_KEY_DADDR_IP6: memcpy(dest, &state->id.daddr.in6, sizeof(struct in6_addr)); return; case NFT_XFRM_KEY_SADDR_IP4: - *dest = state->props.saddr.a4; + *dest = (__force __u32)state->props.saddr.a4; return; case NFT_XFRM_KEY_SADDR_IP6: memcpy(dest, &state->props.saddr.in6, sizeof(struct in6_addr)); @@ -149,7 +149,7 @@ static void nft_xfrm_state_get_key(const struct nft_xfrm *priv, *dest = state->props.reqid; return; case NFT_XFRM_KEY_SPI: - *dest = state->id.spi; + *dest = (__force __u32)state->id.spi; return; } diff --git a/net/netfilter/xt_CT.c b/net/netfilter/xt_CT.c index 267757b0392a..2be2f7a7b60f 100644 --- a/net/netfilter/xt_CT.c +++ b/net/netfilter/xt_CT.c @@ -96,7 +96,7 @@ xt_ct_set_helper(struct nf_conn *ct, const char *helper_name, return -ENOMEM; } - help->helper = helper; + rcu_assign_pointer(help->helper, helper); return 0; } @@ -136,6 +136,21 @@ static u16 xt_ct_flags_to_dir(const struct xt_ct_target_info_v1 *info) } } +static void xt_ct_put_helper(struct nf_conn_help *help) +{ + struct nf_conntrack_helper *helper; + + if (!help) + return; + + /* not yet exposed to other cpus, or ruleset + * already detached (post-replacement). + */ + helper = rcu_dereference_raw(help->helper); + if (helper) + nf_conntrack_helper_put(helper); +} + static int xt_ct_tg_check(const struct xt_tgchk_param *par, struct xt_ct_target_info_v1 *info) { @@ -207,8 +222,7 @@ out: err4: help = nfct_help(ct); - if (help) - nf_conntrack_helper_put(help->helper); + xt_ct_put_helper(help); err3: nf_ct_tmpl_free(ct); err2: @@ -270,8 +284,7 @@ static void xt_ct_tg_destroy(const struct xt_tgdtor_param *par, if (ct) { help = nfct_help(ct); - if (help) - nf_conntrack_helper_put(help->helper); + xt_ct_put_helper(help); nf_ct_netns_put(par->net, par->family); diff --git a/net/netfilter/xt_DSCP.c b/net/netfilter/xt_DSCP.c index eababc354ff1..cfa44515ab72 100644 --- a/net/netfilter/xt_DSCP.c +++ b/net/netfilter/xt_DSCP.c @@ -24,6 +24,8 @@ MODULE_ALIAS("ip6t_DSCP"); MODULE_ALIAS("ipt_TOS"); MODULE_ALIAS("ip6t_TOS"); +#define XT_DSCP_ECN_MASK 3u + static unsigned int dscp_tg(struct sk_buff *skb, const struct xt_action_param *par) { @@ -34,8 +36,7 @@ dscp_tg(struct sk_buff *skb, const struct xt_action_param *par) if (skb_ensure_writable(skb, sizeof(struct iphdr))) return NF_DROP; - ipv4_change_dsfield(ip_hdr(skb), - (__force __u8)(~XT_DSCP_MASK), + ipv4_change_dsfield(ip_hdr(skb), XT_DSCP_ECN_MASK, dinfo->dscp << XT_DSCP_SHIFT); } @@ -52,8 +53,7 @@ dscp_tg6(struct sk_buff *skb, const struct xt_action_param *par) if (skb_ensure_writable(skb, sizeof(struct ipv6hdr))) return NF_DROP; - ipv6_change_dsfield(ipv6_hdr(skb), - (__force __u8)(~XT_DSCP_MASK), + ipv6_change_dsfield(ipv6_hdr(skb), XT_DSCP_ECN_MASK, dinfo->dscp << XT_DSCP_SHIFT); } return XT_CONTINUE; diff --git a/net/netfilter/xt_TCPMSS.c b/net/netfilter/xt_TCPMSS.c index 122db9fbb9f4..116a885adb3c 100644 --- a/net/netfilter/xt_TCPMSS.c +++ b/net/netfilter/xt_TCPMSS.c @@ -239,8 +239,8 @@ tcpmss_tg6(struct sk_buff *skb, const struct xt_action_param *par) oldlen = ipv6h->payload_len; newlen = htons(ntohs(oldlen) + ret); if (skb->ip_summed == CHECKSUM_COMPLETE) - skb->csum = csum_add(csum_sub(skb->csum, oldlen), - newlen); + skb->csum = csum_add(csum_sub(skb->csum, (__force __wsum)oldlen), + (__force __wsum)newlen); ipv6h->payload_len = newlen; } return XT_CONTINUE; diff --git a/net/netfilter/xt_TPROXY.c b/net/netfilter/xt_TPROXY.c index 459d0696c91a..e4bea1d346cf 100644 --- a/net/netfilter/xt_TPROXY.c +++ b/net/netfilter/xt_TPROXY.c @@ -74,18 +74,10 @@ tproxy_tg4(struct net *net, struct sk_buff *skb, __be32 laddr, __be16 lport, /* This should be in a separate target, but we don't do multiple targets on the same rule yet */ skb->mark = (skb->mark & ~mark_mask) ^ mark_value; - - pr_debug("redirecting: proto %hhu %pI4:%hu -> %pI4:%hu, mark: %x\n", - iph->protocol, &iph->daddr, ntohs(hp->dest), - &laddr, ntohs(lport), skb->mark); - nf_tproxy_assign_sock(skb, sk); return NF_ACCEPT; } - pr_debug("no socket, dropping: proto %hhu %pI4:%hu -> %pI4:%hu, mark: %x\n", - iph->protocol, &iph->saddr, ntohs(hp->source), - &iph->daddr, ntohs(hp->dest), skb->mark); return NF_DROP; } @@ -122,16 +114,12 @@ tproxy_tg6_v1(struct sk_buff *skb, const struct xt_action_param *par) int tproto; tproto = ipv6_find_hdr(skb, &thoff, -1, NULL, NULL); - if (tproto < 0) { - pr_debug("unable to find transport header in IPv6 packet, dropping\n"); + if (tproto < 0) return NF_DROP; - } hp = skb_header_pointer(skb, thoff, sizeof(_hdr), &_hdr); - if (hp == NULL) { - pr_debug("unable to grab transport header contents in IPv6 packet, dropping\n"); + if (!hp) return NF_DROP; - } /* check if there's an ongoing connection on the packet * addresses, this happens if the redirect already happened @@ -168,19 +156,10 @@ tproxy_tg6_v1(struct sk_buff *skb, const struct xt_action_param *par) /* This should be in a separate target, but we don't do multiple targets on the same rule yet */ skb->mark = (skb->mark & ~tgi->mark_mask) ^ tgi->mark_value; - - pr_debug("redirecting: proto %hhu %pI6:%hu -> %pI6:%hu, mark: %x\n", - tproto, &iph->saddr, ntohs(hp->source), - laddr, ntohs(lport), skb->mark); - nf_tproxy_assign_sock(skb, sk); return NF_ACCEPT; } - pr_debug("no socket, dropping: proto %hhu %pI6:%hu -> %pI6:%hu, mark: %x\n", - tproto, &iph->saddr, ntohs(hp->source), - &iph->daddr, ntohs(hp->dest), skb->mark); - return NF_DROP; } diff --git a/net/netfilter/xt_connlimit.c b/net/netfilter/xt_connlimit.c index 46fcac75f726..5d04ef80a61d 100644 --- a/net/netfilter/xt_connlimit.c +++ b/net/netfilter/xt_connlimit.c @@ -62,10 +62,10 @@ connlimit_mt(const struct sk_buff *skb, struct xt_action_param *par) key[4] = zone->id; } else { const struct iphdr *iph = ip_hdr(skb); - key[0] = (info->flags & XT_CONNLIMIT_DADDR) ? - iph->daddr : iph->saddr; - key[0] &= info->mask.ip; + key[0] = (info->flags & XT_CONNLIMIT_DADDR) ? + (__force __u32)iph->daddr : (__force __u32)iph->saddr; + key[0] &= (__force __u32)info->mask.ip; key[1] = zone->id; } diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index e013253b10d1..d55afb8d14be 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -277,7 +277,7 @@ static struct nf_flowtable_type flowtable_ct = { .owner = THIS_MODULE, }; -static int tcf_ct_flow_table_get(struct tcf_ct_params *params) +static int tcf_ct_flow_table_get(struct net *net, struct tcf_ct_params *params) { struct tcf_ct_flow_table *ct_ft; int err = -ENOMEM; @@ -303,6 +303,7 @@ static int tcf_ct_flow_table_get(struct tcf_ct_params *params) err = nf_flow_table_init(&ct_ft->nf_ft); if (err) goto err_init; + write_pnet(&ct_ft->nf_ft.net, net); __module_get(THIS_MODULE); out_unlock: @@ -1391,7 +1392,7 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla, if (err) goto cleanup; - err = tcf_ct_flow_table_get(params); + err = tcf_ct_flow_table_get(net, params); if (err) goto cleanup; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index c7a240232b8d..790d6809be81 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -3534,7 +3534,7 @@ int tc_setup_action(struct flow_action *flow_action, struct tc_action *actions[], struct netlink_ext_ack *extack) { - int i, j, index, err = 0; + int i, j, k, index, err = 0; struct tc_action *act; BUILD_BUG_ON(TCA_ACT_HW_STATS_ANY != FLOW_ACTION_HW_STATS_ANY); @@ -3554,14 +3554,18 @@ int tc_setup_action(struct flow_action *flow_action, if (err) goto err_out_locked; - entry->hw_stats = tc_act_hw_stats(act->hw_stats); - entry->hw_index = act->tcfa_index; index = 0; err = tc_setup_offload_act(act, entry, &index, extack); - if (!err) - j += index; - else + if (err) goto err_out_locked; + + for (k = 0; k < index ; k++) { + entry[k].hw_stats = tc_act_hw_stats(act->hw_stats); + entry[k].hw_index = act->tcfa_index; + } + + j += index; + spin_unlock_bh(&act->tcfa_lock); } diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c index 02d9f0dfe356..599e26fc2fa8 100644 --- a/net/sched/sch_cbq.c +++ b/net/sched/sch_cbq.c @@ -149,7 +149,6 @@ struct cbq_sched_data { psched_time_t now; /* Cached timestamp */ unsigned int pmask; - struct hrtimer delay_timer; struct qdisc_watchdog watchdog; /* Watchdog timer, started when CBQ has backlog, but cannot @@ -441,81 +440,6 @@ static void cbq_overlimit(struct cbq_class *cl) } } -static psched_tdiff_t cbq_undelay_prio(struct cbq_sched_data *q, int prio, - psched_time_t now) -{ - struct cbq_class *cl; - struct cbq_class *cl_prev = q->active[prio]; - psched_time_t sched = now; - - if (cl_prev == NULL) - return 0; - - do { - cl = cl_prev->next_alive; - if (now - cl->penalized > 0) { - cl_prev->next_alive = cl->next_alive; - cl->next_alive = NULL; - cl->cpriority = cl->priority; - cl->delayed = 0; - cbq_activate_class(cl); - - if (cl == q->active[prio]) { - q->active[prio] = cl_prev; - if (cl == q->active[prio]) { - q->active[prio] = NULL; - return 0; - } - } - - cl = cl_prev->next_alive; - } else if (sched - cl->penalized > 0) - sched = cl->penalized; - } while ((cl_prev = cl) != q->active[prio]); - - return sched - now; -} - -static enum hrtimer_restart cbq_undelay(struct hrtimer *timer) -{ - struct cbq_sched_data *q = container_of(timer, struct cbq_sched_data, - delay_timer); - struct Qdisc *sch = q->watchdog.qdisc; - psched_time_t now; - psched_tdiff_t delay = 0; - unsigned int pmask; - - now = psched_get_time(); - - pmask = q->pmask; - q->pmask = 0; - - while (pmask) { - int prio = ffz(~pmask); - psched_tdiff_t tmp; - - pmask &= ~(1<<prio); - - tmp = cbq_undelay_prio(q, prio, now); - if (tmp > 0) { - q->pmask |= 1<<prio; - if (tmp < delay || delay == 0) - delay = tmp; - } - } - - if (delay) { - ktime_t time; - - time = 0; - time = ktime_add_ns(time, PSCHED_TICKS2NS(now + delay)); - hrtimer_start(&q->delay_timer, time, HRTIMER_MODE_ABS_PINNED); - } - - __netif_schedule(qdisc_root(sch)); - return HRTIMER_NORESTART; -} - /* * It is mission critical procedure. * @@ -1034,7 +958,6 @@ cbq_reset(struct Qdisc *sch) q->tx_class = NULL; q->tx_borrowed = NULL; qdisc_watchdog_cancel(&q->watchdog); - hrtimer_cancel(&q->delay_timer); q->toplevel = TC_CBQ_MAXLEVEL; q->now = psched_get_time(); @@ -1162,8 +1085,6 @@ static int cbq_init(struct Qdisc *sch, struct nlattr *opt, int err; qdisc_watchdog_init(&q->watchdog, sch); - hrtimer_init(&q->delay_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); - q->delay_timer.function = cbq_undelay; err = cbq_opt_parse(tb, opt, extack); if (err < 0) diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c index fa500ea3a1f1..bcd3384ab07a 100644 --- a/net/sctp/protocol.c +++ b/net/sctp/protocol.c @@ -358,7 +358,7 @@ static int sctp_v4_available(union sctp_addr *addr, struct sctp_sock *sp) if (addr->v4.sin_addr.s_addr != htonl(INADDR_ANY) && ret != RTN_LOCAL && !sp->inet.freebind && - !net->ipv4.sysctl_ip_nonlocal_bind) + !READ_ONCE(net->ipv4.sysctl_ip_nonlocal_bind)) return 0; if (ipv6_only_sock(sctp_opt2sk(sp))) diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 433bb5a7df31..6e70d9c10b78 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -487,6 +487,29 @@ static void smc_copy_sock_settings_to_smc(struct smc_sock *smc) smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC); } +/* register the new vzalloced sndbuf on all links */ +static int smcr_lgr_reg_sndbufs(struct smc_link *link, + struct smc_buf_desc *snd_desc) +{ + struct smc_link_group *lgr = link->lgr; + int i, rc = 0; + + if (!snd_desc->is_vm) + return -EINVAL; + + /* protect against parallel smcr_link_reg_buf() */ + mutex_lock(&lgr->llc_conf_mutex); + for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { + if (!smc_link_active(&lgr->lnk[i])) + continue; + rc = smcr_link_reg_buf(&lgr->lnk[i], snd_desc); + if (rc) + break; + } + mutex_unlock(&lgr->llc_conf_mutex); + return rc; +} + /* register the new rmb on all links */ static int smcr_lgr_reg_rmbs(struct smc_link *link, struct smc_buf_desc *rmb_desc) @@ -498,13 +521,13 @@ static int smcr_lgr_reg_rmbs(struct smc_link *link, if (rc) return rc; /* protect against parallel smc_llc_cli_rkey_exchange() and - * parallel smcr_link_reg_rmb() + * parallel smcr_link_reg_buf() */ mutex_lock(&lgr->llc_conf_mutex); for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { if (!smc_link_active(&lgr->lnk[i])) continue; - rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc); + rc = smcr_link_reg_buf(&lgr->lnk[i], rmb_desc); if (rc) goto out; } @@ -550,8 +573,15 @@ static int smcr_clnt_conf_first_link(struct smc_sock *smc) smc_wr_remember_qp_attr(link); - if (smcr_link_reg_rmb(link, smc->conn.rmb_desc)) - return SMC_CLC_DECL_ERR_REGRMB; + /* reg the sndbuf if it was vzalloced */ + if (smc->conn.sndbuf_desc->is_vm) { + if (smcr_link_reg_buf(link, smc->conn.sndbuf_desc)) + return SMC_CLC_DECL_ERR_REGBUF; + } + + /* reg the rmb */ + if (smcr_link_reg_buf(link, smc->conn.rmb_desc)) + return SMC_CLC_DECL_ERR_REGBUF; /* confirm_rkey is implicit on 1st contact */ smc->conn.rmb_desc->is_conf_rkey = true; @@ -1221,12 +1251,18 @@ static int smc_connect_rdma(struct smc_sock *smc, goto connect_abort; } } else { + /* reg sendbufs if they were vzalloced */ + if (smc->conn.sndbuf_desc->is_vm) { + if (smcr_lgr_reg_sndbufs(link, smc->conn.sndbuf_desc)) { + reason_code = SMC_CLC_DECL_ERR_REGBUF; + goto connect_abort; + } + } if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) { - reason_code = SMC_CLC_DECL_ERR_REGRMB; + reason_code = SMC_CLC_DECL_ERR_REGBUF; goto connect_abort; } } - smc_rmb_sync_sg_for_device(&smc->conn); if (aclc->hdr.version > SMC_V1) { struct smc_clc_msg_accept_confirm_v2 *clc_v2 = @@ -1750,8 +1786,15 @@ static int smcr_serv_conf_first_link(struct smc_sock *smc) struct smc_llc_qentry *qentry; int rc; - if (smcr_link_reg_rmb(link, smc->conn.rmb_desc)) - return SMC_CLC_DECL_ERR_REGRMB; + /* reg the sndbuf if it was vzalloced*/ + if (smc->conn.sndbuf_desc->is_vm) { + if (smcr_link_reg_buf(link, smc->conn.sndbuf_desc)) + return SMC_CLC_DECL_ERR_REGBUF; + } + + /* reg the rmb */ + if (smcr_link_reg_buf(link, smc->conn.rmb_desc)) + return SMC_CLC_DECL_ERR_REGBUF; /* send CONFIRM LINK request to client over the RoCE fabric */ rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ); @@ -2110,10 +2153,15 @@ static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first) struct smc_connection *conn = &new_smc->conn; if (!local_first) { + /* reg sendbufs if they were vzalloced */ + if (conn->sndbuf_desc->is_vm) { + if (smcr_lgr_reg_sndbufs(conn->lnk, + conn->sndbuf_desc)) + return SMC_CLC_DECL_ERR_REGBUF; + } if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc)) - return SMC_CLC_DECL_ERR_REGRMB; + return SMC_CLC_DECL_ERR_REGBUF; } - smc_rmb_sync_sg_for_device(&new_smc->conn); return 0; } diff --git a/net/smc/smc_clc.c b/net/smc/smc_clc.c index f9f3f59c79de..1472f31480d8 100644 --- a/net/smc/smc_clc.c +++ b/net/smc/smc_clc.c @@ -1034,7 +1034,7 @@ static int smc_clc_send_confirm_accept(struct smc_sock *smc, ETH_ALEN); hton24(clc->r0.qpn, link->roce_qp->qp_num); clc->r0.rmb_rkey = - htonl(conn->rmb_desc->mr_rx[link->link_idx]->rkey); + htonl(conn->rmb_desc->mr[link->link_idx]->rkey); clc->r0.rmbe_idx = 1; /* for now: 1 RMB = 1 RMBE */ clc->r0.rmbe_alert_token = htonl(conn->alert_token_local); switch (clc->hdr.type) { @@ -1046,8 +1046,10 @@ static int smc_clc_send_confirm_accept(struct smc_sock *smc, break; } clc->r0.rmbe_size = conn->rmbe_size_short; - clc->r0.rmb_dma_addr = cpu_to_be64((u64)sg_dma_address - (conn->rmb_desc->sgt[link->link_idx].sgl)); + clc->r0.rmb_dma_addr = conn->rmb_desc->is_vm ? + cpu_to_be64((uintptr_t)conn->rmb_desc->cpu_addr) : + cpu_to_be64((u64)sg_dma_address + (conn->rmb_desc->sgt[link->link_idx].sgl)); hton24(clc->r0.psn, link->psn_initial); if (version == SMC_V1) { clc->hdr.length = htons(SMCR_CLC_ACCEPT_CONFIRM_LEN); diff --git a/net/smc/smc_clc.h b/net/smc/smc_clc.h index 83f02f131fc0..5fee545c9a10 100644 --- a/net/smc/smc_clc.h +++ b/net/smc/smc_clc.h @@ -62,7 +62,7 @@ #define SMC_CLC_DECL_INTERR 0x09990000 /* internal error */ #define SMC_CLC_DECL_ERR_RTOK 0x09990001 /* rtoken handling failed */ #define SMC_CLC_DECL_ERR_RDYLNK 0x09990002 /* ib ready link failed */ -#define SMC_CLC_DECL_ERR_REGRMB 0x09990003 /* reg rmb failed */ +#define SMC_CLC_DECL_ERR_REGBUF 0x09990003 /* reg rdma bufs failed */ #define SMC_FIRST_CONTACT_MASK 0b10 /* first contact bit within typev2 */ diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c index f40f6ed0fbdb..ff49a11f57b8 100644 --- a/net/smc/smc_core.c +++ b/net/smc/smc_core.c @@ -347,6 +347,8 @@ static int smc_nl_fill_lgr(struct smc_link_group *lgr, goto errattr; if (nla_put_u8(skb, SMC_NLA_LGR_R_TYPE, lgr->type)) goto errattr; + if (nla_put_u8(skb, SMC_NLA_LGR_R_BUF_TYPE, lgr->buf_type)) + goto errattr; if (nla_put_u8(skb, SMC_NLA_LGR_R_VLAN_ID, lgr->vlan_id)) goto errattr; if (nla_put_u64_64bit(skb, SMC_NLA_LGR_R_NET_COOKIE, @@ -907,6 +909,7 @@ static int smc_lgr_create(struct smc_sock *smc, struct smc_init_info *ini) lgr->net = smc_ib_net(lnk->smcibdev); lgr_list = &smc_lgr_list.list; lgr_lock = &smc_lgr_list.lock; + lgr->buf_type = lgr->net->smc.sysctl_smcr_buf_type; atomic_inc(&lgr_cnt); } smc->conn.lgr = lgr; @@ -1086,34 +1089,37 @@ err_out: return NULL; } -static void smcr_buf_unuse(struct smc_buf_desc *rmb_desc, +static void smcr_buf_unuse(struct smc_buf_desc *buf_desc, bool is_rmb, struct smc_link_group *lgr) { + struct mutex *lock; /* lock buffer list */ int rc; - if (rmb_desc->is_conf_rkey && !list_empty(&lgr->list)) { + if (is_rmb && buf_desc->is_conf_rkey && !list_empty(&lgr->list)) { /* unregister rmb with peer */ rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY); if (!rc) { /* protect against smc_llc_cli_rkey_exchange() */ mutex_lock(&lgr->llc_conf_mutex); - smc_llc_do_delete_rkey(lgr, rmb_desc); - rmb_desc->is_conf_rkey = false; + smc_llc_do_delete_rkey(lgr, buf_desc); + buf_desc->is_conf_rkey = false; mutex_unlock(&lgr->llc_conf_mutex); smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); } } - if (rmb_desc->is_reg_err) { + if (buf_desc->is_reg_err) { /* buf registration failed, reuse not possible */ - mutex_lock(&lgr->rmbs_lock); - list_del(&rmb_desc->list); - mutex_unlock(&lgr->rmbs_lock); + lock = is_rmb ? &lgr->rmbs_lock : + &lgr->sndbufs_lock; + mutex_lock(lock); + list_del(&buf_desc->list); + mutex_unlock(lock); - smc_buf_free(lgr, true, rmb_desc); + smc_buf_free(lgr, is_rmb, buf_desc); } else { - rmb_desc->used = 0; - memset(rmb_desc->cpu_addr, 0, rmb_desc->len); + buf_desc->used = 0; + memset(buf_desc->cpu_addr, 0, buf_desc->len); } } @@ -1121,15 +1127,23 @@ static void smc_buf_unuse(struct smc_connection *conn, struct smc_link_group *lgr) { if (conn->sndbuf_desc) { - conn->sndbuf_desc->used = 0; - memset(conn->sndbuf_desc->cpu_addr, 0, conn->sndbuf_desc->len); + if (!lgr->is_smcd && conn->sndbuf_desc->is_vm) { + smcr_buf_unuse(conn->sndbuf_desc, false, lgr); + } else { + conn->sndbuf_desc->used = 0; + memset(conn->sndbuf_desc->cpu_addr, 0, + conn->sndbuf_desc->len); + } } - if (conn->rmb_desc && lgr->is_smcd) { - conn->rmb_desc->used = 0; - memset(conn->rmb_desc->cpu_addr, 0, conn->rmb_desc->len + - sizeof(struct smcd_cdc_msg)); - } else if (conn->rmb_desc) { - smcr_buf_unuse(conn->rmb_desc, lgr); + if (conn->rmb_desc) { + if (!lgr->is_smcd) { + smcr_buf_unuse(conn->rmb_desc, true, lgr); + } else { + conn->rmb_desc->used = 0; + memset(conn->rmb_desc->cpu_addr, 0, + conn->rmb_desc->len + + sizeof(struct smcd_cdc_msg)); + } } } @@ -1177,20 +1191,21 @@ lgr_put: static void smcr_buf_unmap_link(struct smc_buf_desc *buf_desc, bool is_rmb, struct smc_link *lnk) { - if (is_rmb) + if (is_rmb || buf_desc->is_vm) buf_desc->is_reg_mr[lnk->link_idx] = false; if (!buf_desc->is_map_ib[lnk->link_idx]) return; - if (is_rmb) { - if (buf_desc->mr_rx[lnk->link_idx]) { - smc_ib_put_memory_region( - buf_desc->mr_rx[lnk->link_idx]); - buf_desc->mr_rx[lnk->link_idx] = NULL; - } + + if ((is_rmb || buf_desc->is_vm) && + buf_desc->mr[lnk->link_idx]) { + smc_ib_put_memory_region(buf_desc->mr[lnk->link_idx]); + buf_desc->mr[lnk->link_idx] = NULL; + } + if (is_rmb) smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_FROM_DEVICE); - } else { + else smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_TO_DEVICE); - } + sg_free_table(&buf_desc->sgt[lnk->link_idx]); buf_desc->is_map_ib[lnk->link_idx] = false; } @@ -1279,8 +1294,10 @@ static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb, for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]); - if (buf_desc->pages) + if (!buf_desc->is_vm && buf_desc->pages) __free_pages(buf_desc->pages, buf_desc->order); + else if (buf_desc->is_vm && buf_desc->cpu_addr) + vfree(buf_desc->cpu_addr); kfree(buf_desc); } @@ -1992,39 +2009,69 @@ static inline int smc_rmb_wnd_update_limit(int rmbe_size) return max_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2); } -/* map an rmb buf to a link */ +/* map an buf to a link */ static int smcr_buf_map_link(struct smc_buf_desc *buf_desc, bool is_rmb, struct smc_link *lnk) { - int rc; + int rc, i, nents, offset, buf_size, size, access_flags; + struct scatterlist *sg; + void *buf; if (buf_desc->is_map_ib[lnk->link_idx]) return 0; - rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], 1, GFP_KERNEL); + if (buf_desc->is_vm) { + buf = buf_desc->cpu_addr; + buf_size = buf_desc->len; + offset = offset_in_page(buf_desc->cpu_addr); + nents = PAGE_ALIGN(buf_size + offset) / PAGE_SIZE; + } else { + nents = 1; + } + + rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], nents, GFP_KERNEL); if (rc) return rc; - sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl, - buf_desc->cpu_addr, buf_desc->len); + + if (buf_desc->is_vm) { + /* virtually contiguous buffer */ + for_each_sg(buf_desc->sgt[lnk->link_idx].sgl, sg, nents, i) { + size = min_t(int, PAGE_SIZE - offset, buf_size); + sg_set_page(sg, vmalloc_to_page(buf), size, offset); + buf += size / sizeof(*buf); + buf_size -= size; + offset = 0; + } + } else { + /* physically contiguous buffer */ + sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl, + buf_desc->cpu_addr, buf_desc->len); + } /* map sg table to DMA address */ rc = smc_ib_buf_map_sg(lnk, buf_desc, is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE); /* SMC protocol depends on mapping to one DMA address only */ - if (rc != 1) { + if (rc != nents) { rc = -EAGAIN; goto free_table; } - /* create a new memory region for the RMB */ - if (is_rmb) { - rc = smc_ib_get_memory_region(lnk->roce_pd, - IB_ACCESS_REMOTE_WRITE | - IB_ACCESS_LOCAL_WRITE, + buf_desc->is_dma_need_sync |= + smc_ib_is_sg_need_sync(lnk, buf_desc) << lnk->link_idx; + + if (is_rmb || buf_desc->is_vm) { + /* create a new memory region for the RMB or vzalloced sndbuf */ + access_flags = is_rmb ? + IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : + IB_ACCESS_LOCAL_WRITE; + + rc = smc_ib_get_memory_region(lnk->roce_pd, access_flags, buf_desc, lnk->link_idx); if (rc) goto buf_unmap; - smc_ib_sync_sg_for_device(lnk, buf_desc, DMA_FROM_DEVICE); + smc_ib_sync_sg_for_device(lnk, buf_desc, + is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE); } buf_desc->is_map_ib[lnk->link_idx] = true; return 0; @@ -2037,20 +2084,23 @@ free_table: return rc; } -/* register a new rmb on IB device, +/* register a new buf on IB device, rmb or vzalloced sndbuf * must be called under lgr->llc_conf_mutex lock */ -int smcr_link_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc) +int smcr_link_reg_buf(struct smc_link *link, struct smc_buf_desc *buf_desc) { if (list_empty(&link->lgr->list)) return -ENOLINK; - if (!rmb_desc->is_reg_mr[link->link_idx]) { - /* register memory region for new rmb */ - if (smc_wr_reg_send(link, rmb_desc->mr_rx[link->link_idx])) { - rmb_desc->is_reg_err = true; + if (!buf_desc->is_reg_mr[link->link_idx]) { + /* register memory region for new buf */ + if (buf_desc->is_vm) + buf_desc->mr[link->link_idx]->iova = + (uintptr_t)buf_desc->cpu_addr; + if (smc_wr_reg_send(link, buf_desc->mr[link->link_idx])) { + buf_desc->is_reg_err = true; return -EFAULT; } - rmb_desc->is_reg_mr[link->link_idx] = true; + buf_desc->is_reg_mr[link->link_idx] = true; } return 0; } @@ -2102,18 +2152,38 @@ int smcr_buf_reg_lgr(struct smc_link *lnk) struct smc_buf_desc *buf_desc, *bf; int i, rc = 0; + /* reg all RMBs for a new link */ mutex_lock(&lgr->rmbs_lock); for (i = 0; i < SMC_RMBE_SIZES; i++) { list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) { if (!buf_desc->used) continue; - rc = smcr_link_reg_rmb(lnk, buf_desc); - if (rc) - goto out; + rc = smcr_link_reg_buf(lnk, buf_desc); + if (rc) { + mutex_unlock(&lgr->rmbs_lock); + return rc; + } } } -out: mutex_unlock(&lgr->rmbs_lock); + + if (lgr->buf_type == SMCR_PHYS_CONT_BUFS) + return rc; + + /* reg all vzalloced sndbufs for a new link */ + mutex_lock(&lgr->sndbufs_lock); + for (i = 0; i < SMC_RMBE_SIZES; i++) { + list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i], list) { + if (!buf_desc->used || !buf_desc->is_vm) + continue; + rc = smcr_link_reg_buf(lnk, buf_desc); + if (rc) { + mutex_unlock(&lgr->sndbufs_lock); + return rc; + } + } + } + mutex_unlock(&lgr->sndbufs_lock); return rc; } @@ -2127,18 +2197,39 @@ static struct smc_buf_desc *smcr_new_buf_create(struct smc_link_group *lgr, if (!buf_desc) return ERR_PTR(-ENOMEM); - buf_desc->order = get_order(bufsize); - buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN | - __GFP_NOMEMALLOC | __GFP_COMP | - __GFP_NORETRY | __GFP_ZERO, - buf_desc->order); - if (!buf_desc->pages) { - kfree(buf_desc); - return ERR_PTR(-EAGAIN); - } - buf_desc->cpu_addr = (void *)page_address(buf_desc->pages); - buf_desc->len = bufsize; + switch (lgr->buf_type) { + case SMCR_PHYS_CONT_BUFS: + case SMCR_MIXED_BUFS: + buf_desc->order = get_order(bufsize); + buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN | + __GFP_NOMEMALLOC | __GFP_COMP | + __GFP_NORETRY | __GFP_ZERO, + buf_desc->order); + if (buf_desc->pages) { + buf_desc->cpu_addr = + (void *)page_address(buf_desc->pages); + buf_desc->len = bufsize; + buf_desc->is_vm = false; + break; + } + if (lgr->buf_type == SMCR_PHYS_CONT_BUFS) + goto out; + fallthrough; // try virtually continguous buf + case SMCR_VIRT_CONT_BUFS: + buf_desc->order = get_order(bufsize); + buf_desc->cpu_addr = vzalloc(PAGE_SIZE << buf_desc->order); + if (!buf_desc->cpu_addr) + goto out; + buf_desc->pages = NULL; + buf_desc->len = bufsize; + buf_desc->is_vm = true; + break; + } return buf_desc; + +out: + kfree(buf_desc); + return ERR_PTR(-EAGAIN); } /* map buf_desc on all usable links, @@ -2234,6 +2325,7 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb) /* check for reusable slot in the link group */ buf_desc = smc_buf_get_slot(bufsize_short, lock, buf_list); if (buf_desc) { + buf_desc->is_dma_need_sync = 0; SMC_STAT_RMB_SIZE(smc, is_smcd, is_rmb, bufsize); SMC_STAT_BUF_REUSE(smc, is_smcd, is_rmb); break; /* found reusable slot */ @@ -2268,7 +2360,7 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb) if (!is_smcd) { if (smcr_buf_map_usable_links(lgr, buf_desc, is_rmb)) { - smcr_buf_unuse(buf_desc, lgr); + smcr_buf_unuse(buf_desc, is_rmb, lgr); return -ENOMEM; } } @@ -2290,16 +2382,10 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb) return 0; } -void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn) -{ - if (!smc_conn_lgr_valid(conn) || conn->lgr->is_smcd || - !smc_link_active(conn->lnk)) - return; - smc_ib_sync_sg_for_cpu(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE); -} - void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn) { + if (!conn->sndbuf_desc->is_dma_need_sync) + return; if (!smc_conn_lgr_valid(conn) || conn->lgr->is_smcd || !smc_link_active(conn->lnk)) return; @@ -2310,6 +2396,8 @@ void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn) { int i; + if (!conn->rmb_desc->is_dma_need_sync) + return; if (!smc_conn_lgr_valid(conn) || conn->lgr->is_smcd) return; for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { @@ -2320,20 +2408,6 @@ void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn) } } -void smc_rmb_sync_sg_for_device(struct smc_connection *conn) -{ - int i; - - if (!smc_conn_lgr_valid(conn) || conn->lgr->is_smcd) - return; - for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { - if (!smc_link_active(&conn->lgr->lnk[i])) - continue; - smc_ib_sync_sg_for_device(&conn->lgr->lnk[i], conn->rmb_desc, - DMA_FROM_DEVICE); - } -} - /* create the send and receive buffer for an SMC socket; * receive buffers are called RMBs; * (even though the SMC protocol allows more than one RMB-element per RMB, diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h index 4cb03e942364..fe8b524ad846 100644 --- a/net/smc/smc_core.h +++ b/net/smc/smc_core.h @@ -168,9 +168,11 @@ struct smc_buf_desc { struct { /* SMC-R */ struct sg_table sgt[SMC_LINKS_PER_LGR_MAX]; /* virtual buffer */ - struct ib_mr *mr_rx[SMC_LINKS_PER_LGR_MAX]; - /* for rmb only: memory region + struct ib_mr *mr[SMC_LINKS_PER_LGR_MAX]; + /* memory region: for rmb and + * vzalloced sndbuf * incl. rkey provided to peer + * and lkey provided to local */ u32 order; /* allocation order */ @@ -180,8 +182,11 @@ struct smc_buf_desc { /* mem region registered */ u8 is_map_ib[SMC_LINKS_PER_LGR_MAX]; /* mem region mapped to lnk */ + u8 is_dma_need_sync; u8 is_reg_err; /* buffer registration err */ + u8 is_vm; + /* virtually contiguous */ }; struct { /* SMC-D */ unsigned short sba_idx; @@ -216,6 +221,12 @@ enum smc_lgr_type { /* redundancy state of lgr */ SMC_LGR_ASYMMETRIC_LOCAL, /* local has 1, peer 2 active RNICs */ }; +enum smcr_buf_type { /* types of SMC-R sndbufs and RMBs */ + SMCR_PHYS_CONT_BUFS = 0, + SMCR_VIRT_CONT_BUFS = 1, + SMCR_MIXED_BUFS = 2, +}; + enum smc_llc_flowtype { SMC_LLC_FLOW_NONE = 0, SMC_LLC_FLOW_ADD_LINK = 2, @@ -277,6 +288,7 @@ struct smc_link_group { /* used rtoken elements */ u8 next_link_id; enum smc_lgr_type type; + enum smcr_buf_type buf_type; /* redundancy state */ u8 pnet_id[SMC_MAX_PNETID_LEN + 1]; /* pnet id of this lgr */ @@ -513,10 +525,8 @@ void smc_rtoken_set(struct smc_link_group *lgr, int link_idx, int link_idx_new, __be32 nw_rkey_known, __be64 nw_vaddr, __be32 nw_rkey); void smc_rtoken_set2(struct smc_link_group *lgr, int rtok_idx, int link_id, __be64 nw_vaddr, __be32 nw_rkey); -void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn); void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn); void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn); -void smc_rmb_sync_sg_for_device(struct smc_connection *conn); int smc_vlan_by_tcpsk(struct socket *clcsock, struct smc_init_info *ini); void smc_conn_free(struct smc_connection *conn); @@ -537,7 +547,7 @@ int smcr_buf_reg_lgr(struct smc_link *lnk); void smcr_lgr_set_type(struct smc_link_group *lgr, enum smc_lgr_type new_type); void smcr_lgr_set_type_asym(struct smc_link_group *lgr, enum smc_lgr_type new_type, int asym_lnk_idx); -int smcr_link_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc); +int smcr_link_reg_buf(struct smc_link *link, struct smc_buf_desc *rmb_desc); struct smc_link *smc_switch_conns(struct smc_link_group *lgr, struct smc_link *from_lnk, bool is_dev_err); void smcr_link_down_cond(struct smc_link *lnk); diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c index dcda4165d107..854772dd52fd 100644 --- a/net/smc/smc_ib.c +++ b/net/smc/smc_ib.c @@ -698,7 +698,7 @@ static int smc_ib_map_mr_sg(struct smc_buf_desc *buf_slot, u8 link_idx) int sg_num; /* map the largest prefix of a dma mapped SG list */ - sg_num = ib_map_mr_sg(buf_slot->mr_rx[link_idx], + sg_num = ib_map_mr_sg(buf_slot->mr[link_idx], buf_slot->sgt[link_idx].sgl, buf_slot->sgt[link_idx].orig_nents, &offset, PAGE_SIZE); @@ -710,25 +710,49 @@ static int smc_ib_map_mr_sg(struct smc_buf_desc *buf_slot, u8 link_idx) int smc_ib_get_memory_region(struct ib_pd *pd, int access_flags, struct smc_buf_desc *buf_slot, u8 link_idx) { - if (buf_slot->mr_rx[link_idx]) + if (buf_slot->mr[link_idx]) return 0; /* already done */ - buf_slot->mr_rx[link_idx] = + buf_slot->mr[link_idx] = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 1 << buf_slot->order); - if (IS_ERR(buf_slot->mr_rx[link_idx])) { + if (IS_ERR(buf_slot->mr[link_idx])) { int rc; - rc = PTR_ERR(buf_slot->mr_rx[link_idx]); - buf_slot->mr_rx[link_idx] = NULL; + rc = PTR_ERR(buf_slot->mr[link_idx]); + buf_slot->mr[link_idx] = NULL; return rc; } - if (smc_ib_map_mr_sg(buf_slot, link_idx) != 1) + if (smc_ib_map_mr_sg(buf_slot, link_idx) != + buf_slot->sgt[link_idx].orig_nents) return -EINVAL; return 0; } +bool smc_ib_is_sg_need_sync(struct smc_link *lnk, + struct smc_buf_desc *buf_slot) +{ + struct scatterlist *sg; + unsigned int i; + bool ret = false; + + /* for now there is just one DMA address */ + for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg, + buf_slot->sgt[lnk->link_idx].nents, i) { + if (!sg_dma_len(sg)) + break; + if (dma_need_sync(lnk->smcibdev->ibdev->dma_device, + sg_dma_address(sg))) { + ret = true; + goto out; + } + } + +out: + return ret; +} + /* synchronize buffer usage for cpu access */ void smc_ib_sync_sg_for_cpu(struct smc_link *lnk, struct smc_buf_desc *buf_slot, @@ -737,6 +761,9 @@ void smc_ib_sync_sg_for_cpu(struct smc_link *lnk, struct scatterlist *sg; unsigned int i; + if (!(buf_slot->is_dma_need_sync & (1U << lnk->link_idx))) + return; + /* for now there is just one DMA address */ for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg, buf_slot->sgt[lnk->link_idx].nents, i) { @@ -757,6 +784,9 @@ void smc_ib_sync_sg_for_device(struct smc_link *lnk, struct scatterlist *sg; unsigned int i; + if (!(buf_slot->is_dma_need_sync & (1U << lnk->link_idx))) + return; + /* for now there is just one DMA address */ for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg, buf_slot->sgt[lnk->link_idx].nents, i) { diff --git a/net/smc/smc_ib.h b/net/smc/smc_ib.h index 5d8b49c57f50..034295676e88 100644 --- a/net/smc/smc_ib.h +++ b/net/smc/smc_ib.h @@ -102,6 +102,8 @@ long smc_ib_setup_per_ibdev(struct smc_ib_device *smcibdev); int smc_ib_get_memory_region(struct ib_pd *pd, int access_flags, struct smc_buf_desc *buf_slot, u8 link_idx); void smc_ib_put_memory_region(struct ib_mr *mr); +bool smc_ib_is_sg_need_sync(struct smc_link *lnk, + struct smc_buf_desc *buf_slot); void smc_ib_sync_sg_for_cpu(struct smc_link *lnk, struct smc_buf_desc *buf_slot, enum dma_data_direction data_direction); diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c index c4d057b2941d..175026ae33ae 100644 --- a/net/smc/smc_llc.c +++ b/net/smc/smc_llc.c @@ -505,19 +505,22 @@ static int smc_llc_send_confirm_rkey(struct smc_link *send_link, if (smc_link_active(link) && link != send_link) { rkeyllc->rtoken[rtok_ix].link_id = link->link_id; rkeyllc->rtoken[rtok_ix].rmb_key = - htonl(rmb_desc->mr_rx[link->link_idx]->rkey); - rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( - (u64)sg_dma_address( - rmb_desc->sgt[link->link_idx].sgl)); + htonl(rmb_desc->mr[link->link_idx]->rkey); + rkeyllc->rtoken[rtok_ix].rmb_vaddr = rmb_desc->is_vm ? + cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) : + cpu_to_be64((u64)sg_dma_address + (rmb_desc->sgt[link->link_idx].sgl)); rtok_ix++; } } /* rkey of send_link is in rtoken[0] */ rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; rkeyllc->rtoken[0].rmb_key = - htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); - rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( - (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); + htonl(rmb_desc->mr[send_link->link_idx]->rkey); + rkeyllc->rtoken[0].rmb_vaddr = rmb_desc->is_vm ? + cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) : + cpu_to_be64((u64)sg_dma_address + (rmb_desc->sgt[send_link->link_idx].sgl)); /* send llc message */ rc = smc_wr_tx_send(send_link, pend); put_out: @@ -544,7 +547,7 @@ static int smc_llc_send_delete_rkey(struct smc_link *link, rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY; smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc)); rkeyllc->num_rkeys = 1; - rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); + rkeyllc->rkey[0] = htonl(rmb_desc->mr[link->link_idx]->rkey); /* send llc message */ rc = smc_wr_tx_send(link, pend); put_out: @@ -614,9 +617,10 @@ static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext, if (!buf_pos) break; rmb = buf_pos; - ext->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); - ext->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); - ext->rt[i].rmb_vaddr_new = + ext->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey); + ext->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey); + ext->rt[i].rmb_vaddr_new = rmb->is_vm ? + cpu_to_be64((uintptr_t)rmb->cpu_addr) : cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos); while (buf_pos && !(buf_pos)->used) @@ -852,9 +856,10 @@ static int smc_llc_add_link_cont(struct smc_link *link, } rmb = *buf_pos; - addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); - addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); - addc_llc->rt[i].rmb_vaddr_new = + addc_llc->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey); + addc_llc->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey); + addc_llc->rt[i].rmb_vaddr_new = rmb->is_vm ? + cpu_to_be64((uintptr_t)rmb->cpu_addr) : cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); (*num_rkeys_todo)--; @@ -2122,7 +2127,7 @@ void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) init_waitqueue_head(&lgr->llc_flow_waiter); init_waitqueue_head(&lgr->llc_msg_waiter); mutex_init(&lgr->llc_conf_mutex); - lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; + lgr->llc_testlink_time = READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time); } /* called after lgr was removed from lgr_list */ diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c index 338b9ef806e8..17c5aee7ee4f 100644 --- a/net/smc/smc_rx.c +++ b/net/smc/smc_rx.c @@ -145,35 +145,93 @@ static void smc_rx_spd_release(struct splice_pipe_desc *spd, static int smc_rx_splice(struct pipe_inode_info *pipe, char *src, size_t len, struct smc_sock *smc) { + struct smc_link_group *lgr = smc->conn.lgr; + int offset = offset_in_page(src); + struct partial_page *partial; struct splice_pipe_desc spd; - struct partial_page partial; - struct smc_spd_priv *priv; - int bytes; + struct smc_spd_priv **priv; + struct page **pages; + int bytes, nr_pages; + int i; - priv = kzalloc(sizeof(*priv), GFP_KERNEL); + nr_pages = !lgr->is_smcd && smc->conn.rmb_desc->is_vm ? + PAGE_ALIGN(len + offset) / PAGE_SIZE : 1; + + pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); + if (!pages) + goto out; + partial = kcalloc(nr_pages, sizeof(*partial), GFP_KERNEL); + if (!partial) + goto out_page; + priv = kcalloc(nr_pages, sizeof(*priv), GFP_KERNEL); if (!priv) - return -ENOMEM; - priv->len = len; - priv->smc = smc; - partial.offset = src - (char *)smc->conn.rmb_desc->cpu_addr; - partial.len = len; - partial.private = (unsigned long)priv; - - spd.nr_pages_max = 1; - spd.nr_pages = 1; - spd.pages = &smc->conn.rmb_desc->pages; - spd.partial = &partial; + goto out_part; + for (i = 0; i < nr_pages; i++) { + priv[i] = kzalloc(sizeof(**priv), GFP_KERNEL); + if (!priv[i]) + goto out_priv; + } + + if (lgr->is_smcd || + (!lgr->is_smcd && !smc->conn.rmb_desc->is_vm)) { + /* smcd or smcr that uses physically contiguous RMBs */ + priv[0]->len = len; + priv[0]->smc = smc; + partial[0].offset = src - (char *)smc->conn.rmb_desc->cpu_addr; + partial[0].len = len; + partial[0].private = (unsigned long)priv[0]; + pages[0] = smc->conn.rmb_desc->pages; + } else { + int size, left = len; + void *buf = src; + /* smcr that uses virtually contiguous RMBs*/ + for (i = 0; i < nr_pages; i++) { + size = min_t(int, PAGE_SIZE - offset, left); + priv[i]->len = size; + priv[i]->smc = smc; + pages[i] = vmalloc_to_page(buf); + partial[i].offset = offset; + partial[i].len = size; + partial[i].private = (unsigned long)priv[i]; + buf += size / sizeof(*buf); + left -= size; + offset = 0; + } + } + spd.nr_pages_max = nr_pages; + spd.nr_pages = nr_pages; + spd.pages = pages; + spd.partial = partial; spd.ops = &smc_pipe_ops; spd.spd_release = smc_rx_spd_release; bytes = splice_to_pipe(pipe, &spd); if (bytes > 0) { sock_hold(&smc->sk); - get_page(smc->conn.rmb_desc->pages); + if (!lgr->is_smcd && smc->conn.rmb_desc->is_vm) { + for (i = 0; i < PAGE_ALIGN(bytes + offset) / PAGE_SIZE; i++) + get_page(pages[i]); + } else { + get_page(smc->conn.rmb_desc->pages); + } atomic_add(bytes, &smc->conn.splice_pending); } + kfree(priv); + kfree(partial); + kfree(pages); return bytes; + +out_priv: + for (i = (i - 1); i >= 0; i--) + kfree(priv[i]); + kfree(priv); +out_part: + kfree(partial); +out_page: + kfree(pages); +out: + return -ENOMEM; } static int smc_rx_data_available_and_no_splice_pend(struct smc_connection *conn) @@ -413,7 +471,6 @@ copy: if (rc < 0) { if (!read_done) read_done = -EFAULT; - smc_rmb_sync_sg_for_device(conn); goto out; } } @@ -427,7 +484,6 @@ copy: chunk_len_sum += chunk_len; chunk_off = 0; /* modulo offset in recv ring buffer */ } - smc_rmb_sync_sg_for_device(conn); /* update cursors */ if (!(flags & MSG_PEEK)) { diff --git a/net/smc/smc_sysctl.c b/net/smc/smc_sysctl.c index cf3ab1334c00..0613868fdb97 100644 --- a/net/smc/smc_sysctl.c +++ b/net/smc/smc_sysctl.c @@ -15,6 +15,7 @@ #include <net/net_namespace.h> #include "smc.h" +#include "smc_core.h" #include "smc_sysctl.h" static struct ctl_table smc_table[] = { @@ -25,6 +26,15 @@ static struct ctl_table smc_table[] = { .mode = 0644, .proc_handler = proc_douintvec, }, + { + .procname = "smcr_buf_type", + .data = &init_net.smc.sysctl_smcr_buf_type, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, + }, { } }; @@ -49,6 +59,7 @@ int __net_init smc_sysctl_net_init(struct net *net) goto err_reg; net->smc.sysctl_autocorking_size = SMC_AUTOCORKING_DEFAULT_SIZE; + net->smc.sysctl_smcr_buf_type = SMCR_PHYS_CONT_BUFS; return 0; diff --git a/net/smc/smc_tx.c b/net/smc/smc_tx.c index 805a546e8c04..4e8377657a62 100644 --- a/net/smc/smc_tx.c +++ b/net/smc/smc_tx.c @@ -246,7 +246,6 @@ int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len) tx_cnt_prep); chunk_len_sum = chunk_len; chunk_off = tx_cnt_prep; - smc_sndbuf_sync_sg_for_cpu(conn); for (chunk = 0; chunk < 2; chunk++) { rc = memcpy_from_msg(sndbuf_base + chunk_off, msg, chunk_len); @@ -384,6 +383,7 @@ static int smcr_tx_rdma_writes(struct smc_connection *conn, size_t len, dma_addr_t dma_addr = sg_dma_address(conn->sndbuf_desc->sgt[link->link_idx].sgl); + u64 virt_addr = (uintptr_t)conn->sndbuf_desc->cpu_addr; int src_len_sum = src_len, dst_len_sum = dst_len; int sent_count = src_off; int srcchunk, dstchunk; @@ -396,7 +396,7 @@ static int smcr_tx_rdma_writes(struct smc_connection *conn, size_t len, u64 base_addr = dma_addr; if (dst_len < link->qp_attr.cap.max_inline_data) { - base_addr = (uintptr_t)conn->sndbuf_desc->cpu_addr; + base_addr = virt_addr; wr->wr.send_flags |= IB_SEND_INLINE; } else { wr->wr.send_flags &= ~IB_SEND_INLINE; @@ -404,8 +404,12 @@ static int smcr_tx_rdma_writes(struct smc_connection *conn, size_t len, num_sges = 0; for (srcchunk = 0; srcchunk < 2; srcchunk++) { - sge[srcchunk].addr = base_addr + src_off; + sge[srcchunk].addr = conn->sndbuf_desc->is_vm ? + (virt_addr + src_off) : (base_addr + src_off); sge[srcchunk].length = src_len; + if (conn->sndbuf_desc->is_vm) + sge[srcchunk].lkey = + conn->sndbuf_desc->mr[link->link_idx]->lkey; num_sges++; src_off += src_len; diff --git a/net/socket.c b/net/socket.c index 3d7eb2a79e82..b6bd4cf44d3f 100644 --- a/net/socket.c +++ b/net/socket.c @@ -2103,6 +2103,7 @@ int __sys_sendto(int fd, void __user *buff, size_t len, unsigned int flags, msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_namelen = 0; + msg.msg_ubuf = NULL; if (addr) { err = move_addr_to_kernel(addr, addr_len, &address); if (err < 0) @@ -2402,6 +2403,7 @@ int __copy_msghdr_from_user(struct msghdr *kmsg, return -EMSGSIZE; kmsg->msg_iocb = NULL; + kmsg->msg_ubuf = NULL; *uiov = msg.msg_iov; *nsegs = msg.msg_iovlen; return 0; diff --git a/net/tls/Makefile b/net/tls/Makefile index f1ffbfe8968d..e41c800489ac 100644 --- a/net/tls/Makefile +++ b/net/tls/Makefile @@ -7,7 +7,7 @@ CFLAGS_trace.o := -I$(src) obj-$(CONFIG_TLS) += tls.o -tls-y := tls_main.o tls_sw.o tls_proc.o trace.o +tls-y := tls_main.o tls_sw.o tls_proc.o trace.o tls_strp.o tls-$(CONFIG_TLS_TOE) += tls_toe.o tls-$(CONFIG_TLS_DEVICE) += tls_device.o tls_device_fallback.o diff --git a/net/tls/tls.h b/net/tls/tls.h index e0ccc96a0850..3740740504e3 100644 --- a/net/tls/tls.h +++ b/net/tls/tls.h @@ -39,6 +39,9 @@ #include <linux/skmsg.h> #include <net/tls.h> +#define TLS_PAGE_ORDER (min_t(unsigned int, PAGE_ALLOC_COSTLY_ORDER, \ + TLS_MAX_PAYLOAD_SIZE >> PAGE_SHIFT)) + #define __TLS_INC_STATS(net, field) \ __SNMP_INC_STATS((net)->mib.tls_statistics, field) #define TLS_INC_STATS(net, field) \ @@ -118,13 +121,15 @@ void tls_device_write_space(struct sock *sk, struct tls_context *ctx); int tls_process_cmsg(struct sock *sk, struct msghdr *msg, unsigned char *record_type); -int decrypt_skb(struct sock *sk, struct sk_buff *skb, - struct scatterlist *sgout); +int decrypt_skb(struct sock *sk, struct scatterlist *sgout); int tls_sw_fallback_init(struct sock *sk, struct tls_offload_context_tx *offload_ctx, struct tls_crypto_info *crypto_info); +int tls_strp_msg_hold(struct sock *sk, struct sk_buff *skb, + struct sk_buff_head *dst); + static inline struct tls_msg *tls_msg(struct sk_buff *skb) { struct sk_skb_cb *scb = (struct sk_skb_cb *)skb->cb; @@ -132,6 +137,11 @@ static inline struct tls_msg *tls_msg(struct sk_buff *skb) return &scb->tls; } +static inline struct sk_buff *tls_strp_msg(struct tls_sw_context_rx *ctx) +{ + return ctx->recv_pkt; +} + #ifdef CONFIG_TLS_DEVICE int tls_device_init(void); void tls_device_cleanup(void); @@ -140,8 +150,7 @@ void tls_device_free_resources_tx(struct sock *sk); int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx); void tls_device_offload_cleanup_rx(struct sock *sk); void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq); -int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx, - struct sk_buff *skb, struct strp_msg *rxm); +int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx); #else static inline int tls_device_init(void) { return 0; } static inline void tls_device_cleanup(void) {} @@ -165,8 +174,7 @@ static inline void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) {} static inline int -tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx, - struct sk_buff *skb, struct strp_msg *rxm) +tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx) { return 0; } diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 6f9dff1c11f6..b1fcd61836d1 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -98,13 +98,16 @@ static void tls_device_queue_ctx_destruction(struct tls_context *ctx) unsigned long flags; spin_lock_irqsave(&tls_device_lock, flags); + if (unlikely(!refcount_dec_and_test(&ctx->refcount))) + goto unlock; + list_move_tail(&ctx->list, &tls_device_gc_list); /* schedule_work inside the spinlock * to make sure tls_device_down waits for that work. */ schedule_work(&tls_device_gc_work); - +unlock: spin_unlock_irqrestore(&tls_device_lock, flags); } @@ -195,8 +198,7 @@ void tls_device_sk_destruct(struct sock *sk) clean_acked_data_disable(inet_csk(sk)); } - if (refcount_dec_and_test(&tls_ctx->refcount)) - tls_device_queue_ctx_destruction(tls_ctx); + tls_device_queue_ctx_destruction(tls_ctx); } EXPORT_SYMBOL_GPL(tls_device_sk_destruct); @@ -889,14 +891,19 @@ static void tls_device_core_ctrl_rx_resync(struct tls_context *tls_ctx, } } -static int tls_device_reencrypt(struct sock *sk, struct sk_buff *skb) +static int +tls_device_reencrypt(struct sock *sk, struct tls_sw_context_rx *sw_ctx) { - struct strp_msg *rxm = strp_msg(skb); - int err = 0, offset = rxm->offset, copy, nsg, data_len, pos; - struct sk_buff *skb_iter, *unused; + int err = 0, offset, copy, nsg, data_len, pos; + struct sk_buff *skb, *skb_iter, *unused; struct scatterlist sg[1]; + struct strp_msg *rxm; char *orig_buf, *buf; + skb = tls_strp_msg(sw_ctx); + rxm = strp_msg(skb); + offset = rxm->offset; + orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE + TLS_CIPHER_AES_GCM_128_IV_SIZE, sk->sk_allocation); if (!orig_buf) @@ -919,7 +926,7 @@ static int tls_device_reencrypt(struct sock *sk, struct sk_buff *skb) goto free_buf; /* We are interested only in the decrypted data not the auth */ - err = decrypt_skb(sk, skb, sg); + err = decrypt_skb(sk, sg); if (err != -EBADMSG) goto free_buf; else @@ -974,10 +981,12 @@ free_buf: return err; } -int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx, - struct sk_buff *skb, struct strp_msg *rxm) +int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx) { struct tls_offload_context_rx *ctx = tls_offload_ctx_rx(tls_ctx); + struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx); + struct sk_buff *skb = tls_strp_msg(sw_ctx); + struct strp_msg *rxm = strp_msg(skb); int is_decrypted = skb->decrypted; int is_encrypted = !is_decrypted; struct sk_buff *skb_iter; @@ -1000,7 +1009,7 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx, * likely have initial fragments decrypted, and final ones not * decrypted. We need to reencrypt that single SKB. */ - return tls_device_reencrypt(sk, skb); + return tls_device_reencrypt(sk, sw_ctx); } /* Return immediately if the record is either entirely plaintext or @@ -1017,7 +1026,7 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx, } ctx->resync_nh_reset = 1; - return tls_device_reencrypt(sk, skb); + return tls_device_reencrypt(sk, sw_ctx); } static void tls_device_attach(struct tls_context *ctx, struct sock *sk, diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c new file mode 100644 index 000000000000..9ccab79a6e1e --- /dev/null +++ b/net/tls/tls_strp.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/skbuff.h> + +#include "tls.h" + +int tls_strp_msg_hold(struct sock *sk, struct sk_buff *skb, + struct sk_buff_head *dst) +{ + struct sk_buff *clone; + + clone = skb_clone(skb, sk->sk_allocation); + if (!clone) + return -ENOMEM; + __skb_queue_tail(dst, clone); + return 0; +} diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 68d79ee48a56..ed5e6f1df9c7 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -47,9 +47,13 @@ #include "tls.h" struct tls_decrypt_arg { + struct_group(inargs, bool zc; bool async; u8 tail; + ); + + struct sk_buff *skb; }; struct tls_decrypt_ctx { @@ -180,39 +184,22 @@ static void tls_decrypt_done(struct crypto_async_request *req, int err) struct scatterlist *sgin = aead_req->src; struct tls_sw_context_rx *ctx; struct tls_context *tls_ctx; - struct tls_prot_info *prot; struct scatterlist *sg; - struct sk_buff *skb; unsigned int pages; + struct sock *sk; - skb = (struct sk_buff *)req->data; - tls_ctx = tls_get_ctx(skb->sk); + sk = (struct sock *)req->data; + tls_ctx = tls_get_ctx(sk); ctx = tls_sw_ctx_rx(tls_ctx); - prot = &tls_ctx->prot_info; /* Propagate if there was an err */ if (err) { if (err == -EBADMSG) - TLS_INC_STATS(sock_net(skb->sk), - LINUX_MIB_TLSDECRYPTERROR); + TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTERROR); ctx->async_wait.err = err; - tls_err_abort(skb->sk, err); - } else { - struct strp_msg *rxm = strp_msg(skb); - - /* No TLS 1.3 support with async crypto */ - WARN_ON(prot->tail_size); - - rxm->offset += prot->prepend_size; - rxm->full_len -= prot->overhead_size; + tls_err_abort(sk, err); } - /* After using skb->sk to propagate sk through crypto async callback - * we need to NULL it again. - */ - skb->sk = NULL; - - /* Free the destination pages if skb was not decrypted inplace */ if (sgout != sgin) { /* Skip the first S/G entry as it points to AAD */ @@ -232,7 +219,6 @@ static void tls_decrypt_done(struct crypto_async_request *req, int err) } static int tls_do_decryption(struct sock *sk, - struct sk_buff *skb, struct scatterlist *sgin, struct scatterlist *sgout, char *iv_recv, @@ -252,16 +238,9 @@ static int tls_do_decryption(struct sock *sk, (u8 *)iv_recv); if (darg->async) { - /* Using skb->sk to push sk through to crypto async callback - * handler. This allows propagating errors up to the socket - * if needed. It _must_ be cleared in the async handler - * before consume_skb is called. We _know_ skb->sk is NULL - * because it is a clone from strparser. - */ - skb->sk = sk; aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG, - tls_decrypt_done, skb); + tls_decrypt_done, sk); atomic_inc(&ctx->decrypt_pending); } else { aead_request_set_callback(aead_req, @@ -1404,51 +1383,90 @@ out: return rc; } +static struct sk_buff * +tls_alloc_clrtxt_skb(struct sock *sk, struct sk_buff *skb, + unsigned int full_len) +{ + struct strp_msg *clr_rxm; + struct sk_buff *clr_skb; + int err; + + clr_skb = alloc_skb_with_frags(0, full_len, TLS_PAGE_ORDER, + &err, sk->sk_allocation); + if (!clr_skb) + return NULL; + + skb_copy_header(clr_skb, skb); + clr_skb->len = full_len; + clr_skb->data_len = full_len; + + clr_rxm = strp_msg(clr_skb); + clr_rxm->offset = 0; + + return clr_skb; +} + +/* Decrypt handlers + * + * tls_decrypt_sg() and tls_decrypt_device() are decrypt handlers. + * They must transform the darg in/out argument are as follows: + * | Input | Output + * ------------------------------------------------------------------- + * zc | Zero-copy decrypt allowed | Zero-copy performed + * async | Async decrypt allowed | Async crypto used / in progress + * skb | * | Output skb + */ + /* This function decrypts the input skb into either out_iov or in out_sg - * or in skb buffers itself. The input parameter 'zc' indicates if + * or in skb buffers itself. The input parameter 'darg->zc' indicates if * zero-copy mode needs to be tried or not. With zero-copy mode, either * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are * NULL, then the decryption happens inside skb buffers itself, i.e. - * zero-copy gets disabled and 'zc' is updated. + * zero-copy gets disabled and 'darg->zc' is updated. */ - -static int decrypt_internal(struct sock *sk, struct sk_buff *skb, - struct iov_iter *out_iov, - struct scatterlist *out_sg, - struct tls_decrypt_arg *darg) +static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, + struct scatterlist *out_sg, + struct tls_decrypt_arg *darg) { struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); struct tls_prot_info *prot = &tls_ctx->prot_info; int n_sgin, n_sgout, aead_size, err, pages = 0; - struct strp_msg *rxm = strp_msg(skb); - struct tls_msg *tlm = tls_msg(skb); + struct sk_buff *skb = tls_strp_msg(ctx); + const struct strp_msg *rxm = strp_msg(skb); + const struct tls_msg *tlm = tls_msg(skb); struct aead_request *aead_req; - struct sk_buff *unused; struct scatterlist *sgin = NULL; struct scatterlist *sgout = NULL; const int data_len = rxm->full_len - prot->overhead_size; int tail_pages = !!prot->tail_size; struct tls_decrypt_ctx *dctx; + struct sk_buff *clear_skb; int iv_offset = 0; u8 *mem; + n_sgin = skb_nsg(skb, rxm->offset + prot->prepend_size, + rxm->full_len - prot->prepend_size); + if (n_sgin < 1) + return n_sgin ?: -EBADMSG; + if (darg->zc && (out_iov || out_sg)) { + clear_skb = NULL; + if (out_iov) n_sgout = 1 + tail_pages + iov_iter_npages_cap(out_iov, INT_MAX, data_len); else n_sgout = sg_nents(out_sg); - n_sgin = skb_nsg(skb, rxm->offset + prot->prepend_size, - rxm->full_len - prot->prepend_size); } else { - n_sgout = 0; darg->zc = false; - n_sgin = skb_cow_data(skb, 0, &unused); - } - if (n_sgin < 1) - return -EBADMSG; + clear_skb = tls_alloc_clrtxt_skb(sk, skb, rxm->full_len); + if (!clear_skb) + return -ENOMEM; + + n_sgout = 1 + skb_shinfo(clear_skb)->nr_frags; + } /* Increment to accommodate AAD */ n_sgin = n_sgin + 1; @@ -1460,8 +1478,10 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb, aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv); mem = kmalloc(aead_size + struct_size(dctx, sg, n_sgin + n_sgout), sk->sk_allocation); - if (!mem) - return -ENOMEM; + if (!mem) { + err = -ENOMEM; + goto exit_free_skb; + } /* Segment the allocated memory */ aead_req = (struct aead_request *)mem; @@ -1510,88 +1530,107 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb, if (err < 0) goto exit_free; - if (n_sgout) { - if (out_iov) { - sg_init_table(sgout, n_sgout); - sg_set_buf(&sgout[0], dctx->aad, prot->aad_size); + if (clear_skb) { + sg_init_table(sgout, n_sgout); + sg_set_buf(&sgout[0], dctx->aad, prot->aad_size); - err = tls_setup_from_iter(out_iov, data_len, - &pages, &sgout[1], - (n_sgout - 1 - tail_pages)); - if (err < 0) - goto fallback_to_reg_recv; + err = skb_to_sgvec(clear_skb, &sgout[1], prot->prepend_size, + data_len + prot->tail_size); + if (err < 0) + goto exit_free; + } else if (out_iov) { + sg_init_table(sgout, n_sgout); + sg_set_buf(&sgout[0], dctx->aad, prot->aad_size); - if (prot->tail_size) { - sg_unmark_end(&sgout[pages]); - sg_set_buf(&sgout[pages + 1], &dctx->tail, - prot->tail_size); - sg_mark_end(&sgout[pages + 1]); - } - } else if (out_sg) { - memcpy(sgout, out_sg, n_sgout * sizeof(*sgout)); - } else { - goto fallback_to_reg_recv; + err = tls_setup_from_iter(out_iov, data_len, &pages, &sgout[1], + (n_sgout - 1 - tail_pages)); + if (err < 0) + goto exit_free_pages; + + if (prot->tail_size) { + sg_unmark_end(&sgout[pages]); + sg_set_buf(&sgout[pages + 1], &dctx->tail, + prot->tail_size); + sg_mark_end(&sgout[pages + 1]); } - } else { -fallback_to_reg_recv: - sgout = sgin; - pages = 0; - darg->zc = false; + } else if (out_sg) { + memcpy(sgout, out_sg, n_sgout * sizeof(*sgout)); } /* Prepare and submit AEAD request */ - err = tls_do_decryption(sk, skb, sgin, sgout, dctx->iv, + err = tls_do_decryption(sk, sgin, sgout, dctx->iv, data_len + prot->tail_size, aead_req, darg); - if (darg->async) - return 0; + if (err) + goto exit_free_pages; + + darg->skb = clear_skb ?: tls_strp_msg(ctx); + clear_skb = NULL; + + if (unlikely(darg->async)) { + err = tls_strp_msg_hold(sk, skb, &ctx->async_hold); + if (err) + __skb_queue_tail(&ctx->async_hold, darg->skb); + return err; + } if (prot->tail_size) darg->tail = dctx->tail; +exit_free_pages: /* Release the pages in case iov was mapped to pages */ for (; pages > 0; pages--) put_page(sg_page(&sgout[pages])); exit_free: kfree(mem); +exit_free_skb: + consume_skb(clear_skb); return err; } -static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, - struct iov_iter *dest, - struct tls_decrypt_arg *darg) +static int +tls_decrypt_device(struct sock *sk, struct tls_context *tls_ctx, + struct tls_decrypt_arg *darg) +{ + struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); + int err; + + if (tls_ctx->rx_conf != TLS_HW) + return 0; + + err = tls_device_decrypted(sk, tls_ctx); + if (err <= 0) + return err; + + darg->zc = false; + darg->async = false; + darg->skb = tls_strp_msg(ctx); + ctx->recv_pkt = NULL; + return 1; +} + +static int tls_rx_one_record(struct sock *sk, struct iov_iter *dest, + struct tls_decrypt_arg *darg) { struct tls_context *tls_ctx = tls_get_ctx(sk); + struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); struct tls_prot_info *prot = &tls_ctx->prot_info; - struct strp_msg *rxm = strp_msg(skb); - struct tls_msg *tlm = tls_msg(skb); + struct strp_msg *rxm; int pad, err; - if (tlm->decrypted) { - darg->zc = false; - darg->async = false; - return 0; - } - - if (tls_ctx->rx_conf == TLS_HW) { - err = tls_device_decrypted(sk, tls_ctx, skb, rxm); - if (err < 0) - return err; - if (err > 0) { - tlm->decrypted = 1; - darg->zc = false; - darg->async = false; - goto decrypt_done; - } - } + err = tls_decrypt_device(sk, tls_ctx, darg); + if (err < 0) + return err; + if (err) + goto decrypt_done; - err = decrypt_internal(sk, skb, dest, NULL, darg); + err = tls_decrypt_sg(sk, dest, NULL, darg); if (err < 0) { if (err == -EBADMSG) TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTERROR); return err; } if (darg->async) - goto decrypt_next; + goto decrypt_done; /* If opportunistic TLS 1.3 ZC failed retry without ZC */ if (unlikely(darg->zc && prot->version == TLS_1_3_VERSION && darg->tail != TLS_RECORD_TYPE_DATA)) { @@ -1599,30 +1638,33 @@ static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, if (!darg->tail) TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXNOPADVIOL); TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSDECRYPTRETRY); - return decrypt_skb_update(sk, skb, dest, darg); + return tls_rx_one_record(sk, dest, darg); } decrypt_done: - pad = tls_padding_length(prot, skb, darg); - if (pad < 0) + if (darg->skb == ctx->recv_pkt) + ctx->recv_pkt = NULL; + + pad = tls_padding_length(prot, darg->skb, darg); + if (pad < 0) { + consume_skb(darg->skb); return pad; + } + rxm = strp_msg(darg->skb); rxm->full_len -= pad; rxm->offset += prot->prepend_size; rxm->full_len -= prot->overhead_size; - tlm->decrypted = 1; -decrypt_next: tls_advance_record_sn(sk, prot, &tls_ctx->rx); return 0; } -int decrypt_skb(struct sock *sk, struct sk_buff *skb, - struct scatterlist *sgout) +int decrypt_skb(struct sock *sk, struct scatterlist *sgout) { struct tls_decrypt_arg darg = { .zc = true, }; - return decrypt_internal(sk, skb, NULL, sgout, &darg); + return tls_decrypt_sg(sk, NULL, sgout, &darg); } static int tls_record_content_type(struct msghdr *msg, struct tls_msg *tlm, @@ -1648,6 +1690,13 @@ static int tls_record_content_type(struct msghdr *msg, struct tls_msg *tlm, return 1; } +static void tls_rx_rec_done(struct tls_sw_context_rx *ctx) +{ + consume_skb(ctx->recv_pkt); + ctx->recv_pkt = NULL; + __strp_unpause(&ctx->strp); +} + /* This function traverses the rx_list in tls receive context to copies the * decrypted records into the buffer provided by caller zero copy is not * true. Further, the records are removed from the rx_list if it is not a peek @@ -1658,7 +1707,6 @@ static int process_rx_list(struct tls_sw_context_rx *ctx, u8 *control, size_t skip, size_t len, - bool zc, bool is_peek) { struct sk_buff *skb = skb_peek(&ctx->rx_list); @@ -1692,12 +1740,10 @@ static int process_rx_list(struct tls_sw_context_rx *ctx, if (err <= 0) goto out; - if (!zc || (rxm->full_len - skip) > len) { - err = skb_copy_datagram_msg(skb, rxm->offset + skip, - msg, chunk); - if (err < 0) - goto out; - } + err = skb_copy_datagram_msg(skb, rxm->offset + skip, + msg, chunk); + if (err < 0) + goto out; len = len - chunk; copied = copied + chunk; @@ -1753,6 +1799,60 @@ tls_read_flush_backlog(struct sock *sk, struct tls_prot_info *prot, sk_flush_backlog(sk); } +static long tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx, + bool nonblock) +{ + long timeo; + int err; + + lock_sock(sk); + + timeo = sock_rcvtimeo(sk, nonblock); + + while (unlikely(ctx->reader_present)) { + DEFINE_WAIT_FUNC(wait, woken_wake_function); + + ctx->reader_contended = 1; + + add_wait_queue(&ctx->wq, &wait); + sk_wait_event(sk, &timeo, + !READ_ONCE(ctx->reader_present), &wait); + remove_wait_queue(&ctx->wq, &wait); + + if (timeo <= 0) { + err = -EAGAIN; + goto err_unlock; + } + if (signal_pending(current)) { + err = sock_intr_errno(timeo); + goto err_unlock; + } + } + + WRITE_ONCE(ctx->reader_present, 1); + + return timeo; + +err_unlock: + release_sock(sk); + return err; +} + +static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx) +{ + if (unlikely(ctx->reader_contended)) { + if (wq_has_sleeper(&ctx->wq)) + wake_up(&ctx->wq); + else + ctx->reader_contended = 0; + + WARN_ON_ONCE(!ctx->reader_present); + } + + WRITE_ONCE(ctx->reader_present, 0); + release_sock(sk); +} + int tls_sw_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, @@ -1762,9 +1862,9 @@ int tls_sw_recvmsg(struct sock *sk, struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx); struct tls_prot_info *prot = &tls_ctx->prot_info; + ssize_t decrypted = 0, async_copy_bytes = 0; struct sk_psock *psock; unsigned char control = 0; - ssize_t decrypted = 0; size_t flushed_at = 0; struct strp_msg *rxm; struct tls_msg *tlm; @@ -1782,7 +1882,9 @@ int tls_sw_recvmsg(struct sock *sk, return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR); psock = sk_psock_get(sk); - lock_sock(sk); + timeo = tls_rx_reader_lock(sk, ctx, flags & MSG_DONTWAIT); + if (timeo < 0) + return timeo; bpf_strp_enabled = sk_psock_strp_enabled(psock); /* If crypto failed the connection is broken */ @@ -1791,7 +1893,7 @@ int tls_sw_recvmsg(struct sock *sk, goto end; /* Process pending decrypted records. It must be non-zero-copy */ - err = process_rx_list(ctx, msg, &control, 0, len, false, is_peek); + err = process_rx_list(ctx, msg, &control, 0, len, is_peek); if (err < 0) goto end; @@ -1801,13 +1903,12 @@ int tls_sw_recvmsg(struct sock *sk, target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); len = len - copied; - timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); zc_capable = !bpf_strp_enabled && !is_kvec && !is_peek && ctx->zc_capable; decrypted = 0; while (len && (decrypted + copied < target || ctx->recv_pkt)) { - struct tls_decrypt_arg darg = {}; + struct tls_decrypt_arg darg; int to_decrypt, chunk; err = tls_rx_rec_wait(sk, psock, flags & MSG_DONTWAIT, timeo); @@ -1815,15 +1916,19 @@ int tls_sw_recvmsg(struct sock *sk, if (psock) { chunk = sk_msg_recvmsg(sk, psock, msg, len, flags); - if (chunk > 0) - goto leave_on_list; + if (chunk > 0) { + decrypted += chunk; + len -= chunk; + continue; + } } goto recv_end; } - skb = ctx->recv_pkt; - rxm = strp_msg(skb); - tlm = tls_msg(skb); + memset(&darg.inargs, 0, sizeof(darg.inargs)); + + rxm = strp_msg(ctx->recv_pkt); + tlm = tls_msg(ctx->recv_pkt); to_decrypt = rxm->full_len - prot->overhead_size; @@ -1837,12 +1942,16 @@ int tls_sw_recvmsg(struct sock *sk, else darg.async = false; - err = decrypt_skb_update(sk, skb, &msg->msg_iter, &darg); + err = tls_rx_one_record(sk, &msg->msg_iter, &darg); if (err < 0) { tls_err_abort(sk, -EBADMSG); goto recv_end; } + skb = darg.skb; + rxm = strp_msg(skb); + tlm = tls_msg(skb); + async |= darg.async; /* If the type of records being processed is not known yet, @@ -1853,34 +1962,36 @@ int tls_sw_recvmsg(struct sock *sk, * For tls1.3, we disable async. */ err = tls_record_content_type(msg, tlm, &control); - if (err <= 0) + if (err <= 0) { + tls_rx_rec_done(ctx); +put_on_rx_list_err: + __skb_queue_tail(&ctx->rx_list, skb); goto recv_end; + } /* periodically flush backlog, and feed strparser */ tls_read_flush_backlog(sk, prot, len, to_decrypt, decrypted + copied, &flushed_at); - ctx->recv_pkt = NULL; - __strp_unpause(&ctx->strp); - __skb_queue_tail(&ctx->rx_list, skb); - - if (async) { - /* TLS 1.2-only, to_decrypt must be text length */ - chunk = min_t(int, to_decrypt, len); -leave_on_list: - decrypted += chunk; - len -= chunk; - continue; - } /* TLS 1.3 may have updated the length by more than overhead */ chunk = rxm->full_len; + tls_rx_rec_done(ctx); if (!darg.zc) { bool partially_consumed = chunk > len; + if (async) { + /* TLS 1.2-only, to_decrypt must be text len */ + chunk = min_t(int, to_decrypt, len); + async_copy_bytes += chunk; +put_on_rx_list: + decrypted += chunk; + len -= chunk; + __skb_queue_tail(&ctx->rx_list, skb); + continue; + } + if (bpf_strp_enabled) { - /* BPF may try to queue the skb */ - __skb_unlink(skb, &ctx->rx_list); err = sk_psock_tls_strp_read(psock, skb); if (err != __SK_PASS) { rxm->offset = rxm->offset + rxm->full_len; @@ -1889,7 +2000,6 @@ leave_on_list: consume_skb(skb); continue; } - __skb_queue_tail(&ctx->rx_list, skb); } if (partially_consumed) @@ -1898,22 +2008,21 @@ leave_on_list: err = skb_copy_datagram_msg(skb, rxm->offset, msg, chunk); if (err < 0) - goto recv_end; + goto put_on_rx_list_err; if (is_peek) - goto leave_on_list; + goto put_on_rx_list; if (partially_consumed) { rxm->offset += chunk; rxm->full_len -= chunk; - goto leave_on_list; + goto put_on_rx_list; } } decrypted += chunk; len -= chunk; - __skb_unlink(skb, &ctx->rx_list); consume_skb(skb); /* Return full control message to userspace before trying @@ -1933,30 +2042,32 @@ recv_end: reinit_completion(&ctx->async_wait.completion); pending = atomic_read(&ctx->decrypt_pending); spin_unlock_bh(&ctx->decrypt_compl_lock); - if (pending) { + ret = 0; + if (pending) ret = crypto_wait_req(-EINPROGRESS, &ctx->async_wait); - if (ret) { - if (err >= 0 || err == -EINPROGRESS) - err = ret; - decrypted = 0; - goto end; - } + __skb_queue_purge(&ctx->async_hold); + + if (ret) { + if (err >= 0 || err == -EINPROGRESS) + err = ret; + decrypted = 0; + goto end; } /* Drain records from the rx_list & copy if required */ if (is_peek || is_kvec) err = process_rx_list(ctx, msg, &control, copied, - decrypted, false, is_peek); + decrypted, is_peek); else err = process_rx_list(ctx, msg, &control, 0, - decrypted, true, is_peek); + async_copy_bytes, is_peek); decrypted = max(err, 0); } copied += decrypted; end: - release_sock(sk); + tls_rx_reader_unlock(sk, ctx); if (psock) sk_psock_put(sk, psock); return copied ? : err; @@ -1973,33 +2084,34 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, struct tls_msg *tlm; struct sk_buff *skb; ssize_t copied = 0; - bool from_queue; int err = 0; long timeo; int chunk; - lock_sock(sk); + timeo = tls_rx_reader_lock(sk, ctx, flags & SPLICE_F_NONBLOCK); + if (timeo < 0) + return timeo; - timeo = sock_rcvtimeo(sk, flags & SPLICE_F_NONBLOCK); - - from_queue = !skb_queue_empty(&ctx->rx_list); - if (from_queue) { + if (!skb_queue_empty(&ctx->rx_list)) { skb = __skb_dequeue(&ctx->rx_list); } else { - struct tls_decrypt_arg darg = {}; + struct tls_decrypt_arg darg; err = tls_rx_rec_wait(sk, NULL, flags & SPLICE_F_NONBLOCK, timeo); if (err <= 0) goto splice_read_end; - skb = ctx->recv_pkt; + memset(&darg.inargs, 0, sizeof(darg.inargs)); - err = decrypt_skb_update(sk, skb, NULL, &darg); + err = tls_rx_one_record(sk, NULL, &darg); if (err < 0) { tls_err_abort(sk, -EBADMSG); goto splice_read_end; } + + tls_rx_rec_done(ctx); + skb = darg.skb; } rxm = strp_msg(skb); @@ -2008,29 +2120,29 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, /* splice does not support reading control messages */ if (tlm->control != TLS_RECORD_TYPE_DATA) { err = -EINVAL; - goto splice_read_end; + goto splice_requeue; } chunk = min_t(unsigned int, rxm->full_len, len); copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags); if (copied < 0) - goto splice_read_end; + goto splice_requeue; - if (!from_queue) { - ctx->recv_pkt = NULL; - __strp_unpause(&ctx->strp); - } if (chunk < rxm->full_len) { - __skb_queue_head(&ctx->rx_list, skb); rxm->offset += len; rxm->full_len -= len; - } else { - consume_skb(skb); + goto splice_requeue; } + consume_skb(skb); + splice_read_end: - release_sock(sk); + tls_rx_reader_unlock(sk, ctx); return copied ? : err; + +splice_requeue: + __skb_queue_head(&ctx->rx_list, skb); + goto splice_read_end; } bool tls_sw_sock_is_readable(struct sock *sk) @@ -2076,7 +2188,6 @@ static int tls_read_size(struct strparser *strp, struct sk_buff *skb) if (ret < 0) goto read_failure; - tlm->decrypted = 0; tlm->control = header[0]; data_len = ((header[4] & 0xFF) | (header[3] << 8)); @@ -2371,9 +2482,11 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) } else { crypto_init_wait(&sw_ctx_rx->async_wait); spin_lock_init(&sw_ctx_rx->decrypt_compl_lock); + init_waitqueue_head(&sw_ctx_rx->wq); crypto_info = &ctx->crypto_recv.info; cctx = &ctx->rx; skb_queue_head_init(&sw_ctx_rx->rx_list); + skb_queue_head_init(&sw_ctx_rx->async_hold); aead = &sw_ctx_rx->aead_recv; } diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 09002387987e..5b4ce6ba1bc7 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -639,8 +639,11 @@ static int __xsk_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len if (unlikely(need_wait)) return -EOPNOTSUPP; - if (sk_can_busy_loop(sk)) + if (sk_can_busy_loop(sk)) { + if (xs->zc) + __sk_mark_napi_id_once(sk, xsk_pool_get_napi_id(xs->pool)); sk_busy_loop(sk, 1); /* only support non-blocking sockets */ + } if (xs->zc && xsk_no_wakeup(sk)) return 0; diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index f1876ea61fdc..f1a0bab920a5 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -2678,8 +2678,10 @@ static int xfrm_expand_policies(const struct flowi *fl, u16 family, *num_xfrms = 0; return 0; } - if (IS_ERR(pols[0])) + if (IS_ERR(pols[0])) { + *num_pols = 0; return PTR_ERR(pols[0]); + } *num_xfrms = pols[0]->xfrm_nr; @@ -2694,6 +2696,7 @@ static int xfrm_expand_policies(const struct flowi *fl, u16 family, if (pols[1]) { if (IS_ERR(pols[1])) { xfrm_pols_put(pols, *num_pols); + *num_pols = 0; return PTR_ERR(pols[1]); } (*num_pols)++; diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 08564e0eef20..52e60e607f8a 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -2481,22 +2481,20 @@ EXPORT_SYMBOL(xfrm_user_policy); static DEFINE_SPINLOCK(xfrm_km_lock); -int xfrm_register_km(struct xfrm_mgr *km) +void xfrm_register_km(struct xfrm_mgr *km) { spin_lock_bh(&xfrm_km_lock); list_add_tail_rcu(&km->list, &xfrm_km_list); spin_unlock_bh(&xfrm_km_lock); - return 0; } EXPORT_SYMBOL(xfrm_register_km); -int xfrm_unregister_km(struct xfrm_mgr *km) +void xfrm_unregister_km(struct xfrm_mgr *km) { spin_lock_bh(&xfrm_km_lock); list_del_rcu(&km->list); spin_unlock_bh(&xfrm_km_lock); synchronize_rcu(); - return 0; } EXPORT_SYMBOL(xfrm_unregister_km); @@ -2620,7 +2618,7 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload) int err; if (family == AF_INET && - xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc) + READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc)) x->props.flags |= XFRM_STATE_NOPMTUDISC; err = -EPROTONOSUPPORT; diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 6a58fec6a1fb..2ff017117730 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -3633,10 +3633,8 @@ static int __init xfrm_user_init(void) rv = register_pernet_subsys(&xfrm_user_net_ops); if (rv < 0) return rv; - rv = xfrm_register_km(&netlink_mgr); - if (rv < 0) - unregister_pernet_subsys(&xfrm_user_net_ops); - return rv; + xfrm_register_km(&netlink_mgr); + return 0; } static void __exit xfrm_user_exit(void) diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 5002a5b9a7da..727da3c5879b 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -282,12 +282,10 @@ $(LIBBPF): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OU BPFTOOLDIR := $(TOOLS_PATH)/bpf/bpftool BPFTOOL_OUTPUT := $(abspath $(BPF_SAMPLES_PATH))/bpftool -BPFTOOL := $(BPFTOOL_OUTPUT)/bpftool -$(BPFTOOL): $(LIBBPF) $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) | $(BPFTOOL_OUTPUT) - $(MAKE) -C $(BPFTOOLDIR) srctree=$(BPF_SAMPLES_PATH)/../../ \ - OUTPUT=$(BPFTOOL_OUTPUT)/ \ - LIBBPF_OUTPUT=$(LIBBPF_OUTPUT)/ \ - LIBBPF_DESTDIR=$(LIBBPF_DESTDIR)/ +BPFTOOL := $(BPFTOOL_OUTPUT)/bootstrap/bpftool +$(BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) | $(BPFTOOL_OUTPUT) + $(MAKE) -C $(BPFTOOLDIR) srctree=$(BPF_SAMPLES_PATH)/../../ \ + OUTPUT=$(BPFTOOL_OUTPUT)/ bootstrap $(LIBBPF_OUTPUT) $(BPFTOOL_OUTPUT): $(call msg,MKDIR,$@) diff --git a/samples/bpf/fds_example.c b/samples/bpf/fds_example.c index 16dbf49e0f19..88a26f3ce201 100644 --- a/samples/bpf/fds_example.c +++ b/samples/bpf/fds_example.c @@ -17,6 +17,7 @@ #include <bpf/libbpf.h> #include "bpf_insn.h" #include "sock_example.h" +#include "bpf_util.h" #define BPF_F_PIN (1 << 0) #define BPF_F_GET (1 << 1) @@ -52,7 +53,7 @@ static int bpf_prog_create(const char *object) BPF_MOV64_IMM(BPF_REG_0, 1), BPF_EXIT_INSN(), }; - size_t insns_cnt = sizeof(insns) / sizeof(struct bpf_insn); + size_t insns_cnt = ARRAY_SIZE(insns); struct bpf_object *obj; int err; diff --git a/samples/bpf/sock_example.c b/samples/bpf/sock_example.c index a88f69504c08..5b66f2401b96 100644 --- a/samples/bpf/sock_example.c +++ b/samples/bpf/sock_example.c @@ -29,6 +29,7 @@ #include <bpf/bpf.h> #include "bpf_insn.h" #include "sock_example.h" +#include "bpf_util.h" char bpf_log_buf[BPF_LOG_BUF_SIZE]; @@ -58,7 +59,7 @@ static int test_sock(void) BPF_MOV64_IMM(BPF_REG_0, 0), /* r0 = 0 */ BPF_EXIT_INSN(), }; - size_t insns_cnt = sizeof(prog) / sizeof(struct bpf_insn); + size_t insns_cnt = ARRAY_SIZE(prog); LIBBPF_OPTS(bpf_prog_load_opts, opts, .log_buf = bpf_log_buf, .log_size = BPF_LOG_BUF_SIZE, diff --git a/samples/bpf/test_cgrp2_attach.c b/samples/bpf/test_cgrp2_attach.c index 6d90874b09c3..68ce69457afe 100644 --- a/samples/bpf/test_cgrp2_attach.c +++ b/samples/bpf/test_cgrp2_attach.c @@ -31,6 +31,7 @@ #include <bpf/bpf.h> #include "bpf_insn.h" +#include "bpf_util.h" enum { MAP_KEY_PACKETS, @@ -70,7 +71,7 @@ static int prog_load(int map_fd, int verdict) BPF_MOV64_IMM(BPF_REG_0, verdict), /* r0 = verdict */ BPF_EXIT_INSN(), }; - size_t insns_cnt = sizeof(prog) / sizeof(struct bpf_insn); + size_t insns_cnt = ARRAY_SIZE(prog); LIBBPF_OPTS(bpf_prog_load_opts, opts, .log_buf = bpf_log_buf, .log_size = BPF_LOG_BUF_SIZE, diff --git a/samples/bpf/test_lru_dist.c b/samples/bpf/test_lru_dist.c index be98ccb4952f..5efb91763d65 100644 --- a/samples/bpf/test_lru_dist.c +++ b/samples/bpf/test_lru_dist.c @@ -523,7 +523,7 @@ int main(int argc, char **argv) return -1; } - for (f = 0; f < sizeof(map_flags) / sizeof(*map_flags); f++) { + for (f = 0; f < ARRAY_SIZE(map_flags); f++) { test_lru_loss0(BPF_MAP_TYPE_LRU_HASH, map_flags[f]); test_lru_loss1(BPF_MAP_TYPE_LRU_HASH, map_flags[f]); test_parallel_lru_loss(BPF_MAP_TYPE_LRU_HASH, map_flags[f], diff --git a/samples/bpf/test_map_in_map_user.c b/samples/bpf/test_map_in_map_user.c index e8b4cc184ac9..652ec720533d 100644 --- a/samples/bpf/test_map_in_map_user.c +++ b/samples/bpf/test_map_in_map_user.c @@ -12,6 +12,8 @@ #include <bpf/bpf.h> #include <bpf/libbpf.h> +#include "bpf_util.h" + static int map_fd[7]; #define PORT_A (map_fd[0]) @@ -28,7 +30,7 @@ static const char * const test_names[] = { "Hash of Hash", }; -#define NR_TESTS (sizeof(test_names) / sizeof(*test_names)) +#define NR_TESTS ARRAY_SIZE(test_names) static void check_map_id(int inner_map_fd, int map_in_map_fd, uint32_t key) { diff --git a/samples/bpf/tracex5_user.c b/samples/bpf/tracex5_user.c index e910dc265c31..9d7d79f0d47d 100644 --- a/samples/bpf/tracex5_user.c +++ b/samples/bpf/tracex5_user.c @@ -8,6 +8,7 @@ #include <bpf/bpf.h> #include <bpf/libbpf.h> #include "trace_helpers.h" +#include "bpf_util.h" #ifdef __mips__ #define MAX_ENTRIES 6000 /* MIPS n64 syscalls start at 5000 */ @@ -24,7 +25,7 @@ static void install_accept_all_seccomp(void) BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_ALLOW), }; struct sock_fprog prog = { - .len = (unsigned short)(sizeof(filter)/sizeof(filter[0])), + .len = (unsigned short)ARRAY_SIZE(filter), .filter = filter, }; if (prctl(PR_SET_SECCOMP, 2, &prog)) diff --git a/samples/bpf/xdp_redirect_map.bpf.c b/samples/bpf/xdp_redirect_map.bpf.c index 415bac1758e3..8557c278df77 100644 --- a/samples/bpf/xdp_redirect_map.bpf.c +++ b/samples/bpf/xdp_redirect_map.bpf.c @@ -33,7 +33,7 @@ struct { } tx_port_native SEC(".maps"); /* store egress interface mac address */ -const volatile char tx_mac_addr[ETH_ALEN]; +const volatile __u8 tx_mac_addr[ETH_ALEN]; static __always_inline int xdp_redirect_map(struct xdp_md *ctx, void *redirect_map) { @@ -73,6 +73,7 @@ int xdp_redirect_map_egress(struct xdp_md *ctx) { void *data_end = (void *)(long)ctx->data_end; void *data = (void *)(long)ctx->data; + u8 *mac_addr = (u8 *) tx_mac_addr; struct ethhdr *eth = data; u64 nh_off; @@ -80,7 +81,8 @@ int xdp_redirect_map_egress(struct xdp_md *ctx) if (data + nh_off > data_end) return XDP_DROP; - __builtin_memcpy(eth->h_source, (const char *)tx_mac_addr, ETH_ALEN); + barrier_var(mac_addr); /* prevent optimizing out memcpy */ + __builtin_memcpy(eth->h_source, mac_addr, ETH_ALEN); return XDP_PASS; } diff --git a/samples/bpf/xdp_redirect_map_user.c b/samples/bpf/xdp_redirect_map_user.c index b6e4fc849577..c889a1394dc1 100644 --- a/samples/bpf/xdp_redirect_map_user.c +++ b/samples/bpf/xdp_redirect_map_user.c @@ -40,6 +40,8 @@ static const struct option long_options[] = { {} }; +static int verbose = 0; + int main(int argc, char **argv) { struct bpf_devmap_val devmap_val = {}; @@ -79,6 +81,7 @@ int main(int argc, char **argv) break; case 'v': sample_switch_mode(); + verbose = 1; break; case 's': mask |= SAMPLE_REDIRECT_MAP_CNT; @@ -134,6 +137,12 @@ int main(int argc, char **argv) ret = EXIT_FAIL; goto end_destroy; } + if (verbose) + printf("Egress ifindex:%d using src MAC %02x:%02x:%02x:%02x:%02x:%02x\n", + ifindex_out, + skel->rodata->tx_mac_addr[0], skel->rodata->tx_mac_addr[1], + skel->rodata->tx_mac_addr[2], skel->rodata->tx_mac_addr[3], + skel->rodata->tx_mac_addr[4], skel->rodata->tx_mac_addr[5]); } skel->rodata->from_match[0] = ifindex_in; diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index a0ec321469bd..dfb260de17a8 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -333,27 +333,7 @@ class PrinterRST(Printer): .. Copyright (C) All BPF authors and contributors from 2014 to present. .. See git log include/uapi/linux/bpf.h in kernel tree for details. .. -.. %%%LICENSE_START(VERBATIM) -.. Permission is granted to make and distribute verbatim copies of this -.. manual provided the copyright notice and this permission notice are -.. preserved on all copies. -.. -.. Permission is granted to copy and distribute modified versions of this -.. manual under the conditions for verbatim copying, provided that the -.. entire resulting derived work is distributed under the terms of a -.. permission notice identical to this one. -.. -.. Since the Linux kernel and libraries are constantly changing, this -.. manual page may be incorrect or out-of-date. The author(s) assume no -.. responsibility for errors or omissions, or for damages resulting from -.. the use of the information contained herein. The author(s) may not -.. have taken the same level of care in the production of this manual, -.. which is licensed free of charge, as they might when working -.. professionally. -.. -.. Formatted or processed versions of this manual, if unaccompanied by -.. the source, must acknowledge the copyright and authors of this work. -.. %%%LICENSE_END +.. SPDX-License-Identifier: Linux-man-pages-copyleft .. .. Please do not edit this file. It was generated from the documentation .. located in file include/uapi/linux/bpf.h of the Linux kernel sources diff --git a/scripts/gdb/linux/symbols.py b/scripts/gdb/linux/symbols.py index 46f7542db08c..dc07b6d12e30 100644 --- a/scripts/gdb/linux/symbols.py +++ b/scripts/gdb/linux/symbols.py @@ -180,7 +180,7 @@ lx-symbols command.""" self.breakpoint.delete() self.breakpoint = None self.breakpoint = LoadModuleBreakpoint( - "kernel/module.c:do_init_module", self) + "kernel/module/main.c:do_init_module", self) else: gdb.write("Note: symbol update on module loading not supported " "with this gdb version\n") diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c index 73917413365b..a8802b8da946 100644 --- a/security/integrity/ima/ima_policy.c +++ b/security/integrity/ima/ima_policy.c @@ -2247,6 +2247,10 @@ bool ima_appraise_signature(enum kernel_read_file_id id) if (id >= READING_MAX_ID) return false; + if (id == READING_KEXEC_IMAGE && !(ima_appraise & IMA_APPRAISE_ENFORCE) + && security_locked_down(LOCKDOWN_KEXEC)) + return false; + func = read_idmap[id] ?: FILE_CHECK; rcu_read_lock(); diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index 03acc823838a..00f5227c8459 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -203,8 +203,8 @@ #define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */ #define X86_FEATURE_XCOMPACTED ( 7*32+10) /* "" Use compacted XSTATE (XSAVES or XSAVEC) */ #define X86_FEATURE_PTI ( 7*32+11) /* Kernel Page Table Isolation enabled */ -#define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ -#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCE for Spectre variant 2 */ +#define X86_FEATURE_KERNEL_IBRS ( 7*32+12) /* "" Set/clear IBRS on kernel entry/exit */ +#define X86_FEATURE_RSB_VMEXIT ( 7*32+13) /* "" Fill RSB on VM-Exit */ #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */ #define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */ #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */ @@ -296,6 +296,12 @@ #define X86_FEATURE_PER_THREAD_MBA (11*32+ 7) /* "" Per-thread Memory Bandwidth Allocation */ #define X86_FEATURE_SGX1 (11*32+ 8) /* "" Basic SGX */ #define X86_FEATURE_SGX2 (11*32+ 9) /* "" SGX Enclave Dynamic Memory Management (EDMM) */ +#define X86_FEATURE_ENTRY_IBPB (11*32+10) /* "" Issue an IBPB on kernel entry */ +#define X86_FEATURE_RRSBA_CTRL (11*32+11) /* "" RET prediction control */ +#define X86_FEATURE_RETPOLINE (11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ +#define X86_FEATURE_RETPOLINE_LFENCE (11*32+13) /* "" Use LFENCE for Spectre variant 2 */ +#define X86_FEATURE_RETHUNK (11*32+14) /* "" Use REturn THUNK */ +#define X86_FEATURE_UNRET (11*32+15) /* "" AMD BTB untrain return */ /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX_VNNI (12*32+ 4) /* AVX VNNI instructions */ @@ -316,6 +322,7 @@ #define X86_FEATURE_VIRT_SSBD (13*32+25) /* Virtualized Speculative Store Bypass Disable */ #define X86_FEATURE_AMD_SSB_NO (13*32+26) /* "" Speculative Store Bypass is fixed in hardware. */ #define X86_FEATURE_CPPC (13*32+27) /* Collaborative Processor Performance Control */ +#define X86_FEATURE_BTC_NO (13*32+29) /* "" Not vulnerable to Branch Type Confusion */ #define X86_FEATURE_BRS (13*32+31) /* Branch Sampling available */ /* Thermal and Power Management Leaf, CPUID level 0x00000006 (EAX), word 14 */ @@ -447,5 +454,6 @@ #define X86_BUG_ITLB_MULTIHIT X86_BUG(23) /* CPU may incur MCE during certain page attribute changes */ #define X86_BUG_SRBDS X86_BUG(24) /* CPU may leak RNG bits if not mitigated */ #define X86_BUG_MMIO_STALE_DATA X86_BUG(25) /* CPU is affected by Processor MMIO Stale Data vulnerabilities */ +#define X86_BUG_RETBLEED X86_BUG(26) /* CPU is affected by RETBleed */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/tools/arch/x86/include/asm/disabled-features.h b/tools/arch/x86/include/asm/disabled-features.h index 36369e76cc63..33d2cd04d254 100644 --- a/tools/arch/x86/include/asm/disabled-features.h +++ b/tools/arch/x86/include/asm/disabled-features.h @@ -50,6 +50,25 @@ # define DISABLE_PTI (1 << (X86_FEATURE_PTI & 31)) #endif +#ifdef CONFIG_RETPOLINE +# define DISABLE_RETPOLINE 0 +#else +# define DISABLE_RETPOLINE ((1 << (X86_FEATURE_RETPOLINE & 31)) | \ + (1 << (X86_FEATURE_RETPOLINE_LFENCE & 31))) +#endif + +#ifdef CONFIG_RETHUNK +# define DISABLE_RETHUNK 0 +#else +# define DISABLE_RETHUNK (1 << (X86_FEATURE_RETHUNK & 31)) +#endif + +#ifdef CONFIG_CPU_UNRET_ENTRY +# define DISABLE_UNRET 0 +#else +# define DISABLE_UNRET (1 << (X86_FEATURE_UNRET & 31)) +#endif + #ifdef CONFIG_INTEL_IOMMU_SVM # define DISABLE_ENQCMD 0 #else @@ -82,7 +101,7 @@ #define DISABLED_MASK8 (DISABLE_TDX_GUEST) #define DISABLED_MASK9 (DISABLE_SGX) #define DISABLED_MASK10 0 -#define DISABLED_MASK11 0 +#define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET) #define DISABLED_MASK12 0 #define DISABLED_MASK13 0 #define DISABLED_MASK14 0 diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h index 2eab6a3a8a8c..cc615be27a54 100644 --- a/tools/arch/x86/include/asm/msr-index.h +++ b/tools/arch/x86/include/asm/msr-index.h @@ -95,6 +95,7 @@ #define MSR_IA32_ARCH_CAPABILITIES 0x0000010a #define ARCH_CAP_RDCL_NO BIT(0) /* Not susceptible to Meltdown */ #define ARCH_CAP_IBRS_ALL BIT(1) /* Enhanced IBRS support */ +#define ARCH_CAP_RSBA BIT(2) /* RET may use alternative branch predictors */ #define ARCH_CAP_SKIP_VMENTRY_L1DFLUSH BIT(3) /* Skip L1D flush on vmentry */ #define ARCH_CAP_SSB_NO BIT(4) /* * Not susceptible to Speculative Store Bypass @@ -576,6 +577,9 @@ /* Fam 17h MSRs */ #define MSR_F17H_IRPERF 0xc00000e9 +#define MSR_ZEN2_SPECTRAL_CHICKEN 0xc00110e3 +#define MSR_ZEN2_SPECTRAL_CHICKEN_BIT BIT_ULL(1) + /* Fam 16h MSRs */ #define MSR_F16H_L2I_PERF_CTL 0xc0010230 #define MSR_F16H_L2I_PERF_CTR 0xc0010231 diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c index 5d26f3c6f918..80cd7843c677 100644 --- a/tools/bpf/resolve_btfids/main.c +++ b/tools/bpf/resolve_btfids/main.c @@ -45,6 +45,19 @@ * .zero 4 * __BTF_ID__func__vfs_fallocate__4: * .zero 4 + * + * set8 - store symbol size into first 4 bytes and sort following + * ID list + * + * __BTF_ID__set8__list: + * .zero 8 + * list: + * __BTF_ID__func__vfs_getattr__3: + * .zero 4 + * .word (1 << 0) | (1 << 2) + * __BTF_ID__func__vfs_fallocate__5: + * .zero 4 + * .word (1 << 3) | (1 << 1) | (1 << 2) */ #define _GNU_SOURCE @@ -72,6 +85,7 @@ #define BTF_TYPEDEF "typedef" #define BTF_FUNC "func" #define BTF_SET "set" +#define BTF_SET8 "set8" #define ADDR_CNT 100 @@ -84,6 +98,7 @@ struct btf_id { }; int addr_cnt; bool is_set; + bool is_set8; Elf64_Addr addr[ADDR_CNT]; }; @@ -231,14 +246,14 @@ static char *get_id(const char *prefix_end) return id; } -static struct btf_id *add_set(struct object *obj, char *name) +static struct btf_id *add_set(struct object *obj, char *name, bool is_set8) { /* * __BTF_ID__set__name * name = ^ * id = ^ */ - char *id = name + sizeof(BTF_SET "__") - 1; + char *id = name + (is_set8 ? sizeof(BTF_SET8 "__") : sizeof(BTF_SET "__")) - 1; int len = strlen(name); if (id >= name + len) { @@ -444,9 +459,21 @@ static int symbols_collect(struct object *obj) } else if (!strncmp(prefix, BTF_FUNC, sizeof(BTF_FUNC) - 1)) { obj->nr_funcs++; id = add_symbol(&obj->funcs, prefix, sizeof(BTF_FUNC) - 1); + /* set8 */ + } else if (!strncmp(prefix, BTF_SET8, sizeof(BTF_SET8) - 1)) { + id = add_set(obj, prefix, true); + /* + * SET8 objects store list's count, which is encoded + * in symbol's size, together with 'cnt' field hence + * that - 1. + */ + if (id) { + id->cnt = sym.st_size / sizeof(uint64_t) - 1; + id->is_set8 = true; + } /* set */ } else if (!strncmp(prefix, BTF_SET, sizeof(BTF_SET) - 1)) { - id = add_set(obj, prefix); + id = add_set(obj, prefix, false); /* * SET objects store list's count, which is encoded * in symbol's size, together with 'cnt' field hence @@ -571,7 +598,8 @@ static int id_patch(struct object *obj, struct btf_id *id) int *ptr = data->d_buf; int i; - if (!id->id && !id->is_set) + /* For set, set8, id->id may be 0 */ + if (!id->id && !id->is_set && !id->is_set8) pr_err("WARN: resolve_btfids: unresolved symbol %s\n", id->name); for (i = 0; i < id->addr_cnt; i++) { @@ -643,13 +671,13 @@ static int sets_patch(struct object *obj) } idx = idx / sizeof(int); - base = &ptr[idx] + 1; + base = &ptr[idx] + (id->is_set8 ? 2 : 1); cnt = ptr[idx]; pr_debug("sorting addr %5lu: cnt %6d [%s]\n", (idx + 1) * sizeof(int), cnt, id->name); - qsort(base, cnt, sizeof(int), cmp_id); + qsort(base, cnt, id->is_set8 ? sizeof(uint64_t) : sizeof(int), cmp_id); next = rb_next(next); } diff --git a/tools/bpf/runqslower/Makefile b/tools/bpf/runqslower/Makefile index da6de16a3dfb..8b3d87b82b7a 100644 --- a/tools/bpf/runqslower/Makefile +++ b/tools/bpf/runqslower/Makefile @@ -4,7 +4,7 @@ include ../../scripts/Makefile.include OUTPUT ?= $(abspath .output)/ BPFTOOL_OUTPUT := $(OUTPUT)bpftool/ -DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bpftool +DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bootstrap/bpftool BPFTOOL ?= $(DEFAULT_BPFTOOL) LIBBPF_SRC := $(abspath ../../lib/bpf) BPFOBJ_OUTPUT := $(OUTPUT)libbpf/ @@ -86,6 +86,5 @@ $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(BPFOBJ_OU $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(BPFOBJ_OUTPUT) \ DESTDIR=$(BPFOBJ_OUTPUT) prefix= $(abspath $@) install_headers -$(DEFAULT_BPFTOOL): $(BPFOBJ) | $(BPFTOOL_OUTPUT) - $(Q)$(MAKE) $(submake_extras) -C ../bpftool OUTPUT=$(BPFTOOL_OUTPUT) \ - ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) +$(DEFAULT_BPFTOOL): | $(BPFTOOL_OUTPUT) + $(Q)$(MAKE) $(submake_extras) -C ../bpftool OUTPUT=$(BPFTOOL_OUTPUT) bootstrap diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 3dd13fe738b9..59a217ca2dfd 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -2361,7 +2361,8 @@ union bpf_attr { * Pull in non-linear data in case the *skb* is non-linear and not * all of *len* are part of the linear section. Make *len* bytes * from *skb* readable and writable. If a zero value is passed for - * *len*, then the whole length of the *skb* is pulled. + * *len*, then all bytes in the linear part of *skb* will be made + * readable and writable. * * This helper is only needed for reading and writing with direct * packet access. diff --git a/tools/include/uapi/linux/kvm.h b/tools/include/uapi/linux/kvm.h index 5088bd9f1922..811897dadcae 100644 --- a/tools/include/uapi/linux/kvm.h +++ b/tools/include/uapi/linux/kvm.h @@ -2083,6 +2083,7 @@ struct kvm_stats_header { #define KVM_STATS_UNIT_BYTES (0x1 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_SECONDS (0x2 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_CYCLES (0x3 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_BOOLEAN (0x4 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_MAX KVM_STATS_UNIT_CYCLES #define KVM_STATS_BASE_SHIFT 8 diff --git a/tools/lib/bpf/bpf_tracing.h b/tools/lib/bpf/bpf_tracing.h index 11f9096407fc..f4d3e1e2abe2 100644 --- a/tools/lib/bpf/bpf_tracing.h +++ b/tools/lib/bpf/bpf_tracing.h @@ -2,6 +2,8 @@ #ifndef __BPF_TRACING_H__ #define __BPF_TRACING_H__ +#include <bpf/bpf_helpers.h> + /* Scan the ARCH passed in from ARCH env variable (see Makefile) */ #if defined(__TARGET_ARCH_x86) #define bpf_target_x86 @@ -140,7 +142,7 @@ struct pt_regs___s390 { #define __PT_RC_REG gprs[2] #define __PT_SP_REG gprs[15] #define __PT_IP_REG psw.addr -#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma("GCC error \"use PT_REGS_PARM1_CORE_SYSCALL() instead\""); 0l; }) +#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) #define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ((const struct pt_regs___s390 *)(x), orig_gpr2) #elif defined(bpf_target_arm) @@ -174,7 +176,7 @@ struct pt_regs___arm64 { #define __PT_RC_REG regs[0] #define __PT_SP_REG sp #define __PT_IP_REG pc -#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma("GCC error \"use PT_REGS_PARM1_CORE_SYSCALL() instead\""); 0l; }) +#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) #define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ((const struct pt_regs___arm64 *)(x), orig_x0) #elif defined(bpf_target_mips) @@ -493,39 +495,62 @@ typeof(name(0)) name(struct pt_regs *ctx) \ } \ static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) +/* If kernel has CONFIG_ARCH_HAS_SYSCALL_WRAPPER, read pt_regs directly */ #define ___bpf_syscall_args0() ctx -#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs) -#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs) -#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs) -#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs) -#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs) +#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_SYSCALL(regs) +#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_SYSCALL(regs) +#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_SYSCALL(regs) +#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_SYSCALL(regs) +#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_SYSCALL(regs) #define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args) +/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */ +#define ___bpf_syswrap_args0() ctx +#define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args3(x, args...) ___bpf_syswrap_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args4(x, args...) ___bpf_syswrap_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args5(x, args...) ___bpf_syswrap_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args(args...) ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args) + /* - * BPF_KPROBE_SYSCALL is a variant of BPF_KPROBE, which is intended for + * BPF_KSYSCALL is a variant of BPF_KPROBE, which is intended for * tracing syscall functions, like __x64_sys_close. It hides the underlying * platform-specific low-level way of getting syscall input arguments from * struct pt_regs, and provides a familiar typed and named function arguments * syntax and semantics of accessing syscall input parameters. * - * Original struct pt_regs* context is preserved as 'ctx' argument. This might + * Original struct pt_regs * context is preserved as 'ctx' argument. This might * be necessary when using BPF helpers like bpf_perf_event_output(). * - * This macro relies on BPF CO-RE support. + * At the moment BPF_KSYSCALL does not handle all the calling convention + * quirks for mmap(), clone() and compat syscalls transparrently. This may or + * may not change in the future. User needs to take extra measures to handle + * such quirks explicitly, if necessary. + * + * This macro relies on BPF CO-RE support and virtual __kconfig externs. */ -#define BPF_KPROBE_SYSCALL(name, args...) \ +#define BPF_KSYSCALL(name, args...) \ name(struct pt_regs *ctx); \ +extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \ static __attribute__((always_inline)) typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args); \ typeof(name(0)) name(struct pt_regs *ctx) \ { \ - struct pt_regs *regs = PT_REGS_SYSCALL_REGS(ctx); \ + struct pt_regs *regs = LINUX_HAS_SYSCALL_WRAPPER \ + ? (struct pt_regs *)PT_REGS_PARM1(ctx) \ + : ctx; \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - return ____##name(___bpf_syscall_args(args)); \ + if (LINUX_HAS_SYSCALL_WRAPPER) \ + return ____##name(___bpf_syswrap_args(args)); \ + else \ + return ____##name(___bpf_syscall_args(args)); \ _Pragma("GCC diagnostic pop") \ } \ static __attribute__((always_inline)) typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args) +#define BPF_KPROBE_SYSCALL BPF_KSYSCALL + #endif diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 400e84fd0578..627edb5bb6de 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -2045,7 +2045,7 @@ static int btf_dump_get_enum_value(struct btf_dump *d, *value = *(__s64 *)data; return 0; case 4: - *value = is_signed ? *(__s32 *)data : *(__u32 *)data; + *value = is_signed ? (__s64)*(__s32 *)data : *(__u32 *)data; return 0; case 2: *value = is_signed ? *(__s16 *)data : *(__u16 *)data; diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index 927745b08014..23f5c46708f8 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -533,7 +533,7 @@ void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, gen->attach_kind = kind; ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s", prefix, attach_name); - if (ret == sizeof(gen->attach_target)) + if (ret >= sizeof(gen->attach_target)) gen->error = -ENOSPC; } diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index cb49408eb298..b01fe01b0761 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -1694,7 +1694,7 @@ static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, switch (ext->kcfg.type) { case KCFG_BOOL: if (value == 'm') { - pr_warn("extern (kcfg) %s=%c should be tristate or char\n", + pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n", ext->name, value); return -EINVAL; } @@ -1715,7 +1715,7 @@ static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, case KCFG_INT: case KCFG_CHAR_ARR: default: - pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n", + pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n", ext->name, value); return -EINVAL; } @@ -1729,7 +1729,8 @@ static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, size_t len; if (ext->kcfg.type != KCFG_CHAR_ARR) { - pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value); + pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n", + ext->name, value); return -EINVAL; } @@ -1743,7 +1744,7 @@ static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, /* strip quotes */ len -= 2; if (len >= ext->kcfg.sz) { - pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n", + pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n", ext->name, value, len, ext->kcfg.sz - 1); len = ext->kcfg.sz - 1; } @@ -1800,13 +1801,20 @@ static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, __u64 value) { - if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { - pr_warn("extern (kcfg) %s=%llu should be integer\n", + if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR && + ext->kcfg.type != KCFG_BOOL) { + pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n", ext->name, (unsigned long long)value); return -EINVAL; } + if (ext->kcfg.type == KCFG_BOOL && value > 1) { + pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n", + ext->name, (unsigned long long)value); + return -EINVAL; + + } if (!is_kcfg_value_in_range(ext, value)) { - pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n", + pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n", ext->name, (unsigned long long)value, ext->kcfg.sz); return -ERANGE; } @@ -1870,16 +1878,19 @@ static int bpf_object__process_kconfig_line(struct bpf_object *obj, /* assume integer */ err = parse_u64(value, &num); if (err) { - pr_warn("extern (kcfg) %s=%s should be integer\n", - ext->name, value); + pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value); return err; } + if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { + pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value); + return -EINVAL; + } err = set_kcfg_value_num(ext, ext_val, num); break; } if (err) return err; - pr_debug("extern (kcfg) %s=%s\n", ext->name, value); + pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value); return 0; } @@ -2320,6 +2331,37 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, return 0; } +static size_t adjust_ringbuf_sz(size_t sz) +{ + __u32 page_sz = sysconf(_SC_PAGE_SIZE); + __u32 mul; + + /* if user forgot to set any size, make sure they see error */ + if (sz == 0) + return 0; + /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be + * a power-of-2 multiple of kernel's page size. If user diligently + * satisified these conditions, pass the size through. + */ + if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) + return sz; + + /* Otherwise find closest (page_sz * power_of_2) product bigger than + * user-set size to satisfy both user size request and kernel + * requirements and substitute correct max_entries for map creation. + */ + for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) { + if (mul * page_sz > sz) + return mul * page_sz; + } + + /* if it's impossible to satisfy the conditions (i.e., user size is + * very close to UINT_MAX but is not a power-of-2 multiple of + * page_size) then just return original size and let kernel reject it + */ + return sz; +} + static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) { map->def.type = def->map_type; @@ -2333,6 +2375,10 @@ static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def map->btf_key_type_id = def->key_type_id; map->btf_value_type_id = def->value_type_id; + /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ + if (map->def.type == BPF_MAP_TYPE_RINGBUF) + map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); + if (def->parts & MAP_DEF_MAP_TYPE) pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); @@ -3687,7 +3733,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj) ext->kcfg.type = find_kcfg_type(obj->btf, t->type, &ext->kcfg.is_signed); if (ext->kcfg.type == KCFG_UNKNOWN) { - pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name); + pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name); return -ENOTSUP; } } else if (strcmp(sec_name, KSYMS_SEC) == 0) { @@ -4232,7 +4278,7 @@ int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) int bpf_map__reuse_fd(struct bpf_map *map, int fd) { struct bpf_map_info info = {}; - __u32 len = sizeof(info); + __u32 len = sizeof(info), name_len; int new_fd, err; char *new_name; @@ -4242,7 +4288,12 @@ int bpf_map__reuse_fd(struct bpf_map *map, int fd) if (err) return libbpf_err(err); - new_name = strdup(info.name); + name_len = strlen(info.name); + if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0) + new_name = strdup(map->name); + else + new_name = strdup(info.name); + if (!new_name) return libbpf_err(-errno); @@ -4301,9 +4352,15 @@ struct bpf_map *bpf_map__inner_map(struct bpf_map *map) int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) { - if (map->fd >= 0) + if (map->obj->loaded) return libbpf_err(-EBUSY); + map->def.max_entries = max_entries; + + /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ + if (map->def.type == BPF_MAP_TYPE_RINGBUF) + map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); + return 0; } @@ -4654,6 +4711,8 @@ static int probe_kern_btf_enum64(void) strs, sizeof(strs))); } +static int probe_kern_syscall_wrapper(void); + enum kern_feature_result { FEAT_UNKNOWN = 0, FEAT_SUPPORTED = 1, @@ -4722,6 +4781,9 @@ static struct kern_feature_desc { [FEAT_BTF_ENUM64] = { "BTF_KIND_ENUM64 support", probe_kern_btf_enum64, }, + [FEAT_SYSCALL_WRAPPER] = { + "Kernel using syscall wrapper", probe_kern_syscall_wrapper, + }, }; bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) @@ -4854,37 +4916,6 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) static void bpf_map__destroy(struct bpf_map *map); -static size_t adjust_ringbuf_sz(size_t sz) -{ - __u32 page_sz = sysconf(_SC_PAGE_SIZE); - __u32 mul; - - /* if user forgot to set any size, make sure they see error */ - if (sz == 0) - return 0; - /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be - * a power-of-2 multiple of kernel's page size. If user diligently - * satisified these conditions, pass the size through. - */ - if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) - return sz; - - /* Otherwise find closest (page_sz * power_of_2) product bigger than - * user-set size to satisfy both user size request and kernel - * requirements and substitute correct max_entries for map creation. - */ - for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) { - if (mul * page_sz > sz) - return mul * page_sz; - } - - /* if it's impossible to satisfy the conditions (i.e., user size is - * very close to UINT_MAX but is not a power-of-2 multiple of - * page_size) then just return original size and let kernel reject it - */ - return sz; -} - static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner) { LIBBPF_OPTS(bpf_map_create_opts, create_attr); @@ -4923,9 +4954,6 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b } switch (def->type) { - case BPF_MAP_TYPE_RINGBUF: - map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); - /* fallthrough */ case BPF_MAP_TYPE_PERF_EVENT_ARRAY: case BPF_MAP_TYPE_CGROUP_ARRAY: case BPF_MAP_TYPE_STACK_TRACE: @@ -7282,14 +7310,14 @@ static int kallsyms_cb(unsigned long long sym_addr, char sym_type, return 0; if (ext->is_set && ext->ksym.addr != sym_addr) { - pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n", + pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n", sym_name, ext->ksym.addr, sym_addr); return -EINVAL; } if (!ext->is_set) { ext->is_set = true; ext->ksym.addr = sym_addr; - pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr); + pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr); } return 0; } @@ -7493,28 +7521,52 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, for (i = 0; i < obj->nr_extern; i++) { ext = &obj->externs[i]; - if (ext->type == EXT_KCFG && - strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { - void *ext_val = kcfg_data + ext->kcfg.data_off; - __u32 kver = get_kernel_version(); + if (ext->type == EXT_KSYM) { + if (ext->ksym.type_id) + need_vmlinux_btf = true; + else + need_kallsyms = true; + continue; + } else if (ext->type == EXT_KCFG) { + void *ext_ptr = kcfg_data + ext->kcfg.data_off; + __u64 value = 0; + + /* Kconfig externs need actual /proc/config.gz */ + if (str_has_pfx(ext->name, "CONFIG_")) { + need_config = true; + continue; + } - if (!kver) { - pr_warn("failed to get kernel version\n"); + /* Virtual kcfg externs are customly handled by libbpf */ + if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { + value = get_kernel_version(); + if (!value) { + pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name); + return -EINVAL; + } + } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) { + value = kernel_supports(obj, FEAT_BPF_COOKIE); + } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) { + value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER); + } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) { + /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed + * __kconfig externs, where LINUX_ ones are virtual and filled out + * customly by libbpf (their values don't come from Kconfig). + * If LINUX_xxx variable is not recognized by libbpf, but is marked + * __weak, it defaults to zero value, just like for CONFIG_xxx + * externs. + */ + pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name); return -EINVAL; } - err = set_kcfg_value_num(ext, ext_val, kver); + + err = set_kcfg_value_num(ext, ext_ptr, value); if (err) return err; - pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver); - } else if (ext->type == EXT_KCFG && str_has_pfx(ext->name, "CONFIG_")) { - need_config = true; - } else if (ext->type == EXT_KSYM) { - if (ext->ksym.type_id) - need_vmlinux_btf = true; - else - need_kallsyms = true; + pr_debug("extern (kcfg) '%s': set to 0x%llx\n", + ext->name, (long long)value); } else { - pr_warn("unrecognized extern '%s'\n", ext->name); + pr_warn("extern '%s': unrecognized extern kind\n", ext->name); return -EINVAL; } } @@ -7550,10 +7602,10 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, ext = &obj->externs[i]; if (!ext->is_set && !ext->is_weak) { - pr_warn("extern %s (strong) not resolved\n", ext->name); + pr_warn("extern '%s' (strong): not resolved\n", ext->name); return -ESRCH; } else if (!ext->is_set) { - pr_debug("extern %s (weak) not resolved, defaulting to zero\n", + pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n", ext->name); } } @@ -8381,6 +8433,7 @@ int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); +static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link); static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link); static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); @@ -8401,6 +8454,8 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), + SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), + SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt), SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), @@ -9757,7 +9812,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, { struct perf_event_attr attr = {}; char errmsg[STRERR_BUFSIZE]; - int type, pfd, err; + int type, pfd; if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) return -EINVAL; @@ -9793,14 +9848,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, pid < 0 ? -1 : pid /* pid */, pid == -1 ? 0 : -1 /* cpu */, -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); - if (pfd < 0) { - err = -errno; - pr_warn("%s perf_event_open() failed: %s\n", - uprobe ? "uprobe" : "kprobe", - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - return err; - } - return pfd; + return pfd >= 0 ? pfd : -errno; } static int append_to_file(const char *file, const char *fmt, ...) @@ -9823,6 +9871,34 @@ static int append_to_file(const char *file, const char *fmt, ...) return err; } +#define DEBUGFS "/sys/kernel/debug/tracing" +#define TRACEFS "/sys/kernel/tracing" + +static bool use_debugfs(void) +{ + static int has_debugfs = -1; + + if (has_debugfs < 0) + has_debugfs = access(DEBUGFS, F_OK) == 0; + + return has_debugfs == 1; +} + +static const char *tracefs_path(void) +{ + return use_debugfs() ? DEBUGFS : TRACEFS; +} + +static const char *tracefs_kprobe_events(void) +{ + return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events"; +} + +static const char *tracefs_uprobe_events(void) +{ + return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events"; +} + static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, const char *kfunc_name, size_t offset) { @@ -9835,9 +9911,7 @@ static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, static int add_kprobe_event_legacy(const char *probe_name, bool retprobe, const char *kfunc_name, size_t offset) { - const char *file = "/sys/kernel/debug/tracing/kprobe_events"; - - return append_to_file(file, "%c:%s/%s %s+0x%zx", + return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx", retprobe ? 'r' : 'p', retprobe ? "kretprobes" : "kprobes", probe_name, kfunc_name, offset); @@ -9845,18 +9919,16 @@ static int add_kprobe_event_legacy(const char *probe_name, bool retprobe, static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe) { - const char *file = "/sys/kernel/debug/tracing/kprobe_events"; - - return append_to_file(file, "-:%s/%s", retprobe ? "kretprobes" : "kprobes", probe_name); + return append_to_file(tracefs_kprobe_events(), "-:%s/%s", + retprobe ? "kretprobes" : "kprobes", probe_name); } static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe) { char file[256]; - snprintf(file, sizeof(file), - "/sys/kernel/debug/tracing/events/%s/%s/id", - retprobe ? "kretprobes" : "kprobes", probe_name); + snprintf(file, sizeof(file), "%s/events/%s/%s/id", + tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name); return parse_uint_from_file(file, "%d\n"); } @@ -9905,6 +9977,60 @@ err_clean_legacy: return err; } +static const char *arch_specific_syscall_pfx(void) +{ +#if defined(__x86_64__) + return "x64"; +#elif defined(__i386__) + return "ia32"; +#elif defined(__s390x__) + return "s390x"; +#elif defined(__s390__) + return "s390"; +#elif defined(__arm__) + return "arm"; +#elif defined(__aarch64__) + return "arm64"; +#elif defined(__mips__) + return "mips"; +#elif defined(__riscv) + return "riscv"; +#else + return NULL; +#endif +} + +static int probe_kern_syscall_wrapper(void) +{ + char syscall_name[64]; + const char *ksys_pfx; + + ksys_pfx = arch_specific_syscall_pfx(); + if (!ksys_pfx) + return 0; + + snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx); + + if (determine_kprobe_perf_type() >= 0) { + int pfd; + + pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0); + if (pfd >= 0) + close(pfd); + + return pfd >= 0 ? 1 : 0; + } else { /* legacy mode */ + char probe_name[128]; + + gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0); + if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0) + return 0; + + (void)remove_kprobe_event_legacy(probe_name, false); + return 1; + } +} + struct bpf_link * bpf_program__attach_kprobe_opts(const struct bpf_program *prog, const char *func_name, @@ -9990,6 +10116,29 @@ struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog, return bpf_program__attach_kprobe_opts(prog, func_name, &opts); } +struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog, + const char *syscall_name, + const struct bpf_ksyscall_opts *opts) +{ + LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); + char func_name[128]; + + if (!OPTS_VALID(opts, bpf_ksyscall_opts)) + return libbpf_err_ptr(-EINVAL); + + if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { + snprintf(func_name, sizeof(func_name), "__%s_sys_%s", + arch_specific_syscall_pfx(), syscall_name); + } else { + snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name); + } + + kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false); + kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); + + return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts); +} + /* Adapted from perf/util/string.c */ static bool glob_match(const char *str, const char *pat) { @@ -10160,6 +10309,27 @@ static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf return libbpf_get_error(*link); } +static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link) +{ + LIBBPF_OPTS(bpf_ksyscall_opts, opts); + const char *syscall_name; + + *link = NULL; + + /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */ + if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) + return 0; + + opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); + if (opts.retprobe) + syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; + else + syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; + + *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts); + return *link ? 0 : -errno; +} + static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) { LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); @@ -10208,9 +10378,7 @@ static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz, static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe, const char *binary_path, size_t offset) { - const char *file = "/sys/kernel/debug/tracing/uprobe_events"; - - return append_to_file(file, "%c:%s/%s %s:0x%zx", + return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx", retprobe ? 'r' : 'p', retprobe ? "uretprobes" : "uprobes", probe_name, binary_path, offset); @@ -10218,18 +10386,16 @@ static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe, static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe) { - const char *file = "/sys/kernel/debug/tracing/uprobe_events"; - - return append_to_file(file, "-:%s/%s", retprobe ? "uretprobes" : "uprobes", probe_name); + return append_to_file(tracefs_uprobe_events(), "-:%s/%s", + retprobe ? "uretprobes" : "uprobes", probe_name); } static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe) { char file[512]; - snprintf(file, sizeof(file), - "/sys/kernel/debug/tracing/events/%s/%s/id", - retprobe ? "uretprobes" : "uprobes", probe_name); + snprintf(file, sizeof(file), "%s/events/%s/%s/id", + tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name); return parse_uint_from_file(file, "%d\n"); } @@ -10545,7 +10711,10 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0); pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); - if (binary_path && !strchr(binary_path, '/')) { + if (!binary_path) + return libbpf_err_ptr(-EINVAL); + + if (!strchr(binary_path, '/')) { err = resolve_full_path(binary_path, full_binary_path, sizeof(full_binary_path)); if (err) { @@ -10559,11 +10728,6 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, if (func_name) { long sym_off; - if (!binary_path) { - pr_warn("prog '%s': name-based attach requires binary_path\n", - prog->name); - return libbpf_err_ptr(-EINVAL); - } sym_off = elf_find_func_offset(binary_path, func_name); if (sym_off < 0) return libbpf_err_ptr(sym_off); @@ -10711,6 +10875,9 @@ struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, return libbpf_err_ptr(-EINVAL); } + if (!binary_path) + return libbpf_err_ptr(-EINVAL); + if (!strchr(binary_path, '/')) { err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path)); if (err) { @@ -10776,9 +10943,8 @@ static int determine_tracepoint_id(const char *tp_category, char file[PATH_MAX]; int ret; - ret = snprintf(file, sizeof(file), - "/sys/kernel/debug/tracing/events/%s/%s/id", - tp_category, tp_name); + ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id", + tracefs_path(), tp_category, tp_name); if (ret < 0) return -errno; if (ret >= sizeof(file)) { @@ -11728,6 +11894,22 @@ int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) return cpu_buf->fd; } +int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size) +{ + struct perf_cpu_buf *cpu_buf; + + if (buf_idx >= pb->cpu_cnt) + return libbpf_err(-EINVAL); + + cpu_buf = pb->cpu_bufs[buf_idx]; + if (!cpu_buf) + return libbpf_err(-ENOENT); + + *buf = cpu_buf->base; + *buf_size = pb->mmap_size; + return 0; +} + /* * Consume data from perf ring buffer corresponding to slot *buf_idx* in * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index e4d5353f757b..61493c4cddac 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -457,6 +457,52 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, const char *pattern, const struct bpf_kprobe_multi_opts *opts); +struct bpf_ksyscall_opts { + /* size of this struct, for forward/backward compatiblity */ + size_t sz; + /* custom user-provided value fetchable through bpf_get_attach_cookie() */ + __u64 bpf_cookie; + /* attach as return probe? */ + bool retprobe; + size_t :0; +}; +#define bpf_ksyscall_opts__last_field retprobe + +/** + * @brief **bpf_program__attach_ksyscall()** attaches a BPF program + * to kernel syscall handler of a specified syscall. Optionally it's possible + * to request to install retprobe that will be triggered at syscall exit. It's + * also possible to associate BPF cookie (though options). + * + * Libbpf automatically will determine correct full kernel function name, + * which depending on system architecture and kernel version/configuration + * could be of the form __<arch>_sys_<syscall> or __se_sys_<syscall>, and will + * attach specified program using kprobe/kretprobe mechanism. + * + * **bpf_program__attach_ksyscall()** is an API counterpart of declarative + * **SEC("ksyscall/<syscall>")** annotation of BPF programs. + * + * At the moment **SEC("ksyscall")** and **bpf_program__attach_ksyscall()** do + * not handle all the calling convention quirks for mmap(), clone() and compat + * syscalls. It also only attaches to "native" syscall interfaces. If host + * system supports compat syscalls or defines 32-bit syscalls in 64-bit + * kernel, such syscall interfaces won't be attached to by libbpf. + * + * These limitations may or may not change in the future. Therefore it is + * recommended to use SEC("kprobe") for these syscalls or if working with + * compat and 32-bit interfaces is required. + * + * @param prog BPF program to attach + * @param syscall_name Symbolic name of the syscall (e.g., "bpf") + * @param opts Additional options (see **struct bpf_ksyscall_opts**) + * @return Reference to the newly created BPF link; or NULL is returned on + * error, error code is stored in errno + */ +LIBBPF_API struct bpf_link * +bpf_program__attach_ksyscall(const struct bpf_program *prog, + const char *syscall_name, + const struct bpf_ksyscall_opts *opts); + struct bpf_uprobe_opts { /* size of this struct, for forward/backward compatiblity */ size_t sz; @@ -1053,6 +1099,22 @@ LIBBPF_API int perf_buffer__consume(struct perf_buffer *pb); LIBBPF_API int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx); LIBBPF_API size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb); LIBBPF_API int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx); +/** + * @brief **perf_buffer__buffer()** returns the per-cpu raw mmap()'ed underlying + * memory region of the ring buffer. + * This ring buffer can be used to implement a custom events consumer. + * The ring buffer starts with the *struct perf_event_mmap_page*, which + * holds the ring buffer managment fields, when accessing the header + * structure it's important to be SMP aware. + * You can refer to *perf_event_read_simple* for a simple example. + * @param pb the perf buffer structure + * @param buf_idx the buffer index to retreive + * @param buf (out) gets the base pointer of the mmap()'ed memory + * @param buf_size (out) gets the size of the mmap()'ed region + * @return 0 on success, negative error code for failure + */ +LIBBPF_API int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, + size_t *buf_size); struct bpf_prog_linfo; struct bpf_prog_info; diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 94b589ecfeaa..0625adb9e888 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -356,10 +356,12 @@ LIBBPF_0.8.0 { LIBBPF_1.0.0 { global: bpf_prog_query_opts; + bpf_program__attach_ksyscall; btf__add_enum64; btf__add_enum64_value; libbpf_bpf_attach_type_str; libbpf_bpf_link_type_str; libbpf_bpf_map_type_str; libbpf_bpf_prog_type_str; + perf_buffer__buffer; }; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 9cd7829cbe41..4135ae0a2bc3 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -108,9 +108,9 @@ static inline bool str_has_sfx(const char *str, const char *sfx) size_t str_len = strlen(str); size_t sfx_len = strlen(sfx); - if (sfx_len <= str_len) - return strcmp(str + str_len - sfx_len, sfx); - return false; + if (sfx_len > str_len) + return false; + return strcmp(str + str_len - sfx_len, sfx) == 0; } /* Symbol versioning is different between static and shared library. @@ -352,6 +352,8 @@ enum kern_feature_id { FEAT_BPF_COOKIE, /* BTF_KIND_ENUM64 support and BTF_KIND_ENUM kflag support */ FEAT_BTF_ENUM64, + /* Kernel uses syscall wrapper (CONFIG_ARCH_HAS_SYSCALL_WRAPPER) */ + FEAT_SYSCALL_WRAPPER, __FEAT_CNT, }; diff --git a/tools/lib/bpf/usdt.bpf.h b/tools/lib/bpf/usdt.bpf.h index 4181fddb3687..4f2adc0bd6ca 100644 --- a/tools/lib/bpf/usdt.bpf.h +++ b/tools/lib/bpf/usdt.bpf.h @@ -6,7 +6,6 @@ #include <linux/errno.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> -#include <bpf/bpf_core_read.h> /* Below types and maps are internal implementation details of libbpf's USDT * support and are subjects to change. Also, bpf_usdt_xxx() API helpers should @@ -30,14 +29,6 @@ #ifndef BPF_USDT_MAX_IP_CNT #define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT) #endif -/* We use BPF CO-RE to detect support for BPF cookie from BPF side. This is - * the only dependency on CO-RE, so if it's undesirable, user can override - * BPF_USDT_HAS_BPF_COOKIE to specify whether to BPF cookie is supported or not. - */ -#ifndef BPF_USDT_HAS_BPF_COOKIE -#define BPF_USDT_HAS_BPF_COOKIE \ - bpf_core_enum_value_exists(enum bpf_func_id___usdt, BPF_FUNC_get_attach_cookie___usdt) -#endif enum __bpf_usdt_arg_type { BPF_USDT_ARG_CONST, @@ -83,15 +74,12 @@ struct { __type(value, __u32); } __bpf_usdt_ip_to_spec_id SEC(".maps") __weak; -/* don't rely on user's BPF code to have latest definition of bpf_func_id */ -enum bpf_func_id___usdt { - BPF_FUNC_get_attach_cookie___usdt = 0xBAD, /* value doesn't matter */ -}; +extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig; static __always_inline int __bpf_usdt_spec_id(struct pt_regs *ctx) { - if (!BPF_USDT_HAS_BPF_COOKIE) { + if (!LINUX_HAS_BPF_COOKIE) { long ip = PT_REGS_IP(ctx); int *spec_id_ptr; diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index 897fc504918b..f075cf37a65e 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -4280,6 +4280,7 @@ static int trace__replay(struct trace *trace) goto out; evsel = evlist__find_tracepoint_by_name(session->evlist, "raw_syscalls:sys_enter"); + trace->syscalls.events.sys_enter = evsel; /* older kernels have syscalls tp versus raw_syscalls */ if (evsel == NULL) evsel = evlist__find_tracepoint_by_name(session->evlist, "syscalls:sys_enter"); @@ -4292,6 +4293,7 @@ static int trace__replay(struct trace *trace) } evsel = evlist__find_tracepoint_by_name(session->evlist, "raw_syscalls:sys_exit"); + trace->syscalls.events.sys_exit = evsel; if (evsel == NULL) evsel = evlist__find_tracepoint_by_name(session->evlist, "syscalls:sys_exit"); if (evsel && diff --git a/tools/perf/tests/perf-time-to-tsc.c b/tools/perf/tests/perf-time-to-tsc.c index 4ad0dfbc8b21..7c7d20fc503a 100644 --- a/tools/perf/tests/perf-time-to-tsc.c +++ b/tools/perf/tests/perf-time-to-tsc.c @@ -20,8 +20,6 @@ #include "tsc.h" #include "mmap.h" #include "tests.h" -#include "pmu.h" -#include "pmu-hybrid.h" /* * Except x86_64/i386 and Arm64, other archs don't support TSC in perf. Just @@ -106,28 +104,21 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su evlist__config(evlist, &opts, NULL); - evsel = evlist__first(evlist); - - evsel->core.attr.comm = 1; - evsel->core.attr.disabled = 1; - evsel->core.attr.enable_on_exec = 0; - - /* - * For hybrid "cycles:u", it creates two events. - * Init the second evsel here. - */ - if (perf_pmu__has_hybrid() && perf_pmu__hybrid_mounted("cpu_atom")) { - evsel = evsel__next(evsel); + /* For hybrid "cycles:u", it creates two events */ + evlist__for_each_entry(evlist, evsel) { evsel->core.attr.comm = 1; evsel->core.attr.disabled = 1; evsel->core.attr.enable_on_exec = 0; } - if (evlist__open(evlist) == -ENOENT) { - err = TEST_SKIP; + ret = evlist__open(evlist); + if (ret < 0) { + if (ret == -ENOENT) + err = TEST_SKIP; + else + pr_debug("evlist__open() failed\n"); goto out_err; } - CHECK__(evlist__open(evlist)); CHECK__(evlist__mmap(evlist, UINT_MAX)); @@ -167,10 +158,12 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su goto next_event; if (strcmp(event->comm.comm, comm1) == 0) { + CHECK_NOT_NULL__(evsel = evlist__event2evsel(evlist, event)); CHECK__(evsel__parse_sample(evsel, event, &sample)); comm1_time = sample.time; } if (strcmp(event->comm.comm, comm2) == 0) { + CHECK_NOT_NULL__(evsel = evlist__event2evsel(evlist, event)); CHECK__(evsel__parse_sample(evsel, event, &sample)); comm2_time = sample.time; } diff --git a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c index e585e1cefc77..792cb15bac40 100644 --- a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +++ b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c @@ -148,13 +148,13 @@ static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = { .write = bpf_testmod_test_write, }; -BTF_SET_START(bpf_testmod_check_kfunc_ids) -BTF_ID(func, bpf_testmod_test_mod_kfunc) -BTF_SET_END(bpf_testmod_check_kfunc_ids) +BTF_SET8_START(bpf_testmod_check_kfunc_ids) +BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc) +BTF_SET8_END(bpf_testmod_check_kfunc_ids) static const struct btf_kfunc_id_set bpf_testmod_kfunc_set = { - .owner = THIS_MODULE, - .check_set = &bpf_testmod_check_kfunc_ids, + .owner = THIS_MODULE, + .set = &bpf_testmod_check_kfunc_ids, }; extern int bpf_fentry_test1(int a); diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c index 7ff5fa93d056..a33874b081b6 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c @@ -27,6 +27,7 @@ #include "bpf_iter_test_kern5.skel.h" #include "bpf_iter_test_kern6.skel.h" #include "bpf_iter_bpf_link.skel.h" +#include "bpf_iter_ksym.skel.h" static int duration; @@ -1120,6 +1121,19 @@ static void test_link_iter(void) bpf_iter_bpf_link__destroy(skel); } +static void test_ksym_iter(void) +{ + struct bpf_iter_ksym *skel; + + skel = bpf_iter_ksym__open_and_load(); + if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load")) + return; + + do_dummy_read(skel->progs.dump_ksym); + + bpf_iter_ksym__destroy(skel); +} + #define CMP_BUFFER_SIZE 1024 static char task_vma_output[CMP_BUFFER_SIZE]; static char proc_maps_output[CMP_BUFFER_SIZE]; @@ -1267,4 +1281,6 @@ void test_bpf_iter(void) test_buf_neg_offset(); if (test__start_subtest("link-iter")) test_link_iter(); + if (test__start_subtest("ksym")) + test_ksym_iter(); } diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index dd30b1e3a67c..7a74a1579076 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -2,13 +2,29 @@ #include <test_progs.h> #include <network_helpers.h> #include "test_bpf_nf.skel.h" +#include "test_bpf_nf_fail.skel.h" + +static char log_buf[1024 * 1024]; + +struct { + const char *prog_name; + const char *err_msg; +} test_bpf_nf_fail_tests[] = { + { "alloc_release", "kernel function bpf_ct_release args#0 expected pointer to STRUCT nf_conn but" }, + { "insert_insert", "kernel function bpf_ct_insert_entry args#0 expected pointer to STRUCT nf_conn___init but" }, + { "lookup_insert", "kernel function bpf_ct_insert_entry args#0 expected pointer to STRUCT nf_conn___init but" }, + { "set_timeout_after_insert", "kernel function bpf_ct_set_timeout args#0 expected pointer to STRUCT nf_conn___init but" }, + { "set_status_after_insert", "kernel function bpf_ct_set_status args#0 expected pointer to STRUCT nf_conn___init but" }, + { "change_timeout_after_alloc", "kernel function bpf_ct_change_timeout args#0 expected pointer to STRUCT nf_conn but" }, + { "change_status_after_alloc", "kernel function bpf_ct_change_status args#0 expected pointer to STRUCT nf_conn but" }, +}; enum { TEST_XDP, TEST_TC_BPF, }; -void test_bpf_nf_ct(int mode) +static void test_bpf_nf_ct(int mode) { struct test_bpf_nf *skel; int prog_fd, err; @@ -39,14 +55,60 @@ void test_bpf_nf_ct(int mode) ASSERT_EQ(skel->bss->test_enonet_netns_id, -ENONET, "Test ENONET for bad but valid netns_id"); ASSERT_EQ(skel->bss->test_enoent_lookup, -ENOENT, "Test ENOENT for failed lookup"); ASSERT_EQ(skel->bss->test_eafnosupport, -EAFNOSUPPORT, "Test EAFNOSUPPORT for invalid len__tuple"); + ASSERT_EQ(skel->data->test_alloc_entry, 0, "Test for alloc new entry"); + ASSERT_EQ(skel->data->test_insert_entry, 0, "Test for insert new entry"); + ASSERT_EQ(skel->data->test_succ_lookup, 0, "Test for successful lookup"); + /* allow some tolerance for test_delta_timeout value to avoid races. */ + ASSERT_GT(skel->bss->test_delta_timeout, 8, "Test for min ct timeout update"); + ASSERT_LE(skel->bss->test_delta_timeout, 10, "Test for max ct timeout update"); + /* expected status is IPS_SEEN_REPLY */ + ASSERT_EQ(skel->bss->test_status, 2, "Test for ct status update "); end: test_bpf_nf__destroy(skel); } +static void test_bpf_nf_ct_fail(const char *prog_name, const char *err_msg) +{ + LIBBPF_OPTS(bpf_object_open_opts, opts, .kernel_log_buf = log_buf, + .kernel_log_size = sizeof(log_buf), + .kernel_log_level = 1); + struct test_bpf_nf_fail *skel; + struct bpf_program *prog; + int ret; + + skel = test_bpf_nf_fail__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "test_bpf_nf_fail__open")) + return; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name")) + goto end; + + bpf_program__set_autoload(prog, true); + + ret = test_bpf_nf_fail__load(skel); + if (!ASSERT_ERR(ret, "test_bpf_nf_fail__load must fail")) + goto end; + + if (!ASSERT_OK_PTR(strstr(log_buf, err_msg), "expected error message")) { + fprintf(stderr, "Expected: %s\n", err_msg); + fprintf(stderr, "Verifier: %s\n", log_buf); + } + +end: + test_bpf_nf_fail__destroy(skel); +} + void test_bpf_nf(void) { + int i; if (test__start_subtest("xdp-ct")) test_bpf_nf_ct(TEST_XDP); if (test__start_subtest("tc-bpf-ct")) test_bpf_nf_ct(TEST_TC_BPF); + for (i = 0; i < ARRAY_SIZE(test_bpf_nf_fail_tests); i++) { + if (test__start_subtest(test_bpf_nf_fail_tests[i].prog_name)) + test_bpf_nf_ct_fail(test_bpf_nf_fail_tests[i].prog_name, + test_bpf_nf_fail_tests[i].err_msg); + } } diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 941b0100bafa..ef6528b8084c 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -5338,7 +5338,7 @@ static void do_test_pprint(int test_num) ret = snprintf(pin_path, sizeof(pin_path), "%s/%s", "/sys/fs/bpf", test->map_name); - if (CHECK(ret == sizeof(pin_path), "pin_path %s/%s is too long", + if (CHECK(ret >= sizeof(pin_path), "pin_path %s/%s is too long", "/sys/fs/bpf", test->map_name)) { err = -1; goto done; diff --git a/tools/testing/selftests/bpf/prog_tests/core_extern.c b/tools/testing/selftests/bpf/prog_tests/core_extern.c index 1931a158510e..63a51e9f3630 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_extern.c +++ b/tools/testing/selftests/bpf/prog_tests/core_extern.c @@ -39,6 +39,7 @@ static struct test_case { "CONFIG_STR=\"abracad\"\n" "CONFIG_MISSING=0", .data = { + .unkn_virt_val = 0, .bpf_syscall = false, .tristate_val = TRI_MODULE, .bool_val = true, @@ -121,7 +122,7 @@ static struct test_case { void test_core_extern(void) { const uint32_t kern_ver = get_kernel_version(); - int err, duration = 0, i, j; + int err, i, j; struct test_core_extern *skel = NULL; uint64_t *got, *exp; int n = sizeof(*skel->data) / sizeof(uint64_t); @@ -136,19 +137,17 @@ void test_core_extern(void) continue; skel = test_core_extern__open_opts(&opts); - if (CHECK(!skel, "skel_open", "skeleton open failed\n")) + if (!ASSERT_OK_PTR(skel, "skel_open")) goto cleanup; err = test_core_extern__load(skel); if (t->fails) { - CHECK(!err, "skel_load", - "shouldn't succeed open/load of skeleton\n"); + ASSERT_ERR(err, "skel_load_should_fail"); goto cleanup; - } else if (CHECK(err, "skel_load", - "failed to open/load skeleton\n")) { + } else if (!ASSERT_OK(err, "skel_load")) { goto cleanup; } err = test_core_extern__attach(skel); - if (CHECK(err, "attach_raw_tp", "failed attach: %d\n", err)) + if (!ASSERT_OK(err, "attach_raw_tp")) goto cleanup; usleep(1); @@ -158,9 +157,7 @@ void test_core_extern(void) got = (uint64_t *)skel->data; exp = (uint64_t *)&t->data; for (j = 0; j < n; j++) { - CHECK(got[j] != exp[j], "check_res", - "result #%d: expected %llx, but got %llx\n", - j, (__u64)exp[j], (__u64)got[j]); + ASSERT_EQ(got[j], exp[j], "result"); } cleanup: test_core_extern__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c index 335917df0614..d457a55ff408 100644 --- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c @@ -364,6 +364,8 @@ static int get_syms(char ***symsp, size_t *cntp) continue; if (!strncmp(name, "rcu_", 4)) continue; + if (!strcmp(name, "bpf_dispatcher_xdp_func")) + continue; if (!strncmp(name, "__ftrace_invalid_address__", sizeof("__ftrace_invalid_address__") - 1)) continue; diff --git a/tools/testing/selftests/bpf/prog_tests/ringbuf_multi.c b/tools/testing/selftests/bpf/prog_tests/ringbuf_multi.c index eb5f7f5aa81a..1455911d9fcb 100644 --- a/tools/testing/selftests/bpf/prog_tests/ringbuf_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/ringbuf_multi.c @@ -50,6 +50,13 @@ void test_ringbuf_multi(void) if (CHECK(!skel, "skel_open", "skeleton open failed\n")) return; + /* validate ringbuf size adjustment logic */ + ASSERT_EQ(bpf_map__max_entries(skel->maps.ringbuf1), page_size, "rb1_size_before"); + ASSERT_OK(bpf_map__set_max_entries(skel->maps.ringbuf1, page_size + 1), "rb1_resize"); + ASSERT_EQ(bpf_map__max_entries(skel->maps.ringbuf1), 2 * page_size, "rb1_size_after"); + ASSERT_OK(bpf_map__set_max_entries(skel->maps.ringbuf1, page_size), "rb1_reset"); + ASSERT_EQ(bpf_map__max_entries(skel->maps.ringbuf1), page_size, "rb1_size_final"); + proto_fd = bpf_map_create(BPF_MAP_TYPE_RINGBUF, NULL, 0, 0, page_size, NULL); if (CHECK(proto_fd < 0, "bpf_map_create", "bpf_map_create failed\n")) goto cleanup; @@ -65,6 +72,10 @@ void test_ringbuf_multi(void) close(proto_fd); proto_fd = -1; + /* make sure we can't resize ringbuf after object load */ + if (!ASSERT_ERR(bpf_map__set_max_entries(skel->maps.ringbuf1, 3 * page_size), "rb1_resize_after_load")) + goto cleanup; + /* only trigger BPF program for current process */ skel->bss->pid = getpid(); diff --git a/tools/testing/selftests/bpf/prog_tests/skeleton.c b/tools/testing/selftests/bpf/prog_tests/skeleton.c index 180afd632f4c..99dac5292b41 100644 --- a/tools/testing/selftests/bpf/prog_tests/skeleton.c +++ b/tools/testing/selftests/bpf/prog_tests/skeleton.c @@ -122,6 +122,8 @@ void test_skeleton(void) ASSERT_EQ(skel->bss->out_mostly_var, 123, "out_mostly_var"); + ASSERT_EQ(bss->huge_arr[ARRAY_SIZE(bss->huge_arr) - 1], 123, "huge_arr"); + elf_bytes = test_skeleton__elf_bytes(&elf_bytes_sz); ASSERT_OK_PTR(elf_bytes, "elf_bytes"); ASSERT_GE(elf_bytes_sz, 0, "elf_bytes_sz"); diff --git a/tools/testing/selftests/bpf/progs/bpf_iter.h b/tools/testing/selftests/bpf/progs/bpf_iter.h index 97ec8bc76ae6..e9846606690d 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter.h +++ b/tools/testing/selftests/bpf/progs/bpf_iter.h @@ -22,6 +22,7 @@ #define BTF_F_NONAME BTF_F_NONAME___not_used #define BTF_F_PTR_RAW BTF_F_PTR_RAW___not_used #define BTF_F_ZERO BTF_F_ZERO___not_used +#define bpf_iter__ksym bpf_iter__ksym___not_used #include "vmlinux.h" #undef bpf_iter_meta #undef bpf_iter__bpf_map @@ -44,6 +45,7 @@ #undef BTF_F_NONAME #undef BTF_F_PTR_RAW #undef BTF_F_ZERO +#undef bpf_iter__ksym struct bpf_iter_meta { struct seq_file *seq; @@ -151,3 +153,8 @@ enum { BTF_F_PTR_RAW = (1ULL << 2), BTF_F_ZERO = (1ULL << 3), }; + +struct bpf_iter__ksym { + struct bpf_iter_meta *meta; + struct kallsym_iter *ksym; +}; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_ksym.c b/tools/testing/selftests/bpf/progs/bpf_iter_ksym.c new file mode 100644 index 000000000000..285c008cbf9c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_iter_ksym.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022, Oracle and/or its affiliates. */ +#include "bpf_iter.h" +#include <bpf/bpf_helpers.h> + +char _license[] SEC("license") = "GPL"; + +unsigned long last_sym_value = 0; + +static inline char tolower(char c) +{ + if (c >= 'A' && c <= 'Z') + c += ('a' - 'A'); + return c; +} + +static inline char toupper(char c) +{ + if (c >= 'a' && c <= 'z') + c -= ('a' - 'A'); + return c; +} + +/* Dump symbols with max size; the latter is calculated by caching symbol N value + * and when iterating on symbol N+1, we can print max size of symbol N via + * address of N+1 - address of N. + */ +SEC("iter/ksym") +int dump_ksym(struct bpf_iter__ksym *ctx) +{ + struct seq_file *seq = ctx->meta->seq; + struct kallsym_iter *iter = ctx->ksym; + __u32 seq_num = ctx->meta->seq_num; + unsigned long value; + char type; + int ret; + + if (!iter) + return 0; + + if (seq_num == 0) { + BPF_SEQ_PRINTF(seq, "ADDR TYPE NAME MODULE_NAME KIND MAX_SIZE\n"); + return 0; + } + if (last_sym_value) + BPF_SEQ_PRINTF(seq, "0x%x\n", iter->value - last_sym_value); + else + BPF_SEQ_PRINTF(seq, "\n"); + + value = iter->show_value ? iter->value : 0; + + last_sym_value = value; + + type = iter->type; + + if (iter->module_name[0]) { + type = iter->exported ? toupper(type) : tolower(type); + BPF_SEQ_PRINTF(seq, "0x%llx %c %s [ %s ] ", + value, type, iter->name, iter->module_name); + } else { + BPF_SEQ_PRINTF(seq, "0x%llx %c %s ", value, type, iter->name); + } + if (!iter->pos_arch_end || iter->pos_arch_end > iter->pos) + BPF_SEQ_PRINTF(seq, "CORE "); + else if (!iter->pos_mod_end || iter->pos_mod_end > iter->pos) + BPF_SEQ_PRINTF(seq, "MOD "); + else if (!iter->pos_ftrace_mod_end || iter->pos_ftrace_mod_end > iter->pos) + BPF_SEQ_PRINTF(seq, "FTRACE_MOD "); + else if (!iter->pos_bpf_end || iter->pos_bpf_end > iter->pos) + BPF_SEQ_PRINTF(seq, "BPF "); + else + BPF_SEQ_PRINTF(seq, "KPROBE "); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/bpf_syscall_macro.c b/tools/testing/selftests/bpf/progs/bpf_syscall_macro.c index 05838ed9b89c..e1e11897e99b 100644 --- a/tools/testing/selftests/bpf/progs/bpf_syscall_macro.c +++ b/tools/testing/selftests/bpf/progs/bpf_syscall_macro.c @@ -64,9 +64,9 @@ int BPF_KPROBE(handle_sys_prctl) return 0; } -SEC("kprobe/" SYS_PREFIX "sys_prctl") -int BPF_KPROBE_SYSCALL(prctl_enter, int option, unsigned long arg2, - unsigned long arg3, unsigned long arg4, unsigned long arg5) +SEC("ksyscall/prctl") +int BPF_KSYSCALL(prctl_enter, int option, unsigned long arg2, + unsigned long arg3, unsigned long arg4, unsigned long arg5) { pid_t pid = bpf_get_current_pid_tgid() >> 32; diff --git a/tools/testing/selftests/bpf/progs/test_attach_probe.c b/tools/testing/selftests/bpf/progs/test_attach_probe.c index f1c88ad368ef..a1e45fec8938 100644 --- a/tools/testing/selftests/bpf/progs/test_attach_probe.c +++ b/tools/testing/selftests/bpf/progs/test_attach_probe.c @@ -1,11 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2017 Facebook -#include <linux/ptrace.h> -#include <linux/bpf.h> +#include "vmlinux.h" #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> -#include <stdbool.h> +#include <bpf/bpf_core_read.h> #include "bpf_misc.h" int kprobe_res = 0; @@ -31,8 +30,8 @@ int handle_kprobe(struct pt_regs *ctx) return 0; } -SEC("kprobe/" SYS_PREFIX "sys_nanosleep") -int BPF_KPROBE(handle_kprobe_auto) +SEC("ksyscall/nanosleep") +int BPF_KSYSCALL(handle_kprobe_auto, struct __kernel_timespec *req, struct __kernel_timespec *rem) { kprobe2_res = 11; return 0; @@ -56,11 +55,11 @@ int handle_kretprobe(struct pt_regs *ctx) return 0; } -SEC("kretprobe/" SYS_PREFIX "sys_nanosleep") -int BPF_KRETPROBE(handle_kretprobe_auto) +SEC("kretsyscall/nanosleep") +int BPF_KRETPROBE(handle_kretprobe_auto, int ret) { kretprobe2_res = 22; - return 0; + return ret; } SEC("uprobe") diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf.c b/tools/testing/selftests/bpf/progs/test_bpf_nf.c index f00a9731930e..196cd8dfe42a 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_nf.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf.c @@ -8,6 +8,8 @@ #define EINVAL 22 #define ENOENT 2 +extern unsigned long CONFIG_HZ __kconfig; + int test_einval_bpf_tuple = 0; int test_einval_reserved = 0; int test_einval_netns_id = 0; @@ -16,6 +18,11 @@ int test_eproto_l4proto = 0; int test_enonet_netns_id = 0; int test_enoent_lookup = 0; int test_eafnosupport = 0; +int test_alloc_entry = -EINVAL; +int test_insert_entry = -EAFNOSUPPORT; +int test_succ_lookup = -ENOENT; +u32 test_delta_timeout = 0; +u32 test_status = 0; struct nf_conn; @@ -26,31 +33,44 @@ struct bpf_ct_opts___local { u8 reserved[3]; } __attribute__((preserve_access_index)); +struct nf_conn *bpf_xdp_ct_alloc(struct xdp_md *, struct bpf_sock_tuple *, u32, + struct bpf_ct_opts___local *, u32) __ksym; struct nf_conn *bpf_xdp_ct_lookup(struct xdp_md *, struct bpf_sock_tuple *, u32, struct bpf_ct_opts___local *, u32) __ksym; +struct nf_conn *bpf_skb_ct_alloc(struct __sk_buff *, struct bpf_sock_tuple *, u32, + struct bpf_ct_opts___local *, u32) __ksym; struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *, struct bpf_sock_tuple *, u32, struct bpf_ct_opts___local *, u32) __ksym; +struct nf_conn *bpf_ct_insert_entry(struct nf_conn *) __ksym; void bpf_ct_release(struct nf_conn *) __ksym; +void bpf_ct_set_timeout(struct nf_conn *, u32) __ksym; +int bpf_ct_change_timeout(struct nf_conn *, u32) __ksym; +int bpf_ct_set_status(struct nf_conn *, u32) __ksym; +int bpf_ct_change_status(struct nf_conn *, u32) __ksym; static __always_inline void -nf_ct_test(struct nf_conn *(*func)(void *, struct bpf_sock_tuple *, u32, - struct bpf_ct_opts___local *, u32), +nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, + struct bpf_ct_opts___local *, u32), + struct nf_conn *(*alloc_fn)(void *, struct bpf_sock_tuple *, u32, + struct bpf_ct_opts___local *, u32), void *ctx) { struct bpf_ct_opts___local opts_def = { .l4proto = IPPROTO_TCP, .netns_id = -1 }; struct bpf_sock_tuple bpf_tuple; struct nf_conn *ct; + int err; __builtin_memset(&bpf_tuple, 0, sizeof(bpf_tuple.ipv4)); - ct = func(ctx, NULL, 0, &opts_def, sizeof(opts_def)); + ct = lookup_fn(ctx, NULL, 0, &opts_def, sizeof(opts_def)); if (ct) bpf_ct_release(ct); else test_einval_bpf_tuple = opts_def.error; opts_def.reserved[0] = 1; - ct = func(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def)); opts_def.reserved[0] = 0; opts_def.l4proto = IPPROTO_TCP; if (ct) @@ -59,21 +79,24 @@ nf_ct_test(struct nf_conn *(*func)(void *, struct bpf_sock_tuple *, u32, test_einval_reserved = opts_def.error; opts_def.netns_id = -2; - ct = func(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def)); opts_def.netns_id = -1; if (ct) bpf_ct_release(ct); else test_einval_netns_id = opts_def.error; - ct = func(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def) - 1); + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def) - 1); if (ct) bpf_ct_release(ct); else test_einval_len_opts = opts_def.error; opts_def.l4proto = IPPROTO_ICMP; - ct = func(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def)); opts_def.l4proto = IPPROTO_TCP; if (ct) bpf_ct_release(ct); @@ -81,37 +104,75 @@ nf_ct_test(struct nf_conn *(*func)(void *, struct bpf_sock_tuple *, u32, test_eproto_l4proto = opts_def.error; opts_def.netns_id = 0xf00f; - ct = func(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def)); opts_def.netns_id = -1; if (ct) bpf_ct_release(ct); else test_enonet_netns_id = opts_def.error; - ct = func(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def)); if (ct) bpf_ct_release(ct); else test_enoent_lookup = opts_def.error; - ct = func(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4) - 1, &opts_def, sizeof(opts_def)); + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4) - 1, &opts_def, + sizeof(opts_def)); if (ct) bpf_ct_release(ct); else test_eafnosupport = opts_def.error; + + bpf_tuple.ipv4.saddr = bpf_get_prandom_u32(); /* src IP */ + bpf_tuple.ipv4.daddr = bpf_get_prandom_u32(); /* dst IP */ + bpf_tuple.ipv4.sport = bpf_get_prandom_u32(); /* src port */ + bpf_tuple.ipv4.dport = bpf_get_prandom_u32(); /* dst port */ + + ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def)); + if (ct) { + struct nf_conn *ct_ins; + + bpf_ct_set_timeout(ct, 10000); + bpf_ct_set_status(ct, IPS_CONFIRMED); + + ct_ins = bpf_ct_insert_entry(ct); + if (ct_ins) { + struct nf_conn *ct_lk; + + ct_lk = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), + &opts_def, sizeof(opts_def)); + if (ct_lk) { + /* update ct entry timeout */ + bpf_ct_change_timeout(ct_lk, 10000); + test_delta_timeout = ct_lk->timeout - bpf_jiffies64(); + test_delta_timeout /= CONFIG_HZ; + test_status = IPS_SEEN_REPLY; + bpf_ct_change_status(ct_lk, IPS_SEEN_REPLY); + bpf_ct_release(ct_lk); + test_succ_lookup = 0; + } + bpf_ct_release(ct_ins); + test_insert_entry = 0; + } + test_alloc_entry = 0; + } } SEC("xdp") int nf_xdp_ct_test(struct xdp_md *ctx) { - nf_ct_test((void *)bpf_xdp_ct_lookup, ctx); + nf_ct_test((void *)bpf_xdp_ct_lookup, (void *)bpf_xdp_ct_alloc, ctx); return 0; } SEC("tc") int nf_skb_ct_test(struct __sk_buff *ctx) { - nf_ct_test((void *)bpf_skb_ct_lookup, ctx); + nf_ct_test((void *)bpf_skb_ct_lookup, (void *)bpf_skb_ct_alloc, ctx); return 0; } diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c b/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c new file mode 100644 index 000000000000..bf79af15c808 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_core_read.h> + +struct nf_conn; + +struct bpf_ct_opts___local { + s32 netns_id; + s32 error; + u8 l4proto; + u8 reserved[3]; +} __attribute__((preserve_access_index)); + +struct nf_conn *bpf_skb_ct_alloc(struct __sk_buff *, struct bpf_sock_tuple *, u32, + struct bpf_ct_opts___local *, u32) __ksym; +struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *, struct bpf_sock_tuple *, u32, + struct bpf_ct_opts___local *, u32) __ksym; +struct nf_conn *bpf_ct_insert_entry(struct nf_conn *) __ksym; +void bpf_ct_release(struct nf_conn *) __ksym; +void bpf_ct_set_timeout(struct nf_conn *, u32) __ksym; +int bpf_ct_change_timeout(struct nf_conn *, u32) __ksym; +int bpf_ct_set_status(struct nf_conn *, u32) __ksym; +int bpf_ct_change_status(struct nf_conn *, u32) __ksym; + +SEC("?tc") +int alloc_release(struct __sk_buff *ctx) +{ + struct bpf_ct_opts___local opts = {}; + struct bpf_sock_tuple tup = {}; + struct nf_conn *ct; + + ct = bpf_skb_ct_alloc(ctx, &tup, sizeof(tup.ipv4), &opts, sizeof(opts)); + if (!ct) + return 0; + bpf_ct_release(ct); + return 0; +} + +SEC("?tc") +int insert_insert(struct __sk_buff *ctx) +{ + struct bpf_ct_opts___local opts = {}; + struct bpf_sock_tuple tup = {}; + struct nf_conn *ct; + + ct = bpf_skb_ct_alloc(ctx, &tup, sizeof(tup.ipv4), &opts, sizeof(opts)); + if (!ct) + return 0; + ct = bpf_ct_insert_entry(ct); + if (!ct) + return 0; + ct = bpf_ct_insert_entry(ct); + return 0; +} + +SEC("?tc") +int lookup_insert(struct __sk_buff *ctx) +{ + struct bpf_ct_opts___local opts = {}; + struct bpf_sock_tuple tup = {}; + struct nf_conn *ct; + + ct = bpf_skb_ct_lookup(ctx, &tup, sizeof(tup.ipv4), &opts, sizeof(opts)); + if (!ct) + return 0; + bpf_ct_insert_entry(ct); + return 0; +} + +SEC("?tc") +int set_timeout_after_insert(struct __sk_buff *ctx) +{ + struct bpf_ct_opts___local opts = {}; + struct bpf_sock_tuple tup = {}; + struct nf_conn *ct; + + ct = bpf_skb_ct_alloc(ctx, &tup, sizeof(tup.ipv4), &opts, sizeof(opts)); + if (!ct) + return 0; + ct = bpf_ct_insert_entry(ct); + if (!ct) + return 0; + bpf_ct_set_timeout(ct, 0); + return 0; +} + +SEC("?tc") +int set_status_after_insert(struct __sk_buff *ctx) +{ + struct bpf_ct_opts___local opts = {}; + struct bpf_sock_tuple tup = {}; + struct nf_conn *ct; + + ct = bpf_skb_ct_alloc(ctx, &tup, sizeof(tup.ipv4), &opts, sizeof(opts)); + if (!ct) + return 0; + ct = bpf_ct_insert_entry(ct); + if (!ct) + return 0; + bpf_ct_set_status(ct, 0); + return 0; +} + +SEC("?tc") +int change_timeout_after_alloc(struct __sk_buff *ctx) +{ + struct bpf_ct_opts___local opts = {}; + struct bpf_sock_tuple tup = {}; + struct nf_conn *ct; + + ct = bpf_skb_ct_alloc(ctx, &tup, sizeof(tup.ipv4), &opts, sizeof(opts)); + if (!ct) + return 0; + bpf_ct_change_timeout(ct, 0); + return 0; +} + +SEC("?tc") +int change_status_after_alloc(struct __sk_buff *ctx) +{ + struct bpf_ct_opts___local opts = {}; + struct bpf_sock_tuple tup = {}; + struct nf_conn *ct; + + ct = bpf_skb_ct_alloc(ctx, &tup, sizeof(tup.ipv4), &opts, sizeof(opts)); + if (!ct) + return 0; + bpf_ct_change_status(ct, 0); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_core_extern.c b/tools/testing/selftests/bpf/progs/test_core_extern.c index 3ac3603ad53d..a3c7c1042f35 100644 --- a/tools/testing/selftests/bpf/progs/test_core_extern.c +++ b/tools/testing/selftests/bpf/progs/test_core_extern.c @@ -11,6 +11,7 @@ static int (*bpf_missing_helper)(const void *arg1, int arg2) = (void *) 999; extern int LINUX_KERNEL_VERSION __kconfig; +extern int LINUX_UNKNOWN_VIRTUAL_EXTERN __kconfig __weak; extern bool CONFIG_BPF_SYSCALL __kconfig; /* strong */ extern enum libbpf_tristate CONFIG_TRISTATE __kconfig __weak; extern bool CONFIG_BOOL __kconfig __weak; @@ -22,6 +23,7 @@ extern const char CONFIG_STR[8] __kconfig __weak; extern uint64_t CONFIG_MISSING __kconfig __weak; uint64_t kern_ver = -1; +uint64_t unkn_virt_val = -1; uint64_t bpf_syscall = -1; uint64_t tristate_val = -1; uint64_t bool_val = -1; @@ -38,6 +40,7 @@ int handle_sys_enter(struct pt_regs *ctx) int i; kern_ver = LINUX_KERNEL_VERSION; + unkn_virt_val = LINUX_UNKNOWN_VIRTUAL_EXTERN; bpf_syscall = CONFIG_BPF_SYSCALL; tristate_val = CONFIG_TRISTATE; bool_val = CONFIG_BOOL; diff --git a/tools/testing/selftests/bpf/progs/test_probe_user.c b/tools/testing/selftests/bpf/progs/test_probe_user.c index 702578a5e496..8e1495008e4d 100644 --- a/tools/testing/selftests/bpf/progs/test_probe_user.c +++ b/tools/testing/selftests/bpf/progs/test_probe_user.c @@ -1,35 +1,20 @@ // SPDX-License-Identifier: GPL-2.0 - -#include <linux/ptrace.h> -#include <linux/bpf.h> - -#include <netinet/in.h> - +#include "vmlinux.h" #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> #include "bpf_misc.h" static struct sockaddr_in old; -SEC("kprobe/" SYS_PREFIX "sys_connect") -int BPF_KPROBE(handle_sys_connect) +SEC("ksyscall/connect") +int BPF_KSYSCALL(handle_sys_connect, int fd, struct sockaddr_in *uservaddr, int addrlen) { -#if SYSCALL_WRAPPER == 1 - struct pt_regs *real_regs; -#endif struct sockaddr_in new; - void *ptr; - -#if SYSCALL_WRAPPER == 0 - ptr = (void *)PT_REGS_PARM2(ctx); -#else - real_regs = (struct pt_regs *)PT_REGS_PARM1(ctx); - bpf_probe_read_kernel(&ptr, sizeof(ptr), &PT_REGS_PARM2(real_regs)); -#endif - bpf_probe_read_user(&old, sizeof(old), ptr); + bpf_probe_read_user(&old, sizeof(old), uservaddr); __builtin_memset(&new, 0xab, sizeof(new)); - bpf_probe_write_user(ptr, &new, sizeof(new)); + bpf_probe_write_user(uservaddr, &new, sizeof(new)); return 0; } diff --git a/tools/testing/selftests/bpf/progs/test_skeleton.c b/tools/testing/selftests/bpf/progs/test_skeleton.c index 1b1187d2967b..1a4e93f6d9df 100644 --- a/tools/testing/selftests/bpf/progs/test_skeleton.c +++ b/tools/testing/selftests/bpf/progs/test_skeleton.c @@ -51,6 +51,8 @@ int out_dynarr[4] SEC(".data.dyn") = { 1, 2, 3, 4 }; int read_mostly_var __read_mostly; int out_mostly_var; +char huge_arr[16 * 1024 * 1024]; + SEC("raw_tp/sys_enter") int handler(const void *ctx) { @@ -71,6 +73,8 @@ int handler(const void *ctx) out_mostly_var = read_mostly_var; + huge_arr[sizeof(huge_arr) - 1] = 123; + return 0; } diff --git a/tools/testing/selftests/bpf/progs/test_xdp_noinline.c b/tools/testing/selftests/bpf/progs/test_xdp_noinline.c index 125d872d7981..ba48fcb98ab2 100644 --- a/tools/testing/selftests/bpf/progs/test_xdp_noinline.c +++ b/tools/testing/selftests/bpf/progs/test_xdp_noinline.c @@ -239,7 +239,7 @@ bool parse_udp(void *data, void *data_end, udp = data + off; if (udp + 1 > data_end) - return 0; + return false; if (!is_icmp) { pckt->flow.port16[0] = udp->source; pckt->flow.port16[1] = udp->dest; @@ -247,7 +247,7 @@ bool parse_udp(void *data, void *data_end, pckt->flow.port16[0] = udp->dest; pckt->flow.port16[1] = udp->source; } - return 1; + return true; } static __attribute__ ((noinline)) @@ -261,7 +261,7 @@ bool parse_tcp(void *data, void *data_end, tcp = data + off; if (tcp + 1 > data_end) - return 0; + return false; if (tcp->syn) pckt->flags |= (1 << 1); if (!is_icmp) { @@ -271,7 +271,7 @@ bool parse_tcp(void *data, void *data_end, pckt->flow.port16[0] = tcp->dest; pckt->flow.port16[1] = tcp->source; } - return 1; + return true; } static __attribute__ ((noinline)) @@ -287,7 +287,7 @@ bool encap_v6(struct xdp_md *xdp, struct ctl_value *cval, void *data; if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) - return 0; + return false; data = (void *)(long)xdp->data; data_end = (void *)(long)xdp->data_end; new_eth = data; @@ -295,7 +295,7 @@ bool encap_v6(struct xdp_md *xdp, struct ctl_value *cval, old_eth = data + sizeof(struct ipv6hdr); if (new_eth + 1 > data_end || old_eth + 1 > data_end || ip6h + 1 > data_end) - return 0; + return false; memcpy(new_eth->eth_dest, cval->mac, 6); memcpy(new_eth->eth_source, old_eth->eth_dest, 6); new_eth->eth_proto = 56710; @@ -314,7 +314,7 @@ bool encap_v6(struct xdp_md *xdp, struct ctl_value *cval, ip6h->saddr.in6_u.u6_addr32[2] = 3; ip6h->saddr.in6_u.u6_addr32[3] = ip_suffix; memcpy(ip6h->daddr.in6_u.u6_addr32, dst->dstv6, 16); - return 1; + return true; } static __attribute__ ((noinline)) @@ -335,7 +335,7 @@ bool encap_v4(struct xdp_md *xdp, struct ctl_value *cval, ip_suffix <<= 15; ip_suffix ^= pckt->flow.src; if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) - return 0; + return false; data = (void *)(long)xdp->data; data_end = (void *)(long)xdp->data_end; new_eth = data; @@ -343,7 +343,7 @@ bool encap_v4(struct xdp_md *xdp, struct ctl_value *cval, old_eth = data + sizeof(struct iphdr); if (new_eth + 1 > data_end || old_eth + 1 > data_end || iph + 1 > data_end) - return 0; + return false; memcpy(new_eth->eth_dest, cval->mac, 6); memcpy(new_eth->eth_source, old_eth->eth_dest, 6); new_eth->eth_proto = 8; @@ -367,8 +367,8 @@ bool encap_v4(struct xdp_md *xdp, struct ctl_value *cval, csum += *next_iph_u16++; iph->check = ~((csum & 0xffff) + (csum >> 16)); if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct iphdr))) - return 0; - return 1; + return false; + return true; } static __attribute__ ((noinline)) @@ -386,10 +386,10 @@ bool decap_v6(struct xdp_md *xdp, void **data, void **data_end, bool inner_v4) else new_eth->eth_proto = 56710; if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct ipv6hdr))) - return 0; + return false; *data = (void *)(long)xdp->data; *data_end = (void *)(long)xdp->data_end; - return 1; + return true; } static __attribute__ ((noinline)) @@ -404,10 +404,10 @@ bool decap_v4(struct xdp_md *xdp, void **data, void **data_end) memcpy(new_eth->eth_dest, old_eth->eth_dest, 6); new_eth->eth_proto = 8; if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct iphdr))) - return 0; + return false; *data = (void *)(long)xdp->data; *data_end = (void *)(long)xdp->data_end; - return 1; + return true; } static __attribute__ ((noinline)) diff --git a/tools/testing/selftests/bpf/test_xdp_veth.sh b/tools/testing/selftests/bpf/test_xdp_veth.sh index 392d28cc4e58..49936c4c8567 100755 --- a/tools/testing/selftests/bpf/test_xdp_veth.sh +++ b/tools/testing/selftests/bpf/test_xdp_veth.sh @@ -106,9 +106,9 @@ bpftool prog loadall \ bpftool map update pinned $BPF_DIR/maps/tx_port key 0 0 0 0 value 122 0 0 0 bpftool map update pinned $BPF_DIR/maps/tx_port key 1 0 0 0 value 133 0 0 0 bpftool map update pinned $BPF_DIR/maps/tx_port key 2 0 0 0 value 111 0 0 0 -ip link set dev veth1 xdp pinned $BPF_DIR/progs/redirect_map_0 -ip link set dev veth2 xdp pinned $BPF_DIR/progs/redirect_map_1 -ip link set dev veth3 xdp pinned $BPF_DIR/progs/redirect_map_2 +ip link set dev veth1 xdp pinned $BPF_DIR/progs/xdp_redirect_map_0 +ip link set dev veth2 xdp pinned $BPF_DIR/progs/xdp_redirect_map_1 +ip link set dev veth3 xdp pinned $BPF_DIR/progs/xdp_redirect_map_2 ip -n ${NS1} link set dev veth11 xdp obj xdp_dummy.o sec xdp ip -n ${NS2} link set dev veth22 xdp obj xdp_tx.o sec xdp diff --git a/tools/testing/selftests/bpf/verifier/bpf_loop_inline.c b/tools/testing/selftests/bpf/verifier/bpf_loop_inline.c index 2d0023659d88..a535d41dc20d 100644 --- a/tools/testing/selftests/bpf/verifier/bpf_loop_inline.c +++ b/tools/testing/selftests/bpf/verifier/bpf_loop_inline.c @@ -251,6 +251,7 @@ .expected_insns = { PSEUDO_CALL_INSN() }, .unexpected_insns = { HELPER_CALL_INSN() }, .result = ACCEPT, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, .func_info = { { 0, MAIN_TYPE }, { 16, CALLBACK_TYPE } }, .func_info_cnt = 2, BTF_TYPES diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index 743ed34c1238..3fb4f69b1962 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -219,6 +219,59 @@ .errstr = "variable ptr_ access var_off=(0x0; 0x7) disallowed", }, { + "calls: invalid kfunc call: referenced arg needs refcounted PTR_TO_BTF_ID", + .insns = { + BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), + BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_6, BPF_REG_0), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 16), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_SCHED_CLS, + .fixup_kfunc_btf_id = { + { "bpf_kfunc_call_test_acquire", 3 }, + { "bpf_kfunc_call_test_ref", 8 }, + { "bpf_kfunc_call_test_ref", 10 }, + }, + .result_unpriv = REJECT, + .result = REJECT, + .errstr = "R1 must be referenced", +}, +{ + "calls: valid kfunc call: referenced arg needs refcounted PTR_TO_BTF_ID", + .insns = { + BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), + BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_6, BPF_REG_0), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_SCHED_CLS, + .fixup_kfunc_btf_id = { + { "bpf_kfunc_call_test_acquire", 3 }, + { "bpf_kfunc_call_test_ref", 8 }, + { "bpf_kfunc_call_test_release", 10 }, + }, + .result_unpriv = REJECT, + .result = ACCEPT, +}, +{ "calls: basic sanity", .insns = { BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2), diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index db05b3764b77..80628bf8413a 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile @@ -38,6 +38,7 @@ TEST_PROGS += srv6_end_dt6_l3vpn_test.sh TEST_PROGS += vrf_strict_mode_test.sh TEST_PROGS += arp_ndisc_evict_nocarrier.sh TEST_PROGS += ndisc_unsolicited_na_test.sh +TEST_PROGS += arp_ndisc_untracked_subnets.sh TEST_PROGS += stress_reuseport_listen.sh TEST_PROGS_EXTENDED := in_netns.sh setup_loopback.sh setup_veth.sh TEST_PROGS_EXTENDED += toeplitz_client.sh toeplitz.sh diff --git a/tools/testing/selftests/net/af_unix/unix_connect.c b/tools/testing/selftests/net/af_unix/unix_connect.c index 157e44ef7f37..d799fd8f5c7c 100644 --- a/tools/testing/selftests/net/af_unix/unix_connect.c +++ b/tools/testing/selftests/net/af_unix/unix_connect.c @@ -3,6 +3,7 @@ #define _GNU_SOURCE #include <sched.h> +#include <stddef.h> #include <stdio.h> #include <unistd.h> @@ -112,8 +113,6 @@ FIXTURE_TEARDOWN(unix_connect) remove("test"); } -#define offsetof(type, member) ((size_t)&((type *)0)->(member)) - TEST_F(unix_connect, test) { socklen_t addrlen; diff --git a/tools/testing/selftests/net/arp_ndisc_untracked_subnets.sh b/tools/testing/selftests/net/arp_ndisc_untracked_subnets.sh new file mode 100755 index 000000000000..c899b446acb6 --- /dev/null +++ b/tools/testing/selftests/net/arp_ndisc_untracked_subnets.sh @@ -0,0 +1,308 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# 2 namespaces: one host and one router. Use arping from the host to send a +# garp to the router. Router accepts or ignores based on its arp_accept +# or accept_untracked_na configuration. + +TESTS="arp ndisc" + +ROUTER_NS="ns-router" +ROUTER_NS_V6="ns-router-v6" +ROUTER_INTF="veth-router" +ROUTER_ADDR="10.0.10.1" +ROUTER_ADDR_V6="2001:db8:abcd:0012::1" + +HOST_NS="ns-host" +HOST_NS_V6="ns-host-v6" +HOST_INTF="veth-host" +HOST_ADDR="10.0.10.2" +HOST_ADDR_V6="2001:db8:abcd:0012::2" + +SUBNET_WIDTH=24 +PREFIX_WIDTH_V6=64 + +cleanup() { + ip netns del ${HOST_NS} + ip netns del ${ROUTER_NS} +} + +cleanup_v6() { + ip netns del ${HOST_NS_V6} + ip netns del ${ROUTER_NS_V6} +} + +setup() { + set -e + local arp_accept=$1 + + # Set up two namespaces + ip netns add ${ROUTER_NS} + ip netns add ${HOST_NS} + + # Set up interfaces veth0 and veth1, which are pairs in separate + # namespaces. veth0 is veth-router, veth1 is veth-host. + # first, set up the inteface's link to the namespace + # then, set the interface "up" + ip netns exec ${ROUTER_NS} ip link add name ${ROUTER_INTF} \ + type veth peer name ${HOST_INTF} + + ip netns exec ${ROUTER_NS} ip link set dev ${ROUTER_INTF} up + ip netns exec ${ROUTER_NS} ip link set dev ${HOST_INTF} netns ${HOST_NS} + + ip netns exec ${HOST_NS} ip link set dev ${HOST_INTF} up + ip netns exec ${ROUTER_NS} ip addr add ${ROUTER_ADDR}/${SUBNET_WIDTH} \ + dev ${ROUTER_INTF} + + ip netns exec ${HOST_NS} ip addr add ${HOST_ADDR}/${SUBNET_WIDTH} \ + dev ${HOST_INTF} + ip netns exec ${HOST_NS} ip route add default via ${HOST_ADDR} \ + dev ${HOST_INTF} + ip netns exec ${ROUTER_NS} ip route add default via ${ROUTER_ADDR} \ + dev ${ROUTER_INTF} + + ROUTER_CONF=net.ipv4.conf.${ROUTER_INTF} + ip netns exec ${ROUTER_NS} sysctl -w \ + ${ROUTER_CONF}.arp_accept=${arp_accept} >/dev/null 2>&1 + set +e +} + +setup_v6() { + set -e + local accept_untracked_na=$1 + + # Set up two namespaces + ip netns add ${ROUTER_NS_V6} + ip netns add ${HOST_NS_V6} + + # Set up interfaces veth0 and veth1, which are pairs in separate + # namespaces. veth0 is veth-router, veth1 is veth-host. + # first, set up the inteface's link to the namespace + # then, set the interface "up" + ip -6 -netns ${ROUTER_NS_V6} link add name ${ROUTER_INTF} \ + type veth peer name ${HOST_INTF} + + ip -6 -netns ${ROUTER_NS_V6} link set dev ${ROUTER_INTF} up + ip -6 -netns ${ROUTER_NS_V6} link set dev ${HOST_INTF} netns \ + ${HOST_NS_V6} + + ip -6 -netns ${HOST_NS_V6} link set dev ${HOST_INTF} up + ip -6 -netns ${ROUTER_NS_V6} addr add \ + ${ROUTER_ADDR_V6}/${PREFIX_WIDTH_V6} dev ${ROUTER_INTF} nodad + + HOST_CONF=net.ipv6.conf.${HOST_INTF} + ip netns exec ${HOST_NS_V6} sysctl -qw ${HOST_CONF}.ndisc_notify=1 + ip netns exec ${HOST_NS_V6} sysctl -qw ${HOST_CONF}.disable_ipv6=0 + ip -6 -netns ${HOST_NS_V6} addr add ${HOST_ADDR_V6}/${PREFIX_WIDTH_V6} \ + dev ${HOST_INTF} + + ROUTER_CONF=net.ipv6.conf.${ROUTER_INTF} + + ip netns exec ${ROUTER_NS_V6} sysctl -w \ + ${ROUTER_CONF}.forwarding=1 >/dev/null 2>&1 + ip netns exec ${ROUTER_NS_V6} sysctl -w \ + ${ROUTER_CONF}.drop_unsolicited_na=0 >/dev/null 2>&1 + ip netns exec ${ROUTER_NS_V6} sysctl -w \ + ${ROUTER_CONF}.accept_untracked_na=${accept_untracked_na} \ + >/dev/null 2>&1 + set +e +} + +verify_arp() { + local arp_accept=$1 + local same_subnet=$2 + + neigh_show_output=$(ip netns exec ${ROUTER_NS} ip neigh get \ + ${HOST_ADDR} dev ${ROUTER_INTF} 2>/dev/null) + + if [ ${arp_accept} -eq 1 ]; then + # Neighbor entries expected + [[ ${neigh_show_output} ]] + elif [ ${arp_accept} -eq 2 ]; then + if [ ${same_subnet} -eq 1 ]; then + # Neighbor entries expected + [[ ${neigh_show_output} ]] + else + [[ -z "${neigh_show_output}" ]] + fi + else + [[ -z "${neigh_show_output}" ]] + fi + } + +arp_test_gratuitous() { + set -e + local arp_accept=$1 + local same_subnet=$2 + + if [ ${arp_accept} -eq 2 ]; then + test_msg=("test_arp: " + "accept_arp=$1 " + "same_subnet=$2") + if [ ${same_subnet} -eq 0 ]; then + HOST_ADDR=10.0.11.3 + else + HOST_ADDR=10.0.10.3 + fi + else + test_msg=("test_arp: " + "accept_arp=$1") + fi + # Supply arp_accept option to set up which sets it in sysctl + setup ${arp_accept} + ip netns exec ${HOST_NS} arping -A -U ${HOST_ADDR} -c1 2>&1 >/dev/null + + if verify_arp $1 $2; then + printf " TEST: %-60s [ OK ]\n" "${test_msg[*]}" + else + printf " TEST: %-60s [FAIL]\n" "${test_msg[*]}" + fi + cleanup + set +e +} + +arp_test_gratuitous_combinations() { + arp_test_gratuitous 0 + arp_test_gratuitous 1 + arp_test_gratuitous 2 0 # Second entry indicates subnet or not + arp_test_gratuitous 2 1 +} + +cleanup_tcpdump() { + set -e + [[ ! -z ${tcpdump_stdout} ]] && rm -f ${tcpdump_stdout} + [[ ! -z ${tcpdump_stderr} ]] && rm -f ${tcpdump_stderr} + tcpdump_stdout= + tcpdump_stderr= + set +e +} + +start_tcpdump() { + set -e + tcpdump_stdout=`mktemp` + tcpdump_stderr=`mktemp` + ip netns exec ${ROUTER_NS_V6} timeout 15s \ + tcpdump --immediate-mode -tpni ${ROUTER_INTF} -c 1 \ + "icmp6 && icmp6[0] == 136 && src ${HOST_ADDR_V6}" \ + > ${tcpdump_stdout} 2> /dev/null + set +e +} + +verify_ndisc() { + local accept_untracked_na=$1 + local same_subnet=$2 + + neigh_show_output=$(ip -6 -netns ${ROUTER_NS_V6} neigh show \ + to ${HOST_ADDR_V6} dev ${ROUTER_INTF} nud stale) + + if [ ${accept_untracked_na} -eq 1 ]; then + # Neighbour entry expected to be present + [[ ${neigh_show_output} ]] + elif [ ${accept_untracked_na} -eq 2 ]; then + if [ ${same_subnet} -eq 1 ]; then + [[ ${neigh_show_output} ]] + else + [[ -z "${neigh_show_output}" ]] + fi + else + # Neighbour entry expected to be absent for all other cases + [[ -z "${neigh_show_output}" ]] + fi +} + +ndisc_test_untracked_advertisements() { + set -e + test_msg=("test_ndisc: " + "accept_untracked_na=$1") + + local accept_untracked_na=$1 + local same_subnet=$2 + if [ ${accept_untracked_na} -eq 2 ]; then + test_msg=("test_ndisc: " + "accept_untracked_na=$1 " + "same_subnet=$2") + if [ ${same_subnet} -eq 0 ]; then + # Not same subnet + HOST_ADDR_V6=2000:db8:abcd:0013::4 + else + HOST_ADDR_V6=2001:db8:abcd:0012::3 + fi + fi + setup_v6 $1 $2 + start_tcpdump + + if verify_ndisc $1 $2; then + printf " TEST: %-60s [ OK ]\n" "${test_msg[*]}" + else + printf " TEST: %-60s [FAIL]\n" "${test_msg[*]}" + fi + + cleanup_tcpdump + cleanup_v6 + set +e +} + +ndisc_test_untracked_combinations() { + ndisc_test_untracked_advertisements 0 + ndisc_test_untracked_advertisements 1 + ndisc_test_untracked_advertisements 2 0 + ndisc_test_untracked_advertisements 2 1 +} + +################################################################################ +# usage + +usage() +{ + cat <<EOF +usage: ${0##*/} OPTS + + -t <test> Test(s) to run (default: all) + (options: $TESTS) +EOF +} + +################################################################################ +# main + +while getopts ":t:h" opt; do + case $opt in + t) TESTS=$OPTARG;; + h) usage; exit 0;; + *) usage; exit 1;; + esac +done + +if [ "$(id -u)" -ne 0 ];then + echo "SKIP: Need root privileges" + exit $ksft_skip; +fi + +if [ ! -x "$(command -v ip)" ]; then + echo "SKIP: Could not run test without ip tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v tcpdump)" ]; then + echo "SKIP: Could not run test without tcpdump tool" + exit $ksft_skip +fi + +if [ ! -x "$(command -v arping)" ]; then + echo "SKIP: Could not run test without arping tool" + exit $ksft_skip +fi + +# start clean +cleanup &> /dev/null +cleanup_v6 &> /dev/null + +for t in $TESTS +do + case $t in + arp_test_gratuitous_combinations|arp) arp_test_gratuitous_combinations;; + ndisc_test_untracked_combinations|ndisc) \ + ndisc_test_untracked_combinations;; + help) echo "Test names: $TESTS"; exit 0;; +esac +done diff --git a/tools/testing/selftests/net/ipv6_flowlabel.c b/tools/testing/selftests/net/ipv6_flowlabel.c index a7c41375374f..708a9822259d 100644 --- a/tools/testing/selftests/net/ipv6_flowlabel.c +++ b/tools/testing/selftests/net/ipv6_flowlabel.c @@ -9,6 +9,7 @@ #include <errno.h> #include <fcntl.h> #include <limits.h> +#include <linux/icmpv6.h> #include <linux/in6.h> #include <stdbool.h> #include <stdio.h> @@ -29,26 +30,48 @@ #ifndef IPV6_FLOWLABEL_MGR #define IPV6_FLOWLABEL_MGR 32 #endif +#ifndef IPV6_FLOWINFO_SEND +#define IPV6_FLOWINFO_SEND 33 +#endif #define FLOWLABEL_WILDCARD ((uint32_t) -1) static const char cfg_data[] = "a"; static uint32_t cfg_label = 1; +static bool use_ping; +static bool use_flowinfo_send; + +static struct icmp6hdr icmp6 = { + .icmp6_type = ICMPV6_ECHO_REQUEST +}; + +static struct sockaddr_in6 addr = { + .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_LOOPBACK_INIT, +}; static void do_send(int fd, bool with_flowlabel, uint32_t flowlabel) { char control[CMSG_SPACE(sizeof(flowlabel))] = {0}; struct msghdr msg = {0}; - struct iovec iov = {0}; + struct iovec iov = { + .iov_base = (char *)cfg_data, + .iov_len = sizeof(cfg_data) + }; int ret; - iov.iov_base = (char *)cfg_data; - iov.iov_len = sizeof(cfg_data); + if (use_ping) { + iov.iov_base = &icmp6; + iov.iov_len = sizeof(icmp6); + } msg.msg_iov = &iov; msg.msg_iovlen = 1; - if (with_flowlabel) { + if (use_flowinfo_send) { + msg.msg_name = &addr; + msg.msg_namelen = sizeof(addr); + } else if (with_flowlabel) { struct cmsghdr *cm; cm = (void *)control; @@ -94,6 +117,8 @@ static void do_recv(int fd, bool with_flowlabel, uint32_t expect) ret = recvmsg(fd, &msg, 0); if (ret == -1) error(1, errno, "recv"); + if (use_ping) + goto parse_cmsg; if (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) error(1, 0, "recv: truncated"); if (ret != sizeof(cfg_data)) @@ -101,6 +126,7 @@ static void do_recv(int fd, bool with_flowlabel, uint32_t expect) if (memcmp(data, cfg_data, sizeof(data))) error(1, 0, "recv: data mismatch"); +parse_cmsg: cm = CMSG_FIRSTHDR(&msg); if (with_flowlabel) { if (!cm) @@ -114,9 +140,11 @@ static void do_recv(int fd, bool with_flowlabel, uint32_t expect) flowlabel = ntohl(*(uint32_t *)CMSG_DATA(cm)); fprintf(stderr, "recv with label %u\n", flowlabel); - if (expect != FLOWLABEL_WILDCARD && expect != flowlabel) + if (expect != FLOWLABEL_WILDCARD && expect != flowlabel) { fprintf(stderr, "recv: incorrect flowlabel %u != %u\n", flowlabel, expect); + error(1, 0, "recv: flowlabel is wrong"); + } } else { fprintf(stderr, "recv without label\n"); @@ -165,11 +193,17 @@ static void parse_opts(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "l:")) != -1) { + while ((c = getopt(argc, argv, "l:ps")) != -1) { switch (c) { case 'l': cfg_label = strtoul(optarg, NULL, 0); break; + case 'p': + use_ping = true; + break; + case 's': + use_flowinfo_send = true; + break; default: error(1, 0, "%s: parse error", argv[0]); } @@ -178,27 +212,30 @@ static void parse_opts(int argc, char **argv) int main(int argc, char **argv) { - struct sockaddr_in6 addr = { - .sin6_family = AF_INET6, - .sin6_port = htons(8000), - .sin6_addr = IN6ADDR_LOOPBACK_INIT, - }; const int one = 1; int fdt, fdr; + int prot = 0; + + addr.sin6_port = htons(8000); parse_opts(argc, argv); - fdt = socket(PF_INET6, SOCK_DGRAM, 0); + if (use_ping) { + fprintf(stderr, "attempting to use ping sockets\n"); + prot = IPPROTO_ICMPV6; + } + + fdt = socket(PF_INET6, SOCK_DGRAM, prot); if (fdt == -1) error(1, errno, "socket t"); - fdr = socket(PF_INET6, SOCK_DGRAM, 0); + fdr = use_ping ? fdt : socket(PF_INET6, SOCK_DGRAM, 0); if (fdr == -1) error(1, errno, "socket r"); if (connect(fdt, (void *)&addr, sizeof(addr))) error(1, errno, "connect"); - if (bind(fdr, (void *)&addr, sizeof(addr))) + if (!use_ping && bind(fdr, (void *)&addr, sizeof(addr))) error(1, errno, "bind"); flowlabel_get(fdt, cfg_label, IPV6_FL_S_EXCL, IPV6_FL_F_CREATE); @@ -216,13 +253,21 @@ int main(int argc, char **argv) do_recv(fdr, false, 0); } + if (use_flowinfo_send) { + fprintf(stderr, "using IPV6_FLOWINFO_SEND to send label\n"); + addr.sin6_flowinfo = htonl(cfg_label); + if (setsockopt(fdt, SOL_IPV6, IPV6_FLOWINFO_SEND, &one, + sizeof(one)) == -1) + error(1, errno, "setsockopt flowinfo_send"); + } + fprintf(stderr, "send label\n"); do_send(fdt, true, cfg_label); do_recv(fdr, true, cfg_label); if (close(fdr)) error(1, errno, "close r"); - if (close(fdt)) + if (!use_ping && close(fdt)) error(1, errno, "close t"); return 0; diff --git a/tools/testing/selftests/net/ipv6_flowlabel.sh b/tools/testing/selftests/net/ipv6_flowlabel.sh index d3bc6442704e..cee95e252bee 100755 --- a/tools/testing/selftests/net/ipv6_flowlabel.sh +++ b/tools/testing/selftests/net/ipv6_flowlabel.sh @@ -18,4 +18,20 @@ echo "TEST datapath (with auto-flowlabels)" ./in_netns.sh \ sh -c 'sysctl -q -w net.ipv6.auto_flowlabels=1 && ./ipv6_flowlabel -l 1' +echo "TEST datapath (with ping-sockets)" +./in_netns.sh \ + sh -c 'sysctl -q -w net.ipv6.flowlabel_reflect=4 && \ + sysctl -q -w net.ipv4.ping_group_range="0 2147483647" && \ + ./ipv6_flowlabel -l 1 -p' + +echo "TEST datapath (with flowinfo-send)" +./in_netns.sh \ + sh -c './ipv6_flowlabel -l 1 -s' + +echo "TEST datapath (with ping-sockets flowinfo-send)" +./in_netns.sh \ + sh -c 'sysctl -q -w net.ipv6.flowlabel_reflect=4 && \ + sysctl -q -w net.ipv4.ping_group_range="0 2147483647" && \ + ./ipv6_flowlabel -l 1 -p -s' + echo OK. All tests passed diff --git a/tools/testing/selftests/net/tls.c b/tools/testing/selftests/net/tls.c index dc26aae0feb0..4ecbac197c46 100644 --- a/tools/testing/selftests/net/tls.c +++ b/tools/testing/selftests/net/tls.c @@ -1597,6 +1597,38 @@ TEST_F(tls_err, bad_cmsg) EXPECT_EQ(errno, EBADMSG); } +TEST_F(tls_err, timeo) +{ + struct timeval tv = { .tv_usec = 10000, }; + char buf[128]; + int ret; + + if (self->notls) + SKIP(return, "no TLS support"); + + ret = setsockopt(self->cfd2, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + ASSERT_EQ(ret, 0); + + ret = fork(); + ASSERT_GE(ret, 0); + + if (ret) { + usleep(1000); /* Give child a head start */ + + EXPECT_EQ(recv(self->cfd2, buf, sizeof(buf), 0), -1); + EXPECT_EQ(errno, EAGAIN); + + EXPECT_EQ(recv(self->cfd2, buf, sizeof(buf), 0), -1); + EXPECT_EQ(errno, EAGAIN); + + wait(&ret); + } else { + EXPECT_EQ(recv(self->cfd2, buf, sizeof(buf), 0), -1); + EXPECT_EQ(errno, EAGAIN); + exit(0); + } +} + TEST(non_established) { struct tls12_crypto_info_aes_gcm_256 tls12; struct sockaddr_in addr; |