diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2020-01-29 03:02:33 +0300 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2020-01-29 03:02:33 +0300 |
commit | bd2463ac7d7ec51d432f23bf0e893fb371a908cd (patch) | |
tree | 3da32c23be83adb9d9bda7e51b51fa39f69f2447 /tools/testing/selftests/drivers | |
parent | a78208e2436963d0b2c7d186277d6e1a9755029a (diff) | |
parent | f76e4c167ea2212e23c15ee7e601a865e822c291 (diff) | |
download | linux-bd2463ac7d7ec51d432f23bf0e893fb371a908cd.tar.xz |
Merge git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next
Pull networking updates from David Miller:
1) Add WireGuard
2) Add HE and TWT support to ath11k driver, from John Crispin.
3) Add ESP in TCP encapsulation support, from Sabrina Dubroca.
4) Add variable window congestion control to TIPC, from Jon Maloy.
5) Add BCM84881 PHY driver, from Russell King.
6) Start adding netlink support for ethtool operations, from Michal
Kubecek.
7) Add XDP drop and TX action support to ena driver, from Sameeh
Jubran.
8) Add new ipv4 route notifications so that mlxsw driver does not have
to handle identical routes itself. From Ido Schimmel.
9) Add BPF dynamic program extensions, from Alexei Starovoitov.
10) Support RX and TX timestamping in igc, from Vinicius Costa Gomes.
11) Add support for macsec HW offloading, from Antoine Tenart.
12) Add initial support for MPTCP protocol, from Christoph Paasch,
Matthieu Baerts, Florian Westphal, Peter Krystad, and many others.
13) Add Octeontx2 PF support, from Sunil Goutham, Geetha sowjanya, Linu
Cherian, and others.
* git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next: (1469 commits)
net: phy: add default ARCH_BCM_IPROC for MDIO_BCM_IPROC
udp: segment looped gso packets correctly
netem: change mailing list
qed: FW 8.42.2.0 debug features
qed: rt init valid initialization changed
qed: Debug feature: ilt and mdump
qed: FW 8.42.2.0 Add fw overlay feature
qed: FW 8.42.2.0 HSI changes
qed: FW 8.42.2.0 iscsi/fcoe changes
qed: Add abstraction for different hsi values per chip
qed: FW 8.42.2.0 Additional ll2 type
qed: Use dmae to write to widebus registers in fw_funcs
qed: FW 8.42.2.0 Parser offsets modified
qed: FW 8.42.2.0 Queue Manager changes
qed: FW 8.42.2.0 Expose new registers and change windows
qed: FW 8.42.2.0 Internal ram offsets modifications
MAINTAINERS: Add entry for Marvell OcteonTX2 Physical Function driver
Documentation: net: octeontx2: Add RVU HW and drivers overview
octeontx2-pf: ethtool RSS config support
octeontx2-pf: Add basic ethtool support
...
Diffstat (limited to 'tools/testing/selftests/drivers')
11 files changed, 1526 insertions, 42 deletions
diff --git a/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_l3_drops.sh b/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_l3_drops.sh index b4efb023ae51..d88d8e47d11b 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_l3_drops.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_l3_drops.sh @@ -50,6 +50,8 @@ ALL_TESTS=" ipv6_mc_dip_reserved_scope_test ipv6_mc_dip_interface_local_scope_test blackhole_route_test + irif_disabled_test + erif_disabled_test " NUM_NETIFS=4 @@ -553,6 +555,116 @@ blackhole_route_test() __blackhole_route_test "6" "2001:db8:2::/120" "ipv6" $h2_ipv6 "icmpv6" } +irif_disabled_test() +{ + local trap_name="irif_disabled" + local group_name="l3_drops" + local t0_packets t0_bytes + local t1_packets t1_bytes + local mz_pid + + RET=0 + + ping_check $trap_name + + devlink_trap_action_set $trap_name "trap" + + # When RIF of a physical port ("Sub-port RIF") is destroyed, we first + # block the STP of the {Port, VLAN} so packets cannot get into the RIF. + # Using bridge enables us to see this trap because when bridge is + # destroyed, there is a small time window that packets can go into the + # RIF, while it is disabled. + ip link add dev br0 type bridge + ip link set dev $rp1 master br0 + ip address flush dev $rp1 + __addr_add_del br0 add 192.0.2.2/24 + ip li set dev br0 up + + t0_packets=$(devlink_trap_rx_packets_get $trap_name) + t0_bytes=$(devlink_trap_rx_bytes_get $trap_name) + + # Generate packets to h2 through br0 RIF that will be removed later + $MZ $h1 -t udp "sp=54321,dp=12345" -c 0 -p 100 -a own -b $rp1mac \ + -B $h2_ipv4 -q & + mz_pid=$! + + # Wait before removing br0 RIF to allow packets to go into the bridge. + sleep 1 + + # Flushing address will dismantle the RIF + ip address flush dev br0 + + t1_packets=$(devlink_trap_rx_packets_get $trap_name) + t1_bytes=$(devlink_trap_rx_bytes_get $trap_name) + + if [[ $t0_packets -eq $t1_packets && $t0_bytes -eq $t1_bytes ]]; then + check_err 1 "Trap stats idle when packets should be trapped" + fi + + log_test "Ingress RIF disabled" + + kill $mz_pid && wait $mz_pid &> /dev/null + ip link set dev $rp1 nomaster + __addr_add_del $rp1 add 192.0.2.2/24 2001:db8:1::2/64 + ip link del dev br0 type bridge + devlink_trap_action_set $trap_name "drop" +} + +erif_disabled_test() +{ + local trap_name="erif_disabled" + local group_name="l3_drops" + local t0_packets t0_bytes + local t1_packets t1_bytes + local mz_pid + + RET=0 + + ping_check $trap_name + + devlink_trap_action_set $trap_name "trap" + ip link add dev br0 type bridge + ip add flush dev $rp1 + ip link set dev $rp1 master br0 + __addr_add_del br0 add 192.0.2.2/24 + ip link set dev br0 up + + t0_packets=$(devlink_trap_rx_packets_get $trap_name) + t0_bytes=$(devlink_trap_rx_bytes_get $trap_name) + + rp2mac=$(mac_get $rp2) + + # Generate packets that should go out through br0 RIF that will be + # removed later + $MZ $h2 -t udp "sp=54321,dp=12345" -c 0 -p 100 -a own -b $rp2mac \ + -B 192.0.2.1 -q & + mz_pid=$! + + sleep 5 + # In order to see this trap we need a route that points to disabled RIF. + # When ipv6 address is flushed, there is a delay and the routes are + # deleted before the RIF and we cannot get state that we have route + # to disabled RIF. + # Delete IPv6 address first and then check this trap with flushing IPv4. + ip -6 add flush dev br0 + ip -4 add flush dev br0 + + t1_packets=$(devlink_trap_rx_packets_get $trap_name) + t1_bytes=$(devlink_trap_rx_bytes_get $trap_name) + + if [[ $t0_packets -eq $t1_packets && $t0_bytes -eq $t1_bytes ]]; then + check_err 1 "Trap stats idle when packets should be trapped" + fi + + log_test "Egress RIF disabled" + + kill $mz_pid && wait $mz_pid &> /dev/null + ip link set dev $rp1 nomaster + __addr_add_del $rp1 add 192.0.2.2/24 2001:db8:1::2/64 + ip link del dev br0 type bridge + devlink_trap_action_set $trap_name "drop" +} + trap cleanup EXIT setup_prepare diff --git a/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_tunnel_ipip.sh b/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_tunnel_ipip.sh new file mode 100755 index 000000000000..039629bb92a3 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_tunnel_ipip.sh @@ -0,0 +1,265 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Test devlink-trap tunnel exceptions functionality over mlxsw. +# Check all exception traps to make sure they are triggered under the right +# conditions. + +# +-------------------------+ +# | H1 | +# | $h1 + | +# | 192.0.2.1/28 | | +# +-------------------|-----+ +# | +# +-------------------|-----+ +# | SW1 | | +# | $swp1 + | +# | 192.0.2.2/28 | +# | | +# | + g1a (gre) | +# | loc=192.0.2.65 | +# | rem=192.0.2.66 | +# | tos=inherit | +# | | +# | + $rp1 | +# | | 198.51.100.1/28 | +# +--|----------------------+ +# | +# +--|----------------------+ +# | | VRF2 | +# | + $rp2 | +# | 198.51.100.2/28 | +# +-------------------------+ + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + decap_error_test +" + +NUM_NETIFS=4 +source $lib_dir/lib.sh +source $lib_dir/tc_common.sh +source $lib_dir/devlink_lib.sh + +h1_create() +{ + simple_if_init $h1 192.0.2.1/28 +} + +h1_destroy() +{ + simple_if_fini $h1 192.0.2.1/28 +} + +vrf2_create() +{ + simple_if_init $rp2 198.51.100.2/28 +} + +vrf2_destroy() +{ + simple_if_fini $rp2 198.51.100.2/28 +} + +switch_create() +{ + __addr_add_del $swp1 add 192.0.2.2/28 + tc qdisc add dev $swp1 clsact + ip link set dev $swp1 up + + tunnel_create g1 gre 192.0.2.65 192.0.2.66 tos inherit + __addr_add_del g1 add 192.0.2.65/32 + ip link set dev g1 up + + __addr_add_del $rp1 add 198.51.100.1/28 + ip link set dev $rp1 up +} + +switch_destroy() +{ + ip link set dev $rp1 down + __addr_add_del $rp1 del 198.51.100.1/28 + + ip link set dev g1 down + __addr_add_del g1 del 192.0.2.65/32 + tunnel_destroy g1 + + ip link set dev $swp1 down + tc qdisc del dev $swp1 clsact + __addr_add_del $swp1 del 192.0.2.2/28 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + rp1=${NETIFS[p3]} + rp2=${NETIFS[p4]} + + forwarding_enable + vrf_prepare + h1_create + switch_create + vrf2_create +} + +cleanup() +{ + pre_cleanup + + vrf2_destroy + switch_destroy + h1_destroy + vrf_cleanup + forwarding_restore +} + +ecn_payload_get() +{ + p=$(: + )"0"$( : GRE flags + )"0:00:"$( : Reserved + version + )"08:00:"$( : ETH protocol type + )"4"$( : IP version + )"5:"$( : IHL + )"00:"$( : IP TOS + )"00:14:"$( : IP total length + )"00:00:"$( : IP identification + )"20:00:"$( : IP flags + frag off + )"30:"$( : IP TTL + )"01:"$( : IP proto + )"E7:E6:"$( : IP header csum + )"C0:00:01:01:"$( : IP saddr : 192.0.1.1 + )"C0:00:02:01:"$( : IP daddr : 192.0.2.1 + ) + echo $p +} + +ecn_decap_test() +{ + local trap_name="decap_error" + local group_name="tunnel_drops" + local desc=$1; shift + local ecn_desc=$1; shift + local outer_tos=$1; shift + local mz_pid + + RET=0 + + tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \ + flower src_ip 192.0.1.1 dst_ip 192.0.2.1 action pass + + rp1_mac=$(mac_get $rp1) + rp2_mac=$(mac_get $rp2) + payload=$(ecn_payload_get) + + ip vrf exec v$rp2 $MZ $rp2 -c 0 -d 1msec -a $rp2_mac -b $rp1_mac \ + -A 192.0.2.66 -B 192.0.2.65 -t ip \ + len=48,tos=$outer_tos,proto=47,p=$payload -q & + + mz_pid=$! + + devlink_trap_exception_test $trap_name $group_name + + tc_check_packets "dev $swp1 egress" 101 0 + check_err $? "Packets were not dropped" + + log_test "$desc: Inner ECN is not ECT and outer is $ecn_desc" + + kill $mz_pid && wait $mz_pid &> /dev/null + tc filter del dev $swp1 egress protocol ip pref 1 handle 101 flower +} + +ipip_payload_get() +{ + local flags=$1; shift + local key=$1; shift + + p=$(: + )"$flags"$( : GRE flags + )"0:00:"$( : Reserved + version + )"08:00:"$( : ETH protocol type + )"$key"$( : Key + )"4"$( : IP version + )"5:"$( : IHL + )"00:"$( : IP TOS + )"00:14:"$( : IP total length + )"00:00:"$( : IP identification + )"20:00:"$( : IP flags + frag off + )"30:"$( : IP TTL + )"01:"$( : IP proto + )"E7:E6:"$( : IP header csum + )"C0:00:01:01:"$( : IP saddr : 192.0.1.1 + )"C0:00:02:01:"$( : IP daddr : 192.0.2.1 + ) + echo $p +} + +no_matching_tunnel_test() +{ + local trap_name="decap_error" + local group_name="tunnel_drops" + local desc=$1; shift + local sip=$1; shift + local mz_pid + + RET=0 + + tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \ + flower src_ip 192.0.1.1 dst_ip 192.0.2.1 action pass + + rp1_mac=$(mac_get $rp1) + rp2_mac=$(mac_get $rp2) + payload=$(ipip_payload_get "$@") + + ip vrf exec v$rp2 $MZ $rp2 -c 0 -d 1msec -a $rp2_mac -b $rp1_mac \ + -A $sip -B 192.0.2.65 -t ip len=48,proto=47,p=$payload -q & + mz_pid=$! + + devlink_trap_exception_test $trap_name $group_name + + tc_check_packets "dev $swp1 egress" 101 0 + check_err $? "Packets were not dropped" + + log_test "$desc" + + kill $mz_pid && wait $mz_pid &> /dev/null + tc filter del dev $swp1 egress protocol ip pref 1 handle 101 flower +} + +decap_error_test() +{ + # Correct source IP - the remote address + local sip=192.0.2.66 + + ecn_decap_test "Decap error" "ECT(1)" 01 + ecn_decap_test "Decap error" "ECT(0)" 02 + ecn_decap_test "Decap error" "CE" 03 + + no_matching_tunnel_test "Decap error: Source IP check failed" \ + 192.0.2.68 "0" + no_matching_tunnel_test \ + "Decap error: Key exists but was not expected" $sip "2" ":E9:" + + # Destroy the tunnel and create new one with key + __addr_add_del g1 del 192.0.2.65/32 + tunnel_destroy g1 + + tunnel_create g1 gre 192.0.2.65 192.0.2.66 tos inherit key 233 + __addr_add_del g1 add 192.0.2.65/32 + + no_matching_tunnel_test \ + "Decap error: Key does not exist but was expected" $sip "0" + no_matching_tunnel_test \ + "Decap error: Packet has a wrong key field" $sip "2" "E8:" +} + +trap cleanup EXIT + +setup_prepare +setup_wait +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_tunnel_vxlan.sh b/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_tunnel_vxlan.sh new file mode 100755 index 000000000000..fd19161dd4ec --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/devlink_trap_tunnel_vxlan.sh @@ -0,0 +1,330 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Test devlink-trap tunnel drops and exceptions functionality over mlxsw. +# Check all traps to make sure they are triggered under the right +# conditions. + +# +--------------------+ +# | H1 (vrf) | +# | + $h1 | +# | | 192.0.2.1/28 | +# +----|---------------+ +# | +# +----|----------------------------------------------------------------------+ +# | SW | | +# | +--|--------------------------------------------------------------------+ | +# | | + $swp1 BR1 (802.1d) | | +# | | | | +# | | + vx1 (vxlan) | | +# | | local 192.0.2.17 | | +# | | id 1000 dstport $VXPORT | | +# | +-----------------------------------------------------------------------+ | +# | | +# | + $rp1 | +# | | 192.0.2.17/28 | +# +----|----------------------------------------------------------------------+ +# | +# +----|--------------------------------------------------------+ +# | | VRF2 | +# | + $rp2 | +# | 192.0.2.18/28 | +# | | +# +-------------------------------------------------------------+ + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + decap_error_test + overlay_smac_is_mc_test +" + +NUM_NETIFS=4 +source $lib_dir/lib.sh +source $lib_dir/tc_common.sh +source $lib_dir/devlink_lib.sh + +: ${VXPORT:=4789} +export VXPORT + +h1_create() +{ + simple_if_init $h1 192.0.2.1/28 +} + +h1_destroy() +{ + simple_if_fini $h1 192.0.2.1/28 +} + +switch_create() +{ + ip link add name br1 type bridge vlan_filtering 0 mcast_snooping 0 + # Make sure the bridge uses the MAC address of the local port and not + # that of the VxLAN's device. + ip link set dev br1 address $(mac_get $swp1) + ip link set dev br1 up + + tc qdisc add dev $swp1 clsact + ip link set dev $swp1 master br1 + ip link set dev $swp1 up + + ip link add name vx1 type vxlan id 1000 local 192.0.2.17 \ + dstport "$VXPORT" nolearning noudpcsum tos inherit ttl 100 + ip link set dev vx1 master br1 + ip link set dev vx1 up + + ip address add dev $rp1 192.0.2.17/28 + ip link set dev $rp1 up +} + +switch_destroy() +{ + ip link set dev $rp1 down + ip address del dev $rp1 192.0.2.17/28 + + ip link set dev vx1 down + ip link set dev vx1 nomaster + ip link del dev vx1 + + ip link set dev $swp1 down + ip link set dev $swp1 nomaster + tc qdisc del dev $swp1 clsact + + ip link set dev br1 down + ip link del dev br1 +} + +vrf2_create() +{ + simple_if_init $rp2 192.0.2.18/28 +} + +vrf2_destroy() +{ + simple_if_fini $rp2 192.0.2.18/28 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + rp1=${NETIFS[p3]} + rp2=${NETIFS[p4]} + + vrf_prepare + forwarding_enable + h1_create + switch_create + vrf2_create +} + +cleanup() +{ + pre_cleanup + + vrf2_destroy + switch_destroy + h1_destroy + forwarding_restore + vrf_cleanup +} + +ecn_payload_get() +{ + dest_mac=$(mac_get $h1) + p=$(: + )"08:"$( : VXLAN flags + )"00:00:00:"$( : VXLAN reserved + )"00:03:e8:"$( : VXLAN VNI : 1000 + )"00:"$( : VXLAN reserved + )"$dest_mac:"$( : ETH daddr + )"00:00:00:00:00:00:"$( : ETH saddr + )"08:00:"$( : ETH type + )"45:"$( : IP version + IHL + )"00:"$( : IP TOS + )"00:14:"$( : IP total length + )"00:00:"$( : IP identification + )"20:00:"$( : IP flags + frag off + )"40:"$( : IP TTL + )"00:"$( : IP proto + )"D6:E5:"$( : IP header csum + )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 + )"c0:00:02:01:"$( : IP daddr: 192.0.2.1 + ) + echo $p +} + +ecn_decap_test() +{ + local trap_name="decap_error" + local group_name="tunnel_drops" + local desc=$1; shift + local ecn_desc=$1; shift + local outer_tos=$1; shift + local mz_pid + + RET=0 + + tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \ + flower src_ip 192.0.2.3 dst_ip 192.0.2.1 action pass + + rp1_mac=$(mac_get $rp1) + payload=$(ecn_payload_get) + + ip vrf exec v$rp2 $MZ $rp2 -c 0 -d 1msec -b $rp1_mac -B 192.0.2.17 \ + -t udp sp=12345,dp=$VXPORT,tos=$outer_tos,p=$payload -q & + mz_pid=$! + + devlink_trap_exception_test $trap_name $group_name + + tc_check_packets "dev $swp1 egress" 101 0 + check_err $? "Packets were not dropped" + + log_test "$desc: Inner ECN is not ECT and outer is $ecn_desc" + + kill $mz_pid && wait $mz_pid &> /dev/null + tc filter del dev $swp1 egress protocol ip pref 1 handle 101 flower +} + +reserved_bits_payload_get() +{ + dest_mac=$(mac_get $h1) + p=$(: + )"08:"$( : VXLAN flags + )"01:00:00:"$( : VXLAN reserved + )"00:03:e8:"$( : VXLAN VNI : 1000 + )"00:"$( : VXLAN reserved + )"$dest_mac:"$( : ETH daddr + )"00:00:00:00:00:00:"$( : ETH saddr + )"08:00:"$( : ETH type + )"45:"$( : IP version + IHL + )"00:"$( : IP TOS + )"00:14:"$( : IP total length + )"00:00:"$( : IP identification + )"20:00:"$( : IP flags + frag off + )"40:"$( : IP TTL + )"00:"$( : IP proto + )"00:00:"$( : IP header csum + )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 + )"c0:00:02:01:"$( : IP daddr: 192.0.2.1 + ) + echo $p +} + +short_payload_get() +{ + dest_mac=$(mac_get $h1) + p=$(: + )"08:"$( : VXLAN flags + )"01:00:00:"$( : VXLAN reserved + )"00:03:e8:"$( : VXLAN VNI : 1000 + )"00:"$( : VXLAN reserved + ) + echo $p +} + +corrupted_packet_test() +{ + local trap_name="decap_error" + local group_name="tunnel_drops" + local desc=$1; shift + local payload_get=$1; shift + local mz_pid + + RET=0 + + # In case of too short packet, there is no any inner packet, + # so the matching will always succeed + tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \ + flower skip_hw src_ip 192.0.2.3 dst_ip 192.0.2.1 action pass + + rp1_mac=$(mac_get $rp1) + payload=$($payload_get) + ip vrf exec v$rp2 $MZ $rp2 -c 0 -d 1msec -b $rp1_mac \ + -B 192.0.2.17 -t udp sp=12345,dp=$VXPORT,p=$payload -q & + mz_pid=$! + + devlink_trap_exception_test $trap_name $group_name + + tc_check_packets "dev $swp1 egress" 101 0 + check_err $? "Packets were not dropped" + + log_test "$desc" + + kill $mz_pid && wait $mz_pid &> /dev/null + tc filter del dev $swp1 egress protocol ip pref 1 handle 101 flower +} + +decap_error_test() +{ + ecn_decap_test "Decap error" "ECT(1)" 01 + ecn_decap_test "Decap error" "ECT(0)" 02 + ecn_decap_test "Decap error" "CE" 03 + + corrupted_packet_test "Decap error: Reserved bits in use" \ + "reserved_bits_payload_get" + corrupted_packet_test "Decap error: No L2 header" "short_payload_get" +} + +mc_smac_payload_get() +{ + dest_mac=$(mac_get $h1) + source_mac=01:02:03:04:05:06 + p=$(: + )"08:"$( : VXLAN flags + )"00:00:00:"$( : VXLAN reserved + )"00:03:e8:"$( : VXLAN VNI : 1000 + )"00:"$( : VXLAN reserved + )"$dest_mac:"$( : ETH daddr + )"$source_mac:"$( : ETH saddr + )"08:00:"$( : ETH type + )"45:"$( : IP version + IHL + )"00:"$( : IP TOS + )"00:14:"$( : IP total length + )"00:00:"$( : IP identification + )"20:00:"$( : IP flags + frag off + )"40:"$( : IP TTL + )"00:"$( : IP proto + )"00:00:"$( : IP header csum + )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 + )"c0:00:02:01:"$( : IP daddr: 192.0.2.1 + ) + echo $p +} + +overlay_smac_is_mc_test() +{ + local trap_name="overlay_smac_is_mc" + local group_name="tunnel_drops" + local mz_pid + + RET=0 + + # The matching will be checked on devlink_trap_drop_test() + # and the filter will be removed on devlink_trap_drop_cleanup() + tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \ + flower src_mac 01:02:03:04:05:06 action pass + + rp1_mac=$(mac_get $rp1) + payload=$(mc_smac_payload_get) + + ip vrf exec v$rp2 $MZ $rp2 -c 0 -d 1msec -b $rp1_mac \ + -B 192.0.2.17 -t udp sp=12345,dp=$VXPORT,p=$payload -q & + mz_pid=$! + + devlink_trap_drop_test $trap_name $group_name $swp1 + + log_test "Overlay source MAC is multicast" + + devlink_trap_drop_cleanup $mz_pid $swp1 "ip" +} + +trap cleanup EXIT + +setup_prepare +setup_wait +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/fib.sh b/tools/testing/selftests/drivers/net/mlxsw/fib.sh new file mode 100755 index 000000000000..45115f81c2b1 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/fib.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# This test is for checking the FIB offload API on top of mlxsw. + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + ipv4_identical_routes + ipv4_tos + ipv4_metric + ipv4_replace + ipv4_delete + ipv4_plen + ipv4_replay + ipv4_flush + ipv6_add + ipv6_metric + ipv6_append_single + ipv6_replace_single + ipv6_metric_multipath + ipv6_append_multipath + ipv6_replace_multipath + ipv6_append_multipath_to_single + ipv6_delete_single + ipv6_delete_multipath + ipv6_replay_single + ipv6_replay_multipath +" +NUM_NETIFS=0 +source $lib_dir/lib.sh +source $lib_dir/devlink_lib.sh +source $lib_dir/fib_offload_lib.sh + +ipv4_identical_routes() +{ + fib_ipv4_identical_routes_test "testns1" +} + +ipv4_tos() +{ + fib_ipv4_tos_test "testns1" +} + +ipv4_metric() +{ + fib_ipv4_metric_test "testns1" +} + +ipv4_replace() +{ + fib_ipv4_replace_test "testns1" +} + +ipv4_delete() +{ + fib_ipv4_delete_test "testns1" +} + +ipv4_plen() +{ + fib_ipv4_plen_test "testns1" +} + +ipv4_replay_metric() +{ + fib_ipv4_replay_metric_test "testns1" "$DEVLINK_DEV" +} + +ipv4_replay_tos() +{ + fib_ipv4_replay_tos_test "testns1" "$DEVLINK_DEV" +} + +ipv4_replay_plen() +{ + fib_ipv4_replay_plen_test "testns1" "$DEVLINK_DEV" +} + +ipv4_replay() +{ + ipv4_replay_metric + ipv4_replay_tos + ipv4_replay_plen +} + +ipv4_flush() +{ + fib_ipv4_flush_test "testns1" +} + +ipv6_add() +{ + fib_ipv6_add_test "testns1" +} + +ipv6_metric() +{ + fib_ipv6_metric_test "testns1" +} + +ipv6_append_single() +{ + fib_ipv6_append_single_test "testns1" +} + +ipv6_replace_single() +{ + fib_ipv6_replace_single_test "testns1" +} + +ipv6_metric_multipath() +{ + fib_ipv6_metric_multipath_test "testns1" +} + +ipv6_append_multipath() +{ + fib_ipv6_append_multipath_test "testns1" +} + +ipv6_replace_multipath() +{ + fib_ipv6_replace_multipath_test "testns1" +} + +ipv6_append_multipath_to_single() +{ + fib_ipv6_append_multipath_to_single_test "testns1" +} + +ipv6_delete_single() +{ + fib_ipv6_delete_single_test "testns1" +} + +ipv6_delete_multipath() +{ + fib_ipv6_delete_multipath_test "testns1" +} + +ipv6_replay_single() +{ + fib_ipv6_replay_single_test "testns1" "$DEVLINK_DEV" +} + +ipv6_replay_multipath() +{ + fib_ipv6_replay_multipath_test "testns1" "$DEVLINK_DEV" +} + +setup_prepare() +{ + ip netns add testns1 + if [ $? -ne 0 ]; then + echo "Failed to add netns \"testns1\"" + exit 1 + fi + + devlink dev reload $DEVLINK_DEV netns testns1 + if [ $? -ne 0 ]; then + echo "Failed to reload into netns \"testns1\"" + exit 1 + fi +} + +cleanup() +{ + pre_cleanup + devlink -N testns1 dev reload $DEVLINK_DEV netns $$ + ip netns del testns1 +} + +trap cleanup EXIT + +setup_prepare + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_defprio.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_defprio.sh new file mode 100755 index 000000000000..eff6393ce974 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_defprio.sh @@ -0,0 +1,176 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# Test for port-default priority. Non-IP packets ingress $swp1 and are +# prioritized according to the default priority specified at the port. +# rx_octets_prio_* counters are used to verify the prioritization. +# +# +-----------------------+ +# | H1 | +# | + $h1 | +# | | 192.0.2.1/28 | +# +----|------------------+ +# | +# +----|------------------+ +# | SW | | +# | + $swp1 | +# | 192.0.2.2/28 | +# | APP=<prio>,1,0 | +# +-----------------------+ + +ALL_TESTS=" + ping_ipv4 + test_defprio +" + +lib_dir=$(dirname $0)/../../../net/forwarding + +NUM_NETIFS=2 +: ${HIT_TIMEOUT:=1000} # ms +source $lib_dir/lib.sh + +declare -a APP + +defprio_install() +{ + local dev=$1; shift + local prio=$1; shift + local app="app=$prio,1,0" + + lldptool -T -i $dev -V APP $app >/dev/null + lldpad_app_wait_set $dev + APP[$prio]=$app +} + +defprio_uninstall() +{ + local dev=$1; shift + local prio=$1; shift + local app=${APP[$prio]} + + lldptool -T -i $dev -V APP -d $app >/dev/null + lldpad_app_wait_del + unset APP[$prio] +} + +defprio_flush() +{ + local dev=$1; shift + local prio + + if ((${#APP[@]})); then + lldptool -T -i $dev -V APP -d ${APP[@]} >/dev/null + fi + lldpad_app_wait_del + APP=() +} + +h1_create() +{ + simple_if_init $h1 192.0.2.1/28 +} + +h1_destroy() +{ + simple_if_fini $h1 192.0.2.1/28 +} + +switch_create() +{ + ip link set dev $swp1 up + ip addr add dev $swp1 192.0.2.2/28 +} + +switch_destroy() +{ + defprio_flush $swp1 + ip addr del dev $swp1 192.0.2.2/28 + ip link set dev $swp1 down +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + vrf_prepare + + h1_create + switch_create +} + +cleanup() +{ + pre_cleanup + + switch_destroy + h1_destroy + + vrf_cleanup +} + +ping_ipv4() +{ + ping_test $h1 192.0.2.2 +} + +wait_for_packets() +{ + local t0=$1; shift + local prio_observe=$1; shift + + local t1=$(ethtool_stats_get $swp1 rx_frames_prio_$prio_observe) + local delta=$((t1 - t0)) + echo $delta + ((delta >= 10)) +} + +__test_defprio() +{ + local prio_install=$1; shift + local prio_observe=$1; shift + local delta + local key + local i + + RET=0 + + defprio_install $swp1 $prio_install + + local t0=$(ethtool_stats_get $swp1 rx_frames_prio_$prio_observe) + mausezahn -q $h1 -d 100m -c 10 -t arp reply + delta=$(busywait "$HIT_TIMEOUT" wait_for_packets $t0 $prio_observe) + + check_err $? "Default priority $prio_install/$prio_observe: Expected to capture 10 packets, got $delta." + log_test "Default priority $prio_install/$prio_observe" + + defprio_uninstall $swp1 $prio_install +} + +test_defprio() +{ + local prio + + for prio in {0..7}; do + __test_defprio $prio $prio + done + + defprio_install $swp1 3 + __test_defprio 0 3 + __test_defprio 1 3 + __test_defprio 2 3 + __test_defprio 4 4 + __test_defprio 5 5 + __test_defprio 6 6 + __test_defprio 7 7 + defprio_uninstall $swp1 3 +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_lib.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_lib.sh index e80be65799ad..faa51012cdac 100644 --- a/tools/testing/selftests/drivers/net/mlxsw/qos_lib.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_lib.sh @@ -1,47 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -humanize() -{ - local speed=$1; shift - - for unit in bps Kbps Mbps Gbps; do - if (($(echo "$speed < 1024" | bc))); then - break - fi - - speed=$(echo "scale=1; $speed / 1024" | bc) - done - - echo "$speed${unit}" -} - -rate() -{ - local t0=$1; shift - local t1=$1; shift - local interval=$1; shift - - echo $((8 * (t1 - t0) / interval)) -} - -start_traffic() -{ - local h_in=$1; shift # Where the traffic egresses the host - local sip=$1; shift - local dip=$1; shift - local dmac=$1; shift - - $MZ $h_in -p 8000 -A $sip -B $dip -c 0 \ - -a own -b $dmac -t udp -q & - sleep 1 -} - -stop_traffic() -{ - # Suppress noise from killing mausezahn. - { kill %% && wait %%; } 2>/dev/null -} - check_rate() { local rate=$1; shift @@ -96,3 +54,31 @@ measure_rate() echo $ir $er return $ret } + +bail_on_lldpad() +{ + if systemctl is-active --quiet lldpad; then + + cat >/dev/stderr <<-EOF + WARNING: lldpad is running + + lldpad will likely configure DCB, and this test will + configure Qdiscs. mlxsw does not support both at the + same time, one of them is arbitrarily going to overwrite + the other. That will cause spurious failures (or, + unlikely, passes) of this test. + EOF + + if [[ -z $ALLOW_LLDPAD ]]; then + cat >/dev/stderr <<-EOF + + If you want to run the test anyway, please set + an environment variable ALLOW_LLDPAD to a + non-empty string. + EOF + exit 1 + else + return + fi + fi +} diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh new file mode 100755 index 000000000000..c9fc4d4885c1 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# A driver for the ETS selftest that implements testing in offloaded datapath. +lib_dir=$(dirname $0)/../../../net/forwarding +source $lib_dir/sch_ets_core.sh +source $lib_dir/devlink_lib.sh +source qos_lib.sh + +ALL_TESTS=" + ping_ipv4 + priomap_mode + ets_test_strict + ets_test_mixed + ets_test_dwrr +" + +switch_create() +{ + ets_switch_create + + # Create a bottleneck so that the DWRR process can kick in. + ethtool -s $h2 speed 1000 autoneg off + ethtool -s $swp2 speed 1000 autoneg off + + # Set the ingress quota high and use the three egress TCs to limit the + # amount of traffic that is admitted to the shared buffers. This makes + # sure that there is always enough traffic of all types to select from + # for the DWRR process. + devlink_port_pool_th_set $swp1 0 12 + devlink_tc_bind_pool_th_set $swp1 0 ingress 0 12 + devlink_port_pool_th_set $swp2 4 12 + devlink_tc_bind_pool_th_set $swp2 7 egress 4 5 + devlink_tc_bind_pool_th_set $swp2 6 egress 4 5 + devlink_tc_bind_pool_th_set $swp2 5 egress 4 5 + + # Note: sch_ets_core.sh uses VLAN ingress-qos-map to assign packet + # priorities at $swp1 based on their 802.1p headers. ingress-qos-map is + # not offloaded by mlxsw as of this writing, but the mapping used is + # 1:1, which is the mapping currently hard-coded by the driver. +} + +switch_destroy() +{ + devlink_tc_bind_pool_th_restore $swp2 5 egress + devlink_tc_bind_pool_th_restore $swp2 6 egress + devlink_tc_bind_pool_th_restore $swp2 7 egress + devlink_port_pool_th_restore $swp2 4 + devlink_tc_bind_pool_th_restore $swp1 0 ingress + devlink_port_pool_th_restore $swp1 0 + + ethtool -s $swp2 autoneg on + ethtool -s $h2 autoneg on + + ets_switch_destroy +} + +# Callback from sch_ets_tests.sh +get_stats() +{ + local band=$1; shift + + ethtool_stats_get "$h2" rx_octets_prio_$band +} + +bail_on_lldpad +ets_run diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_tbf_ets.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_tbf_ets.sh new file mode 100755 index 000000000000..c6ce0b448bf3 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_tbf_ets.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source qos_lib.sh +bail_on_lldpad + +lib_dir=$(dirname $0)/../../../net/forwarding +TCFLAGS=skip_sw +source $lib_dir/sch_tbf_ets.sh diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_tbf_prio.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_tbf_prio.sh new file mode 100755 index 000000000000..8d245f331619 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_tbf_prio.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source qos_lib.sh +bail_on_lldpad + +lib_dir=$(dirname $0)/../../../net/forwarding +TCFLAGS=skip_sw +source $lib_dir/sch_tbf_prio.sh diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_tbf_root.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_tbf_root.sh new file mode 100755 index 000000000000..013886061f15 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_tbf_root.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source qos_lib.sh +bail_on_lldpad + +lib_dir=$(dirname $0)/../../../net/forwarding +TCFLAGS=skip_sw +source $lib_dir/sch_tbf_root.sh diff --git a/tools/testing/selftests/drivers/net/netdevsim/fib.sh b/tools/testing/selftests/drivers/net/netdevsim/fib.sh new file mode 100755 index 000000000000..2f87c3be76a9 --- /dev/null +++ b/tools/testing/selftests/drivers/net/netdevsim/fib.sh @@ -0,0 +1,341 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# This test is for checking the FIB offload API. It makes use of netdevsim +# which registers a listener to the FIB notification chain. + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + ipv4_identical_routes + ipv4_tos + ipv4_metric + ipv4_replace + ipv4_delete + ipv4_plen + ipv4_replay + ipv4_flush + ipv4_error_path + ipv6_add + ipv6_metric + ipv6_append_single + ipv6_replace_single + ipv6_metric_multipath + ipv6_append_multipath + ipv6_replace_multipath + ipv6_append_multipath_to_single + ipv6_delete_single + ipv6_delete_multipath + ipv6_replay_single + ipv6_replay_multipath + ipv6_error_path +" +NETDEVSIM_PATH=/sys/bus/netdevsim/ +DEV_ADDR=1337 +DEV=netdevsim${DEV_ADDR} +DEVLINK_DEV=netdevsim/${DEV} +SYSFS_NET_DIR=/sys/bus/netdevsim/devices/$DEV/net/ +NUM_NETIFS=0 +source $lib_dir/lib.sh +source $lib_dir/devlink_lib.sh +source $lib_dir/fib_offload_lib.sh + +ipv4_identical_routes() +{ + fib_ipv4_identical_routes_test "testns1" +} + +ipv4_tos() +{ + fib_ipv4_tos_test "testns1" +} + +ipv4_metric() +{ + fib_ipv4_metric_test "testns1" +} + +ipv4_replace() +{ + fib_ipv4_replace_test "testns1" +} + +ipv4_delete() +{ + fib_ipv4_delete_test "testns1" +} + +ipv4_plen() +{ + fib_ipv4_plen_test "testns1" +} + +ipv4_replay_metric() +{ + fib_ipv4_replay_metric_test "testns1" "$DEVLINK_DEV" +} + +ipv4_replay_tos() +{ + fib_ipv4_replay_tos_test "testns1" "$DEVLINK_DEV" +} + +ipv4_replay_plen() +{ + fib_ipv4_replay_plen_test "testns1" "$DEVLINK_DEV" +} + +ipv4_replay() +{ + ipv4_replay_metric + ipv4_replay_tos + ipv4_replay_plen +} + +ipv4_flush() +{ + fib_ipv4_flush_test "testns1" +} + +ipv4_error_path_add() +{ + local lsb + + RET=0 + + ip -n testns1 link add name dummy1 type dummy + ip -n testns1 link set dev dummy1 up + + devlink -N testns1 resource set $DEVLINK_DEV path IPv4/fib size 10 + devlink -N testns1 dev reload $DEVLINK_DEV + + for lsb in $(seq 1 20); do + ip -n testns1 route add 192.0.2.${lsb}/32 dev dummy1 \ + &> /dev/null + done + + log_test "IPv4 error path - add" + + ip -n testns1 link del dev dummy1 +} + +ipv4_error_path_replay() +{ + local lsb + + RET=0 + + ip -n testns1 link add name dummy1 type dummy + ip -n testns1 link set dev dummy1 up + + devlink -N testns1 resource set $DEVLINK_DEV path IPv4/fib size 100 + devlink -N testns1 dev reload $DEVLINK_DEV + + for lsb in $(seq 1 20); do + ip -n testns1 route add 192.0.2.${lsb}/32 dev dummy1 + done + + devlink -N testns1 resource set $DEVLINK_DEV path IPv4/fib size 10 + devlink -N testns1 dev reload $DEVLINK_DEV &> /dev/null + + log_test "IPv4 error path - replay" + + ip -n testns1 link del dev dummy1 + + # Successfully reload after deleting all the routes. + devlink -N testns1 resource set $DEVLINK_DEV path IPv4/fib size 100 + devlink -N testns1 dev reload $DEVLINK_DEV +} + +ipv4_error_path() +{ + # Test the different error paths of the notifiers by limiting the size + # of the "IPv4/fib" resource. + ipv4_error_path_add + ipv4_error_path_replay +} + +ipv6_add() +{ + fib_ipv6_add_test "testns1" +} + +ipv6_metric() +{ + fib_ipv6_metric_test "testns1" +} + +ipv6_append_single() +{ + fib_ipv6_append_single_test "testns1" +} + +ipv6_replace_single() +{ + fib_ipv6_replace_single_test "testns1" +} + +ipv6_metric_multipath() +{ + fib_ipv6_metric_multipath_test "testns1" +} + +ipv6_append_multipath() +{ + fib_ipv6_append_multipath_test "testns1" +} + +ipv6_replace_multipath() +{ + fib_ipv6_replace_multipath_test "testns1" +} + +ipv6_append_multipath_to_single() +{ + fib_ipv6_append_multipath_to_single_test "testns1" +} + +ipv6_delete_single() +{ + fib_ipv6_delete_single_test "testns1" +} + +ipv6_delete_multipath() +{ + fib_ipv6_delete_multipath_test "testns1" +} + +ipv6_replay_single() +{ + fib_ipv6_replay_single_test "testns1" "$DEVLINK_DEV" +} + +ipv6_replay_multipath() +{ + fib_ipv6_replay_multipath_test "testns1" "$DEVLINK_DEV" +} + +ipv6_error_path_add_single() +{ + local lsb + + RET=0 + + ip -n testns1 link add name dummy1 type dummy + ip -n testns1 link set dev dummy1 up + + devlink -N testns1 resource set $DEVLINK_DEV path IPv6/fib size 10 + devlink -N testns1 dev reload $DEVLINK_DEV + + for lsb in $(seq 1 20); do + ip -n testns1 route add 2001:db8:1::${lsb}/128 dev dummy1 \ + &> /dev/null + done + + log_test "IPv6 error path - add single" + + ip -n testns1 link del dev dummy1 +} + +ipv6_error_path_add_multipath() +{ + local lsb + + RET=0 + + for i in $(seq 1 2); do + ip -n testns1 link add name dummy$i type dummy + ip -n testns1 link set dev dummy$i up + ip -n testns1 address add 2001:db8:$i::1/64 dev dummy$i + done + + devlink -N testns1 resource set $DEVLINK_DEV path IPv6/fib size 10 + devlink -N testns1 dev reload $DEVLINK_DEV + + for lsb in $(seq 1 20); do + ip -n testns1 route add 2001:db8:10::${lsb}/128 \ + nexthop via 2001:db8:1::2 dev dummy1 \ + nexthop via 2001:db8:2::2 dev dummy2 &> /dev/null + done + + log_test "IPv6 error path - add multipath" + + for i in $(seq 1 2); do + ip -n testns1 link del dev dummy$i + done +} + +ipv6_error_path_replay() +{ + local lsb + + RET=0 + + ip -n testns1 link add name dummy1 type dummy + ip -n testns1 link set dev dummy1 up + + devlink -N testns1 resource set $DEVLINK_DEV path IPv6/fib size 100 + devlink -N testns1 dev reload $DEVLINK_DEV + + for lsb in $(seq 1 20); do + ip -n testns1 route add 2001:db8:1::${lsb}/128 dev dummy1 + done + + devlink -N testns1 resource set $DEVLINK_DEV path IPv6/fib size 10 + devlink -N testns1 dev reload $DEVLINK_DEV &> /dev/null + + log_test "IPv6 error path - replay" + + ip -n testns1 link del dev dummy1 + + # Successfully reload after deleting all the routes. + devlink -N testns1 resource set $DEVLINK_DEV path IPv6/fib size 100 + devlink -N testns1 dev reload $DEVLINK_DEV +} + +ipv6_error_path() +{ + # Test the different error paths of the notifiers by limiting the size + # of the "IPv6/fib" resource. + ipv6_error_path_add_single + ipv6_error_path_add_multipath + ipv6_error_path_replay +} + +setup_prepare() +{ + local netdev + + modprobe netdevsim &> /dev/null + + echo "$DEV_ADDR 1" > ${NETDEVSIM_PATH}/new_device + while [ ! -d $SYSFS_NET_DIR ] ; do :; done + + ip netns add testns1 + if [ $? -ne 0 ]; then + echo "Failed to add netns \"testns1\"" + exit 1 + fi + + devlink dev reload $DEVLINK_DEV netns testns1 + if [ $? -ne 0 ]; then + echo "Failed to reload into netns \"testns1\"" + exit 1 + fi +} + +cleanup() +{ + pre_cleanup + ip netns del testns1 + echo "$DEV_ADDR" > ${NETDEVSIM_PATH}/del_device + modprobe -r netdevsim &> /dev/null +} + +trap cleanup EXIT + +setup_prepare + +tests_run + +exit $EXIT_STATUS |