Merge branch 'netdevsim-implement-proper-device-model'

Jiri Pirko says:

====================
netdevsim: implement proper device model

Currently the model of netdevsim is a bit odd in multiple ways.
1) devlink instance is not in any way related with actual netdevsim
   netdevices. Instead, it is created per-namespace.
2) multi-port netdevsim device is done using "link" attribute.
3) netdevsim bus is there only to have something to bind the netdev to,
   it really does not act as a bus.
4) netdevsim instances are created by "ip link add" which is great for
   soft devices with no hw backend. The rtnl core allocates netdev and
   calls into driver holding rtnl mutex. For hw-backed devices, this
   flow is wrong as it breaks order in which things are done.

This patchset adjust netdevsim to fix all above.

In order to support proper devlink and devlink port instances and to be
able to emulate real devices, there is need to implement bus probe and
instantiate everything from there. User can specify device id and port
count to be instantianted. For example:

$ echo "10 4" > /sys/bus/netdevsim/new_device

Then devlink shows this:

$ devlink dev
netdevsim/netdevsim10

$ devlink port
netdevsim/netdevsim10/0: type eth netdev eni0np1 flavour physical
netdevsim/netdevsim10/1: type eth netdev eni0np2 flavour physical
netdevsim/netdevsim10/2: type eth netdev eni0np3 flavour physical
netdevsim/netdevsim10/3: type eth netdev eni0np4 flavour physical

There is possible to add and delete ports using their indexes
during netdevsim device lifetime like this:

Then devlink shows this:

$ devlink port
netdevsim/netdevsim10/1: type eth netdev eni10np2 flavour physical
netdevsim/netdevsim10/2: type eth netdev eni10np3 flavour physical
netdevsim/netdevsim10/3: type eth netdev eni10np4 flavour physical
netdevsim/netdevsim10/43: type eth netdev eni10np44 flavour physical

Debugfs topology is also adjusted a bit. The rest stays the same as
before.

Udev bits are merged un systemd upstream git:
https://github.com/systemd/systemd/commit/eaa9d507d85509c8bf727356e3884ec54b0fc646

See individual patches for changelog.
====================

Acked-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

2 files changed, 157 insertions, 81 deletions

diff --git a/tools/testing/selftests/bpf/test_offload.py b/tools/testing/selftests/bpf/test_offload.py
index a7f95106119f..425f9ed27c3b 100755
--- a/tools/testing/selftests/bpf/test_offload.py
+++ b/tools/testing/selftests/bpf/test_offload.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 # Copyright (C) 2017 Netronome Systems, Inc.
+# Copyright (c) 2019 Mellanox Technologies. All rights reserved
 #
 # This software is licensed under the GNU General License Version 2,
 # June 1991 as shown in the file COPYING in the top-level directory of this
@@ -15,10 +16,12 @@
 
 from datetime import datetime
 import argparse
+import errno
 import json
 import os
 import pprint
 import random
+import re
 import string
 import struct
 import subprocess
@@ -306,6 +309,8 @@ class DebugfsDir:
 
         _, out = cmd('ls ' + path)
         for f in out.split():
+            if f == "ports":
+                continue
             p = os.path.join(path, f)
             if os.path.isfile(p):
                 _, out = cmd('cat %s/%s' % (path, f))
@@ -321,42 +326,112 @@ class DebugfsDir:
 
         return dfs
 
-class NetdevSim:
+class NetdevSimDev:
     """
-    Class for netdevsim netdevice and its attributes.
+    Class for netdevsim bus device and its attributes.
     """
 
-    def __init__(self, link=None):
-        self.link = link
+    def __init__(self, port_count=1):
+        addr = 0
+        while True:
+            try:
+                with open("/sys/bus/netdevsim/new_device", "w") as f:
+                    f.write("%u %u" % (addr, port_count))
+            except OSError as e:
+                if e.errno == errno.ENOSPC:
+                    addr += 1
+                    continue
+                raise e
+            break
+        self.addr = addr
+
+        # As probe of netdevsim device might happen from a workqueue,
+        # so wait here until all netdevs appear.
+        self.wait_for_netdevs(port_count)
+
+        ret, out = cmd("udevadm settle", fail=False)
+        if ret:
+            raise Exception("udevadm settle failed")
+        ifnames = self.get_ifnames()
 
-        self.dev = self._netdevsim_create()
         devs.append(self)
+        self.dfs_dir = "/sys/kernel/debug/netdevsim/netdevsim%u/" % addr
+
+        self.nsims = []
+        for port_index in range(port_count):
+            self.nsims.append(NetdevSim(self, port_index, ifnames[port_index]))
+
+    def get_ifnames(self):
+        ifnames = []
+        listdir = os.listdir("/sys/bus/netdevsim/devices/netdevsim%u/net/" % self.addr)
+        for ifname in listdir:
+            ifnames.append(ifname)
+        ifnames.sort()
+        return ifnames
+
+    def wait_for_netdevs(self, port_count):
+        timeout = 5
+        timeout_start = time.time()
+
+        while True:
+            try:
+                ifnames = self.get_ifnames()
+            except FileNotFoundError as e:
+                ifnames = []
+            if len(ifnames) == port_count:
+                break
+            if time.time() < timeout_start + timeout:
+                continue
+            raise Exception("netdevices did not appear within timeout")
 
-        self.ns = ""
+    def dfs_num_bound_progs(self):
+        path = os.path.join(self.dfs_dir, "bpf_bound_progs")
+        _, progs = cmd('ls %s' % (path))
+        return len(progs.split())
 
-        self.dfs_dir = '/sys/kernel/debug/netdevsim/%s' % (self.dev['ifname'])
-        self.sdev_dir = self.dfs_dir + '/sdev/'
-        self.dfs_refresh()
+    def dfs_get_bound_progs(self, expected):
+        progs = DebugfsDir(os.path.join(self.dfs_dir, "bpf_bound_progs"))
+        if expected is not None:
+            if len(progs) != expected:
+                fail(True, "%d BPF programs bound, expected %d" %
+                     (len(progs), expected))
+        return progs
 
-    def __getitem__(self, key):
-        return self.dev[key]
+    def remove(self):
+        with open("/sys/bus/netdevsim/del_device", "w") as f:
+            f.write("%u" % self.addr)
+        devs.remove(self)
 
-    def _netdevsim_create(self):
-        link = "" if self.link is None else "link " + self.link.dev['ifname']
-        _, old  = ip("link show")
-        ip("link add sim%d {link} type netdevsim".format(link=link))
-        _, new  = ip("link show")
+    def remove_nsim(self, nsim):
+        self.nsims.remove(nsim)
+        with open("/sys/bus/netdevsim/devices/netdevsim%u/del_port" % self.addr ,"w") as f:
+            f.write("%u" % nsim.port_index)
 
-        for dev in new:
-            f = filter(lambda x: x["ifname"] == dev["ifname"], old)
-            if len(list(f)) == 0:
-                return dev
+class NetdevSim:
+    """
+    Class for netdevsim netdevice and its attributes.
+    """
+
+    def __init__(self, nsimdev, port_index, ifname):
+        # In case udev renamed the netdev to according to new schema,
+        # check if the name matches the port_index.
+        nsimnamere = re.compile("eni\d+np(\d+)")
+        match = nsimnamere.match(ifname)
+        if match and int(match.groups()[0]) != port_index + 1:
+            raise Exception("netdevice name mismatches the expected one")
+
+        self.nsimdev = nsimdev
+        self.port_index = port_index
+        self.ns = ""
+        self.dfs_dir = "%s/ports/%u/" % (nsimdev.dfs_dir, port_index)
+        self.dfs_refresh()
+        _, [self.dev] = ip("link show dev %s" % ifname)
 
-        raise Exception("failed to create netdevsim device")
+    def __getitem__(self, key):
+        return self.dev[key]
 
     def remove(self):
-        devs.remove(self)
-        ip("link del dev %s" % (self.dev["ifname"]), ns=self.ns)
+        self.nsimdev.remove_nsim(self)
 
     def dfs_refresh(self):
         self.dfs = DebugfsDir(self.dfs_dir)
@@ -367,22 +442,9 @@ class NetdevSim:
         _, data = cmd('cat %s' % (path))
         return data.strip()
 
-    def dfs_num_bound_progs(self):
-        path = os.path.join(self.sdev_dir, "bpf_bound_progs")
-        _, progs = cmd('ls %s' % (path))
-        return len(progs.split())
-
-    def dfs_get_bound_progs(self, expected):
-        progs = DebugfsDir(os.path.join(self.sdev_dir, "bpf_bound_progs"))
-        if expected is not None:
-            if len(progs) != expected:
-                fail(True, "%d BPF programs bound, expected %d" %
-                     (len(progs), expected))
-        return progs
-
     def wait_for_flush(self, bound=0, total=0, n_retry=20):
         for i in range(n_retry):
-            nbound = self.dfs_num_bound_progs()
+            nbound = self.nsimdev.dfs_num_bound_progs()
             nprogs = len(bpftool_prog_list())
             if nbound == bound and nprogs == total:
                 return
@@ -612,7 +674,7 @@ def test_spurios_extack(sim, obj, skip_hw, needle):
                             include_stderr=True)
     check_no_extack(res, needle)
 
-def test_multi_prog(sim, obj, modename, modeid):
+def test_multi_prog(simdev, sim, obj, modename, modeid):
     start_test("Test multi-attachment XDP - %s + offload..." %
                (modename or "default", ))
     sim.set_xdp(obj, "offload")
@@ -668,11 +730,12 @@ def test_multi_prog(sim, obj, modename, modeid):
     check_multi_basic(two_xdps)
 
     start_test("Test multi-attachment XDP - device remove...")
-    sim.remove()
+    simdev.remove()
 
-    sim = NetdevSim()
+    simdev = NetdevSimDev()
+    sim, = simdev.nsims
     sim.set_ethtool_tc_offloads(True)
-    return sim
+    return [simdev, sim]
 
 # Parse command line
 parser = argparse.ArgumentParser()
@@ -729,12 +792,14 @@ try:
     bytecode = bpf_bytecode("1,6 0 0 4294967295,")
 
     start_test("Test destruction of generic XDP...")
-    sim = NetdevSim()
+    simdev = NetdevSimDev()
+    sim, = simdev.nsims
     sim.set_xdp(obj, "generic")
-    sim.remove()
+    simdev.remove()
     bpftool_prog_list_wait(expected=0)
 
-    sim = NetdevSim()
+    simdev = NetdevSimDev()
+    sim, = simdev.nsims
     sim.tc_add_ingress()
 
     start_test("Test TC non-offloaded...")
@@ -744,7 +809,7 @@ try:
     start_test("Test TC non-offloaded isn't getting bound...")
     ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
     fail(ret != 0, "Software TC filter did not load")
-    sim.dfs_get_bound_progs(expected=0)
+    simdev.dfs_get_bound_progs(expected=0)
 
     sim.tc_flush_filters()
 
@@ -761,7 +826,7 @@ try:
     start_test("Test TC offload by default...")
     ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
     fail(ret != 0, "Software TC filter did not load")
-    sim.dfs_get_bound_progs(expected=0)
+    simdev.dfs_get_bound_progs(expected=0)
     ingress = sim.tc_show_ingress(expected=1)
     fltr = ingress[0]
     fail(not fltr["in_hw"], "Filter not offloaded by default")
@@ -771,7 +836,7 @@ try:
     start_test("Test TC cBPF bytcode tries offload by default...")
     ret, _ = sim.cls_bpf_add_filter(bytecode, fail=False)
     fail(ret != 0, "Software TC filter did not load")
-    sim.dfs_get_bound_progs(expected=0)
+    simdev.dfs_get_bound_progs(expected=0)
     ingress = sim.tc_show_ingress(expected=1)
     fltr = ingress[0]
     fail(not fltr["in_hw"], "Bytecode not offloaded by default")
@@ -839,7 +904,7 @@ try:
     check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
 
     start_test("Test TC offload basics...")
-    dfs = sim.dfs_get_bound_progs(expected=1)
+    dfs = simdev.dfs_get_bound_progs(expected=1)
     progs = bpftool_prog_list(expected=1)
     ingress = sim.tc_show_ingress(expected=1)
 
@@ -874,18 +939,20 @@ try:
 
     start_test("Test destroying device gets rid of TC filters...")
     sim.cls_bpf_add_filter(obj, skip_sw=True)
-    sim.remove()
+    simdev.remove()
     bpftool_prog_list_wait(expected=0)
 
-    sim = NetdevSim()
+    simdev = NetdevSimDev()
+    sim, = simdev.nsims
     sim.set_ethtool_tc_offloads(True)
 
     start_test("Test destroying device gets rid of XDP...")
     sim.set_xdp(obj, "offload")
-    sim.remove()
+    simdev.remove()
     bpftool_prog_list_wait(expected=0)
 
-    sim = NetdevSim()
+    simdev = NetdevSimDev()
+    sim, = simdev.nsims
     sim.set_ethtool_tc_offloads(True)
 
     start_test("Test XDP prog reporting...")
@@ -971,7 +1038,7 @@ try:
     check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
 
     start_test("Test XDP offload is device bound...")
-    dfs = sim.dfs_get_bound_progs(expected=1)
+    dfs = simdev.dfs_get_bound_progs(expected=1)
     dprog = dfs[0]
 
     fail(prog["id"] != link_xdp["id"], "Program IDs don't match")
@@ -990,7 +1057,8 @@ try:
     bpftool_prog_list_wait(expected=0)
 
     start_test("Test attempt to use a program for a wrong device...")
-    sim2 = NetdevSim()
+    simdev2 = NetdevSimDev()
+    sim2, = simdev2.nsims
     sim2.set_xdp(obj, "offload")
     pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
 
@@ -998,7 +1066,7 @@ try:
                               fail=False, include_stderr=True)
     fail(ret == 0, "Pinned program loaded for a different device accepted")
     check_extack_nsim(err, "program bound to different dev.", args)
-    sim2.remove()
+    simdev2.remove()
     ret, _, err = sim.set_xdp(pinned, "offload",
                               fail=False, include_stderr=True)
     fail(ret == 0, "Pinned program loaded for a removed device accepted")
@@ -1006,9 +1074,9 @@ try:
     rm(pin_file)
     bpftool_prog_list_wait(expected=0)
 
-    sim = test_multi_prog(sim, obj, "", 1)
-    sim = test_multi_prog(sim, obj, "drv", 1)
-    sim = test_multi_prog(sim, obj, "generic", 2)
+    simdev, sim = test_multi_prog(simdev, sim, obj, "", 1)
+    simdev, sim = test_multi_prog(simdev, sim, obj, "drv", 1)
+    simdev, sim = test_multi_prog(simdev, sim, obj, "generic", 2)
 
     start_test("Test mixing of TC and XDP...")
     sim.tc_add_ingress()
@@ -1055,15 +1123,15 @@ try:
 
     start_test("Test if netdev removal waits for translation...")
     delay_msec = 500
-    sim.dfs["sdev/bpf_bind_verifier_delay"] = delay_msec
+    sim.dfs["dev/bpf_bind_verifier_delay"] = delay_msec
     start = time.time()
     cmd_line = "tc filter add dev %s ingress bpf %s da skip_sw" % \
                (sim['ifname'], obj)
     tc_proc = cmd(cmd_line, background=True, fail=False)
     # Wait for the verifier to start
-    while sim.dfs_num_bound_progs() <= 2:
+    while simdev.dfs_num_bound_progs() <= 2:
         pass
-    sim.remove()
+    simdev.remove()
     end = time.time()
     ret, _ = cmd_result(tc_proc, fail=False)
     time_diff = end - start
@@ -1078,7 +1146,8 @@ try:
     clean_up()
     bpftool_prog_list_wait(expected=0)
 
-    sim = NetdevSim()
+    simdev = NetdevSimDev()
+    sim, = simdev.nsims
     map_obj = bpf_obj("sample_map_ret0.o")
     start_test("Test loading program with maps...")
     sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
@@ -1100,7 +1169,7 @@ try:
 
     prog_file, _ = pin_prog("/sys/fs/bpf/tmp_prog")
     map_file, _ = pin_map("/sys/fs/bpf/tmp_map", idx=1, expected=2)
-    sim.remove()
+    simdev.remove()
 
     start_test("Test bpftool bound info reporting (removed dev)...")
     check_dev_info_removed(prog_file=prog_file, map_file=map_file)
@@ -1109,7 +1178,8 @@ try:
     clean_up()
     bpftool_prog_list_wait(expected=0)
 
-    sim = NetdevSim()
+    simdev = NetdevSimDev()
+    sim, = simdev.nsims
 
     start_test("Test map update (no flags)...")
     sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
@@ -1190,27 +1260,29 @@ try:
     start_test("Test map remove...")
     sim.unset_xdp("offload")
     bpftool_map_list_wait(expected=0)
-    sim.remove()
+    simdev.remove()
 
-    sim = NetdevSim()
+    simdev = NetdevSimDev()
+    sim, = simdev.nsims
     sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
-    sim.remove()
+    simdev.remove()
     bpftool_map_list_wait(expected=0)
 
     start_test("Test map creation fail path...")
-    sim = NetdevSim()
+    simdev = NetdevSimDev()
+    sim, = simdev.nsims
     sim.dfs["bpf_map_accept"] = "N"
     ret, _ = sim.set_xdp(map_obj, "offload", JSON=False, fail=False)
     fail(ret == 0,
          "netdevsim didn't refuse to create a map with offload disabled")
 
-    sim.remove()
+    simdev.remove()
 
     start_test("Test multi-dev ASIC program reuse...")
-    simA = NetdevSim()
-    simB1 = NetdevSim()
-    simB2 = NetdevSim(link=simB1)
-    simB3 = NetdevSim(link=simB1)
+    simdevA = NetdevSimDev()
+    simA, = simdevA.nsims
+    simdevB = NetdevSimDev(3)
+    simB1, simB2, simB3 = simdevB.nsims
     sims = (simA, simB1, simB2, simB3)
     simB = (simB1, simB2, simB3)
 
@@ -1222,13 +1294,13 @@ try:
     progB = bpf_pinned("/sys/fs/bpf/nsimB")
 
     simA.set_xdp(progA, "offload", JSON=False)
-    for d in simB:
+    for d in simdevB.nsims:
         d.set_xdp(progB, "offload", JSON=False)
 
     start_test("Test multi-dev ASIC cross-dev replace...")
     ret, _ = simA.set_xdp(progB, "offload", force=True, JSON=False, fail=False)
     fail(ret == 0, "cross-ASIC program allowed")
-    for d in simB:
+    for d in simdevB.nsims:
         ret, _ = d.set_xdp(progA, "offload", force=True, JSON=False, fail=False)
         fail(ret == 0, "cross-ASIC program allowed")
 
@@ -1240,7 +1312,7 @@ try:
                                fail=False, include_stderr=True)
     fail(ret == 0, "cross-ASIC program allowed")
     check_extack_nsim(err, "program bound to different dev.", args)
-    for d in simB:
+    for d in simdevB.nsims:
         ret, _, err = d.set_xdp(progA, "offload", force=True, JSON=False,
                                 fail=False, include_stderr=True)
         fail(ret == 0, "cross-ASIC program allowed")
@@ -1277,7 +1349,7 @@ try:
     start_test("Test multi-dev ASIC cross-dev destruction...")
     bpftool_prog_list_wait(expected=2)
 
-    simA.remove()
+    simdevA.remove()
     bpftool_prog_list_wait(expected=1)
 
     ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
@@ -1295,6 +1367,7 @@ try:
     fail(ifnameB != simB3['ifname'], "program not bound to remaining device")
 
     simB3.remove()
+    simdevB.remove()
     bpftool_prog_list_wait(expected=0)
 
     start_test("Test multi-dev ASIC cross-dev destruction - orphaned...")
diff --git a/tools/testing/selftests/net/rtnetlink.sh b/tools/testing/selftests/net/rtnetlink.sh
index b447803f3f8a..b25c9fe019d2 100755
--- a/tools/testing/selftests/net/rtnetlink.sh
+++ b/tools/testing/selftests/net/rtnetlink.sh
@@ -696,9 +696,9 @@ kci_test_ipsec_offload()
 	algo="aead rfc4106(gcm(aes)) 0x3132333435363738393031323334353664636261 128"
 	srcip=192.168.123.3
 	dstip=192.168.123.4
-	dev=simx1
-	sysfsd=/sys/kernel/debug/netdevsim/$dev
+	sysfsd=/sys/kernel/debug/netdevsim/netdevsim0/ports/0/
 	sysfsf=$sysfsd/ipsec
+	sysfsnet=/sys/bus/netdevsim/devices/netdevsim0/net/
 
 	# setup netdevsim since dummydev doesn't have offload support
 	modprobe netdevsim
@@ -708,7 +708,11 @@ kci_test_ipsec_offload()
 		return 1
 	fi
 
-	ip link add $dev type netdevsim
+	echo "0" > /sys/bus/netdevsim/new_device
+	while [ ! -d $sysfsnet ] ; do :; done
+	udevadm settle
+	dev=`ls $sysfsnet`
+
 	ip addr add $srcip dev $dev
 	ip link set $dev up
 	if [ ! -d $sysfsd ] ; then
@@ -781,7 +785,6 @@ EOF
 	fi
 
 	# clean up any leftovers
-	ip link del $dev
 	rmmod netdevsim
 
 	if [ $ret -ne 0 ]; then