tc-testing: Allow test cases to be skipped

By adding a check for an optional key/value pair to the test case
data, individual test cases may be skipped to prevent tdc from
aborting a test run due to setup or teardown failure.

If a test case is skipped, it will still appear in the results
output to allow for a consistent number of executed tests in each
run. However, the test will be marked as skipped.

This support for skipping extends to any plugins that may generate
additional results for each executed test.

Signed-off-by: Lucas Bates <lucasb@mojatatu.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

4 files changed, 27 insertions, 13 deletions

diff --git a/tools/testing/selftests/tc-testing/TdcPlugin.py b/tools/testing/selftests/tc-testing/TdcPlugin.py
index 1d9e279331eb..b980a565fa89 100644
--- a/tools/testing/selftests/tc-testing/TdcPlugin.py
+++ b/tools/testing/selftests/tc-testing/TdcPlugin.py
@@ -18,13 +18,13 @@ class TdcPlugin:
         if self.args.verbose > 1:
             print(' -- {}.post_suite'.format(self.sub_class))
 
-    def pre_case(self, test_ordinal, testid, test_name):
+    def pre_case(self, testid, test_name, test_skip):
         '''run commands before test_runner does one test'''
         if self.args.verbose > 1:
             print(' -- {}.pre_case'.format(self.sub_class))
         self.args.testid = testid
         self.args.test_name = test_name
-        self.args.test_ordinal = test_ordinal
+        self.args.test_skip = test_skip
 
     def post_case(self):
         '''run commands after test_runner does one test'''
diff --git a/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt b/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt
index 17b267dedbd9..a28571aff0e1 100644
--- a/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt
+++ b/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt
@@ -33,6 +33,11 @@ Each test case has required data:
 
 id:           A unique alphanumeric value to identify a particular test case
 name:         Descriptive name that explains the command under test
+skip:         A completely optional key, if the corresponding value is "yes"
+              then tdc will not execute the test case in question. However,
+              this test case will still appear in the results output but
+              marked as skipped. This key can be placed anywhere inside the
+              test case at the top level.
 category:     A list of single-word descriptions covering what the command
               under test is testing. Example: filter, actions, u32, gact, etc.
 setup:        The list of commands required to ensure the command under test
diff --git a/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py b/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py
index e00c798de0bb..4bb866575ea1 100644
--- a/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py
+++ b/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py
@@ -102,6 +102,14 @@ class SubPlugin(TdcPlugin):
         if not self.args.valgrind:
             return
 
+        res = TestResult('{}-mem'.format(self.args.testid),
+              '{} memory leak check'.format(self.args.test_name))
+        if self.args.test_skip:
+            res.set_result(ResultState.skip)
+            res.set_errormsg('Test case designated as skipped.')
+            self._add_results(res)
+            return
+
         self.definitely_lost_re = re.compile(
             r'definitely lost:\s+([,0-9]+)\s+bytes in\s+([,0-9]+)\sblocks', re.MULTILINE | re.DOTALL)
         self.indirectly_lost_re = re.compile(
@@ -134,8 +142,6 @@ class SubPlugin(TdcPlugin):
                 nle_num = int(nle_mo.group(1))
 
         mem_results = ''
-        res = TestResult('{}-mem'.format(self.args.testid),
-              '{} memory leak check'.format(self.args.test_name))
         if (def_num > 0) or (ind_num > 0) or (pos_num > 0) or (nle_num > 0):
             mem_results += 'not '
             res.set_result(ResultState.fail)
@@ -146,12 +152,6 @@ class SubPlugin(TdcPlugin):
 
         self._add_results(res)
 
-        mem_results += 'ok {} - {}-mem # {}\n'.format(
-            self.args.test_ordinal, self.args.testid, 'memory leak check')
-        self._add_to_tap(mem_results)
-        if mem_results.startswith('not '):
-            print('{}'.format(content))
-            self._add_to_tap(content)
 
     def _add_results(self, res):
         self._tsr.add_resultdata(res)
diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py
index e6e4ce80a726..5cee15659e5f 100755
--- a/tools/testing/selftests/tc-testing/tdc.py
+++ b/tools/testing/selftests/tc-testing/tdc.py
@@ -61,10 +61,10 @@ class PluginMgr:
         for pgn_inst in reversed(self.plugin_instances):
             pgn_inst.post_suite(index)
 
-    def call_pre_case(self, test_ordinal, testid, test_name):
+    def call_pre_case(self, testid, test_name, *, test_skip=False):
         for pgn_inst in self.plugin_instances:
             try:
-                pgn_inst.pre_case(test_ordinal, testid, test_name)
+                pgn_inst.pre_case(testid, test_name, test_skip)
             except Exception as ee:
                 print('exception {} in call to pre_case for {} plugin'.
                       format(ee, pgn_inst.__class__))
@@ -192,10 +192,19 @@ def run_one_test(pm, args, index, tidx):
         print("\t====================\n=====> ", end="")
     print("Test " + tidx["id"] + ": " + tidx["name"])
 
+    if 'skip' in tidx:
+        if tidx['skip'] == 'yes':
+            res = TestResult(tidx['id'], tidx['name'])
+            res.set_result(ResultState.skip)
+            res.set_errormsg('Test case designated as skipped.')
+            pm.call_pre_case(tidx['id'], tidx['name'], test_skip=True)
+            pm.call_post_execute()
+            return res
+
     # populate NAMES with TESTID for this test
     NAMES['TESTID'] = tidx['id']
 
-    pm.call_pre_case(index, tidx['id'], tidx['name'])
+    pm.call_pre_case(tidx['id'], tidx['name'])
     prepare_env(args, pm, 'setup', "-----> prepare stage", tidx["setup"])
 
     if (args.verbose > 0):