From fe79f85fc85691b64db1997a3b1a2df5d6d378ae Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 17 Mar 2026 19:09:27 +0100 Subject: unittests: test_private: modify it to use CTokenizer directly Change the logic to use the tokenizer directly. This allows adding more unit tests to check the validty of the tokenizer itself. Signed-off-by: Mauro Carvalho Chehab Message-ID: <2672257233ff73a9464c09b50924be51e25d4f59.1773074166.git.mchehab+huawei@kernel.org> Signed-off-by: Jonathan Corbet Message-ID: <66e6320a4d5ad9730c1c0ceea79b5021e90c66c6.1773770483.git.mchehab+huawei@kernel.org> --- tools/unittests/test_private.py | 331 ----------------------------------- tools/unittests/test_tokenizer.py | 358 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 358 insertions(+), 331 deletions(-) delete mode 100755 tools/unittests/test_private.py create mode 100755 tools/unittests/test_tokenizer.py diff --git a/tools/unittests/test_private.py b/tools/unittests/test_private.py deleted file mode 100755 index eae245ae8a12..000000000000 --- a/tools/unittests/test_private.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env python3 - -""" -Unit tests for struct/union member extractor class. -""" - - -import os -import re -import unittest -import sys - -from unittest.mock import MagicMock - -SRC_DIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python")) - -from kdoc.kdoc_parser import trim_private_members -from unittest_helper import run_unittest - -# -# List of tests. -# -# The code will dynamically generate one test for each key on this dictionary. -# - -#: Tests to check if CTokenizer is handling properly public/private comments. -TESTS_PRIVATE = { - # - # Simplest case: no private. Ensure that trimming won't affect struct - # - "no private": { - "source": """ - struct foo { - int a; - int b; - int c; - }; - """, - "trimmed": """ - struct foo { - int a; - int b; - int c; - }; - """, - }, - - # - # Play "by the books" by always having a public in place - # - - "balanced_private": { - "source": """ - struct foo { - int a; - /* private: */ - int b; - /* public: */ - int c; - }; - """, - "trimmed": """ - struct foo { - int a; - int c; - }; - """, - }, - - "balanced_non_greddy_private": { - "source": """ - struct foo { - int a; - /* private: */ - int b; - /* public: */ - int c; - /* private: */ - int d; - /* public: */ - int e; - - }; - """, - "trimmed": """ - struct foo { - int a; - int c; - int e; - }; - """, - }, - - "balanced_inner_private": { - "source": """ - struct foo { - struct { - int a; - /* private: ignore below */ - int b; - /* public: but this should not be ignored */ - }; - int b; - }; - """, - "trimmed": """ - struct foo { - struct { - int a; - }; - int b; - }; - """, - }, - - # - # Test what happens if there's no public after private place - # - - "unbalanced_private": { - "source": """ - struct foo { - int a; - /* private: */ - int b; - int c; - }; - """, - "trimmed": """ - struct foo { - int a; - }; - """, - }, - - "unbalanced_inner_private": { - "source": """ - struct foo { - struct { - int a; - /* private: ignore below */ - int b; - /* but this should not be ignored */ - }; - int b; - }; - """, - "trimmed": """ - struct foo { - struct { - int a; - }; - int b; - }; - """, - }, - - "unbalanced_struct_group_tagged_with_private": { - "source": """ - struct page_pool_params { - struct_group_tagged(page_pool_params_fast, fast, - unsigned int order; - unsigned int pool_size; - int nid; - struct device *dev; - struct napi_struct *napi; - enum dma_data_direction dma_dir; - unsigned int max_len; - unsigned int offset; - }; - struct_group_tagged(page_pool_params_slow, slow, - struct net_device *netdev; - unsigned int queue_idx; - unsigned int flags; - /* private: used by test code only */ - void (*init_callback)(netmem_ref netmem, void *arg); - void *init_arg; - }; - }; - """, - "trimmed": """ - struct page_pool_params { - struct_group_tagged(page_pool_params_fast, fast, - unsigned int order; - unsigned int pool_size; - int nid; - struct device *dev; - struct napi_struct *napi; - enum dma_data_direction dma_dir; - unsigned int max_len; - unsigned int offset; - }; - struct_group_tagged(page_pool_params_slow, slow, - struct net_device *netdev; - unsigned int queue_idx; - unsigned int flags; - }; - }; - """, - }, - - "unbalanced_two_struct_group_tagged_first_with_private": { - "source": """ - struct page_pool_params { - struct_group_tagged(page_pool_params_slow, slow, - struct net_device *netdev; - unsigned int queue_idx; - unsigned int flags; - /* private: used by test code only */ - void (*init_callback)(netmem_ref netmem, void *arg); - void *init_arg; - }; - struct_group_tagged(page_pool_params_fast, fast, - unsigned int order; - unsigned int pool_size; - int nid; - struct device *dev; - struct napi_struct *napi; - enum dma_data_direction dma_dir; - unsigned int max_len; - unsigned int offset; - }; - }; - """, - "trimmed": """ - struct page_pool_params { - struct_group_tagged(page_pool_params_slow, slow, - struct net_device *netdev; - unsigned int queue_idx; - unsigned int flags; - }; - struct_group_tagged(page_pool_params_fast, fast, - unsigned int order; - unsigned int pool_size; - int nid; - struct device *dev; - struct napi_struct *napi; - enum dma_data_direction dma_dir; - unsigned int max_len; - unsigned int offset; - }; - }; - """, - }, - "unbalanced_without_end_of_line": { - "source": """ \ - struct page_pool_params { \ - struct_group_tagged(page_pool_params_slow, slow, \ - struct net_device *netdev; \ - unsigned int queue_idx; \ - unsigned int flags; - /* private: used by test code only */ - void (*init_callback)(netmem_ref netmem, void *arg); \ - void *init_arg; \ - }; \ - struct_group_tagged(page_pool_params_fast, fast, \ - unsigned int order; \ - unsigned int pool_size; \ - int nid; \ - struct device *dev; \ - struct napi_struct *napi; \ - enum dma_data_direction dma_dir; \ - unsigned int max_len; \ - unsigned int offset; \ - }; \ - }; - """, - "trimmed": """ - struct page_pool_params { - struct_group_tagged(page_pool_params_slow, slow, - struct net_device *netdev; - unsigned int queue_idx; - unsigned int flags; - }; - struct_group_tagged(page_pool_params_fast, fast, - unsigned int order; - unsigned int pool_size; - int nid; - struct device *dev; - struct napi_struct *napi; - enum dma_data_direction dma_dir; - unsigned int max_len; - unsigned int offset; - }; - }; - """, - }, -} - - -class TestPublicPrivate(unittest.TestCase): - """ - Main test class. Populated dynamically at runtime. - """ - - def setUp(self): - self.maxDiff = None - - def add_test(cls, name, source, trimmed): - """ - Dynamically add a test to the class - """ - def test(cls): - result = trim_private_members(source) - - result = re.sub(r"\s++", " ", result).strip() - expected = re.sub(r"\s++", " ", trimmed).strip() - - msg = f"failed when parsing this source:\n" + source - - cls.assertEqual(result, expected, msg=msg) - - test.__name__ = f'test {name}' - - setattr(TestPublicPrivate, test.__name__, test) - - -# -# Populate TestPublicPrivate class -# -test_class = TestPublicPrivate() -for name, test in TESTS_PRIVATE.items(): - test_class.add_test(name, test["source"], test["trimmed"]) - - -# -# main -# -if __name__ == "__main__": - run_unittest(__file__) diff --git a/tools/unittests/test_tokenizer.py b/tools/unittests/test_tokenizer.py new file mode 100755 index 000000000000..3b1d0b5bd311 --- /dev/null +++ b/tools/unittests/test_tokenizer.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 + +""" +Unit tests for struct/union member extractor class. +""" + + +import os +import re +import unittest +import sys + +from unittest.mock import MagicMock + +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python")) + +from kdoc.c_lex import CTokenizer +from unittest_helper import run_unittest + + +# +# List of tests. +# +# The code will dynamically generate one test for each key on this dictionary. +# + +def make_private_test(name, data): + """ + Create a test named ``name`` using parameters given by ``data`` dict. + """ + + def test(self): + """In-lined lambda-like function to run the test""" + tokens = CTokenizer(data["source"]) + result = str(tokens) + + # + # Avoid whitespace false positives + # + result = re.sub(r"\s++", " ", result).strip() + expected = re.sub(r"\s++", " ", data["trimmed"]).strip() + + msg = f"failed when parsing this source:\n{data['source']}" + self.assertEqual(result, expected, msg=msg) + + return test + +#: Tests to check if CTokenizer is handling properly public/private comments. +TESTS_PRIVATE = { + # + # Simplest case: no private. Ensure that trimming won't affect struct + # + "__run__": make_private_test, + "no private": { + "source": """ + struct foo { + int a; + int b; + int c; + }; + """, + "trimmed": """ + struct foo { + int a; + int b; + int c; + }; + """, + }, + + # + # Play "by the books" by always having a public in place + # + + "balanced_private": { + "source": """ + struct foo { + int a; + /* private: */ + int b; + /* public: */ + int c; + }; + """, + "trimmed": """ + struct foo { + int a; + int c; + }; + """, + }, + + "balanced_non_greddy_private": { + "source": """ + struct foo { + int a; + /* private: */ + int b; + /* public: */ + int c; + /* private: */ + int d; + /* public: */ + int e; + + }; + """, + "trimmed": """ + struct foo { + int a; + int c; + int e; + }; + """, + }, + + "balanced_inner_private": { + "source": """ + struct foo { + struct { + int a; + /* private: ignore below */ + int b; + /* public: but this should not be ignored */ + }; + int b; + }; + """, + "trimmed": """ + struct foo { + struct { + int a; + }; + int b; + }; + """, + }, + + # + # Test what happens if there's no public after private place + # + + "unbalanced_private": { + "source": """ + struct foo { + int a; + /* private: */ + int b; + int c; + }; + """, + "trimmed": """ + struct foo { + int a; + }; + """, + }, + + "unbalanced_inner_private": { + "source": """ + struct foo { + struct { + int a; + /* private: ignore below */ + int b; + /* but this should not be ignored */ + }; + int b; + }; + """, + "trimmed": """ + struct foo { + struct { + int a; + }; + int b; + }; + """, + }, + + "unbalanced_struct_group_tagged_with_private": { + "source": """ + struct page_pool_params { + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + }; + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + /* private: used by test code only */ + void (*init_callback)(netmem_ref netmem, void *arg); + void *init_arg; + }; + }; + """, + "trimmed": """ + struct page_pool_params { + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + }; + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + }; + }; + """, + }, + + "unbalanced_two_struct_group_tagged_first_with_private": { + "source": """ + struct page_pool_params { + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + /* private: used by test code only */ + void (*init_callback)(netmem_ref netmem, void *arg); + void *init_arg; + }; + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + }; + }; + """, + "trimmed": """ + struct page_pool_params { + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + }; + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + }; + }; + """, + }, + "unbalanced_without_end_of_line": { + "source": """ \ + struct page_pool_params { \ + struct_group_tagged(page_pool_params_slow, slow, \ + struct net_device *netdev; \ + unsigned int queue_idx; \ + unsigned int flags; + /* private: used by test code only */ + void (*init_callback)(netmem_ref netmem, void *arg); \ + void *init_arg; \ + }; \ + struct_group_tagged(page_pool_params_fast, fast, \ + unsigned int order; \ + unsigned int pool_size; \ + int nid; \ + struct device *dev; \ + struct napi_struct *napi; \ + enum dma_data_direction dma_dir; \ + unsigned int max_len; \ + unsigned int offset; \ + }; \ + }; + """, + "trimmed": """ + struct page_pool_params { + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + }; + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + }; + }; + """, + }, +} + +#: Dict containing all test groups fror CTokenizer +TESTS = { + "TestPublicPrivate": TESTS_PRIVATE, +} + +def setUp(self): + self.maxDiff = None + +def build_test_class(group_name, table): + """ + Dynamically creates a class instance using type() as a generator + for a new class derivated from unittest.TestCase. + + We're opting to do it inside a function to avoid the risk of + changing the globals() dictionary. + """ + + class_dict = { + "setUp": setUp + } + + run = table["__run__"] + + for test_name, data in table.items(): + if test_name == "__run__": + continue + + class_dict[f"test_{test_name}"] = run(test_name, data) + + cls = type(group_name, (unittest.TestCase,), class_dict) + + return cls.__name__, cls + +# +# Create classes and add them to the global dictionary +# +for group, table in TESTS.items(): + t = build_test_class(group, table) + globals()[t[0]] = t[1] + +# +# main +# +if __name__ == "__main__": + run_unittest(__file__) -- cgit v1.2.3