From 7c2435ef76e5f2d9fac44b241e4a54113f6eafbf Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Tue, 23 May 2023 10:37:47 +0100 Subject: tools: ynl: Use dict of predefined Structs to decode scalar types Use a dict of predefined Struct() objects to decode scalar types in native, big or little endian format. This removes the repetitive code for the scalar variants and ensures all the signed variants are supported. Signed-off-by: Donald Hunter Signed-off-by: David S. Miller --- tools/net/ynl/lib/ynl.py | 101 +++++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 57 deletions(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index aa77bcae4807..6185ba27f2e7 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -1,10 +1,12 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +from collections import namedtuple import functools import os import random import socket import struct +from struct import Struct import yaml from .nlspec import SpecFamily @@ -76,10 +78,17 @@ class NlError(Exception): class NlAttr: - type_formats = { 'u8' : ('B', 1), 's8' : ('b', 1), - 'u16': ('H', 2), 's16': ('h', 2), - 'u32': ('I', 4), 's32': ('i', 4), - 'u64': ('Q', 8), 's64': ('q', 8) } + ScalarFormat = namedtuple('ScalarFormat', ['native', 'big', 'little']) + type_formats = { + 'u8' : ScalarFormat(Struct('B'), Struct("B"), Struct("B")), + 's8' : ScalarFormat(Struct('b'), Struct("b"), Struct("b")), + 'u16': ScalarFormat(Struct('H'), Struct(">H"), Struct("h"), Struct("I"), Struct("i"), Struct("Q"), Struct("q"), Struct("" if byte_order == "big-endian" else "<" - return "" + return format.big if byte_order == "big-endian" \ + else format.little + return format.native - def as_u8(self): - return struct.unpack("B", self.raw)[0] - - def as_u16(self, byte_order=None): - endian = NlAttr.format_byte_order(byte_order) - return struct.unpack(f"{endian}H", self.raw)[0] - - def as_u32(self, byte_order=None): - endian = NlAttr.format_byte_order(byte_order) - return struct.unpack(f"{endian}I", self.raw)[0] - - def as_u64(self, byte_order=None): - endian = NlAttr.format_byte_order(byte_order) - return struct.unpack(f"{endian}Q", self.raw)[0] + def as_scalar(self, attr_type, byte_order=None): + format = self.get_format(attr_type, byte_order) + return format.unpack(self.raw)[0] def as_strz(self): return self.raw.decode('ascii')[:-1] @@ -115,17 +116,17 @@ class NlAttr: return self.raw def as_c_array(self, type): - format, _ = self.type_formats[type] - return list({ x[0] for x in struct.iter_unpack(format, self.raw) }) + format = self.get_format(type) + return [ x[0] for x in format.iter_unpack(self.raw) ] def as_struct(self, members): value = dict() offset = 0 for m in members: # TODO: handle non-scalar members - format, size = self.type_formats[m.type] - decoded = struct.unpack_from(format, self.raw, offset) - offset += size + format = self.get_format(m.type) + decoded = format.unpack_from(self.raw, offset) + offset += format.size value[m.name] = decoded[0] return value @@ -184,11 +185,11 @@ class NlMsg: if extack.type == Netlink.NLMSGERR_ATTR_MSG: self.extack['msg'] = extack.as_strz() elif extack.type == Netlink.NLMSGERR_ATTR_MISS_TYPE: - self.extack['miss-type'] = extack.as_u32() + self.extack['miss-type'] = extack.as_scalar('u32') elif extack.type == Netlink.NLMSGERR_ATTR_MISS_NEST: - self.extack['miss-nest'] = extack.as_u32() + self.extack['miss-nest'] = extack.as_scalar('u32') elif extack.type == Netlink.NLMSGERR_ATTR_OFFS: - self.extack['bad-attr-offs'] = extack.as_u32() + self.extack['bad-attr-offs'] = extack.as_scalar('u32') else: if 'unknown' not in self.extack: self.extack['unknown'] = [] @@ -272,11 +273,11 @@ def _genl_load_families(): fam = dict() for attr in gm.raw_attrs: if attr.type == Netlink.CTRL_ATTR_FAMILY_ID: - fam['id'] = attr.as_u16() + fam['id'] = attr.as_scalar('u16') elif attr.type == Netlink.CTRL_ATTR_FAMILY_NAME: fam['name'] = attr.as_strz() elif attr.type == Netlink.CTRL_ATTR_MAXATTR: - fam['maxattr'] = attr.as_u32() + fam['maxattr'] = attr.as_scalar('u32') elif attr.type == Netlink.CTRL_ATTR_MCAST_GROUPS: fam['mcast'] = dict() for entry in NlAttrs(attr.raw): @@ -286,7 +287,7 @@ def _genl_load_families(): if entry_attr.type == Netlink.CTRL_ATTR_MCAST_GRP_NAME: mcast_name = entry_attr.as_strz() elif entry_attr.type == Netlink.CTRL_ATTR_MCAST_GRP_ID: - mcast_id = entry_attr.as_u32() + mcast_id = entry_attr.as_scalar('u32') if mcast_name and mcast_id is not None: fam['mcast'][mcast_name] = mcast_id if 'name' in fam and 'id' in fam: @@ -304,9 +305,9 @@ class GenlMsg: self.fixed_header_attrs = dict() for m in fixed_header_members: - format, size = NlAttr.type_formats[m.type] - decoded = struct.unpack_from(format, nl_msg.raw, offset) - offset += size + format = NlAttr.get_format(m.type) + decoded = format.unpack_from(nl_msg.raw, offset) + offset += format.size self.fixed_header_attrs[m.name] = decoded[0] self.raw = nl_msg.raw[offset:] @@ -381,21 +382,13 @@ class YnlFamily(SpecFamily): attr_payload += self._add_attr(attr['nested-attributes'], subname, subvalue) elif attr["type"] == 'flag': attr_payload = b'' - elif attr["type"] == 'u8': - attr_payload = struct.pack("B", int(value)) - elif attr["type"] == 'u16': - endian = NlAttr.format_byte_order(attr.byte_order) - attr_payload = struct.pack(f"{endian}H", int(value)) - elif attr["type"] == 'u32': - endian = NlAttr.format_byte_order(attr.byte_order) - attr_payload = struct.pack(f"{endian}I", int(value)) - elif attr["type"] == 'u64': - endian = NlAttr.format_byte_order(attr.byte_order) - attr_payload = struct.pack(f"{endian}Q", int(value)) elif attr["type"] == 'string': attr_payload = str(value).encode('ascii') + b'\x00' elif attr["type"] == 'binary': attr_payload = value + elif attr['type'] in NlAttr.type_formats: + format = NlAttr.get_format(attr['type'], attr.byte_order) + attr_payload = format.pack(int(value)) else: raise Exception(f'Unknown type at {space} {name} {value} {attr["type"]}') @@ -434,22 +427,16 @@ class YnlFamily(SpecFamily): if attr_spec["type"] == 'nest': subdict = self._decode(NlAttrs(attr.raw), attr_spec['nested-attributes']) decoded = subdict - elif attr_spec['type'] == 'u8': - decoded = attr.as_u8() - elif attr_spec['type'] == 'u16': - decoded = attr.as_u16(attr_spec.byte_order) - elif attr_spec['type'] == 'u32': - decoded = attr.as_u32(attr_spec.byte_order) - elif attr_spec['type'] == 'u64': - decoded = attr.as_u64(attr_spec.byte_order) elif attr_spec["type"] == 'string': decoded = attr.as_strz() elif attr_spec["type"] == 'binary': decoded = self._decode_binary(attr, attr_spec) elif attr_spec["type"] == 'flag': decoded = True + elif attr_spec["type"] in NlAttr.type_formats: + decoded = attr.as_scalar(attr_spec['type'], attr_spec.byte_order) else: - raise Exception(f'Unknown {attr.type} {attr_spec["name"]} {attr_spec["type"]}') + raise Exception(f'Unknown {attr_spec["type"]} with name {attr_spec["name"]}') if not attr_spec.is_multi: rsp[attr_spec['name']] = decoded @@ -555,8 +542,8 @@ class YnlFamily(SpecFamily): fixed_header_members = self.consts[op.fixed_header].members for m in fixed_header_members: value = vals.pop(m.name) - format, _ = NlAttr.type_formats[m.type] - msg += struct.pack(format, value) + format = NlAttr.get_format(m.type) + msg += format.pack(value) for name, value in vals.items(): msg += self._add_attr(op.attr_set.name, name, value) msg = _genl_msg_finalize(msg) -- cgit v1.2.3 From bddd2e561b0ad5ca42e16fb26a20fc806d521912 Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Tue, 23 May 2023 10:37:48 +0100 Subject: tools: ynl: Handle byte-order in struct members Add support for byte-order in struct members in the genetlink-legacy spec. Signed-off-by: Donald Hunter Acked-by: Jakub Kicinski Signed-off-by: David S. Miller --- Documentation/netlink/genetlink-legacy.yaml | 2 ++ tools/net/ynl/lib/nlspec.py | 4 +++- tools/net/ynl/lib/ynl.py | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/Documentation/netlink/genetlink-legacy.yaml b/Documentation/netlink/genetlink-legacy.yaml index b33541a51d6b..b5319cde9e17 100644 --- a/Documentation/netlink/genetlink-legacy.yaml +++ b/Documentation/netlink/genetlink-legacy.yaml @@ -122,6 +122,8 @@ properties: enum: [ u8, u16, u32, u64, s8, s16, s32, s64, string ] len: $ref: '#/$defs/len-or-define' + byte-order: + enum: [ little-endian, big-endian ] # End genetlink-legacy attribute-sets: diff --git a/tools/net/ynl/lib/nlspec.py b/tools/net/ynl/lib/nlspec.py index a0241add3839..c624cdfde223 100644 --- a/tools/net/ynl/lib/nlspec.py +++ b/tools/net/ynl/lib/nlspec.py @@ -226,11 +226,13 @@ class SpecStructMember(SpecElement): Represents a single struct member attribute. Attributes: - type string, type of the member attribute + type string, type of the member attribute + byte_order string or None for native byte order """ def __init__(self, family, yaml): super().__init__(family, yaml) self.type = yaml['type'] + self.byte_order = yaml.get('byte-order') class SpecStruct(SpecElement): diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index 6185ba27f2e7..39a2296c0003 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -124,7 +124,7 @@ class NlAttr: offset = 0 for m in members: # TODO: handle non-scalar members - format = self.get_format(m.type) + format = self.get_format(m.type, m.byte_order) decoded = format.unpack_from(self.raw, offset) offset += format.size value[m.name] = decoded[0] @@ -305,7 +305,7 @@ class GenlMsg: self.fixed_header_attrs = dict() for m in fixed_header_members: - format = NlAttr.get_format(m.type) + format = NlAttr.get_format(m.type, m.byte_order) decoded = format.unpack_from(nl_msg.raw, offset) offset += format.size self.fixed_header_attrs[m.name] = decoded[0] @@ -542,7 +542,7 @@ class YnlFamily(SpecFamily): fixed_header_members = self.consts[op.fixed_header].members for m in fixed_header_members: value = vals.pop(m.name) - format = NlAttr.get_format(m.type) + format = NlAttr.get_format(m.type, m.byte_order) msg += format.pack(value) for name, value in vals.items(): msg += self._add_attr(op.attr_set.name, name, value) -- cgit v1.2.3 From 081e8df6819997eae236f75dd52f0c147c4be939 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 24 May 2023 10:07:12 -0700 Subject: tools: ynl: avoid dict errors on older Python versions Python 3.9.0 or newer supports combining dicts() with |, but older versions of Python are still used in the wild (e.g. on CentOS 8, which goes EoL May 31, 2024). With Python 3.6.8 we get: TypeError: unsupported operand type(s) for |: 'dict' and 'dict' Use older syntax. Tested with non-legacy families only. Fixes: f036d936ca57 ("tools: ynl: Add fixed-header support to ynl") Reviewed-by: Simon Horman Reviewed-by: Donald Hunter Tested-by: Donald Hunter Link: https://lore.kernel.org/r/20230524170712.2036128-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- tools/net/ynl/lib/ynl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index aa77bcae4807..3144f33196be 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -591,8 +591,9 @@ class YnlFamily(SpecFamily): print('Unexpected message: ' + repr(gm)) continue - rsp.append(self._decode(gm.raw_attrs, op.attr_set.name) - | gm.fixed_header_attrs) + rsp_msg = self._decode(gm.raw_attrs, op.attr_set.name) + rsp_msg.update(gm.fixed_header_attrs) + rsp.append(rsp_msg) if not rsp: return None -- cgit v1.2.3 From 5ac18889bde04ed2d4f2559da01e9b160234525c Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Sat, 27 May 2023 14:31:05 +0100 Subject: tools: ynl: Initialise fixed headers to 0 in genetlink-legacy This eliminates the need for e.g. --json '{"dp-ifindex":0}' which is not too big a deal for ovs but will get tiresome for fixed header structs that have many members. Signed-off-by: Donald Hunter Signed-off-by: Jakub Kicinski --- tools/net/ynl/lib/ynl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index 39a2296c0003..85ee6a4bee72 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -541,7 +541,7 @@ class YnlFamily(SpecFamily): if op.fixed_header: fixed_header_members = self.consts[op.fixed_header].members for m in fixed_header_members: - value = vals.pop(m.name) + value = vals.pop(m.name) if m.name in vals else 0 format = NlAttr.get_format(m.type, m.byte_order) msg += format.pack(value) for name, value in vals.items(): -- cgit v1.2.3 From 313a7a808ca8ca0fe08e2175eb145479bd86937e Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Sat, 27 May 2023 14:31:06 +0100 Subject: tools: ynl: Support enums in struct members in genetlink-legacy Support decoding scalars as enums in struct members for genetlink-legacy specs. Signed-off-by: Donald Hunter Signed-off-by: Jakub Kicinski --- Documentation/netlink/genetlink-legacy.yaml | 3 +++ tools/net/ynl/lib/nlspec.py | 2 ++ tools/net/ynl/lib/ynl.py | 6 +++++- 3 files changed, 10 insertions(+), 1 deletion(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/Documentation/netlink/genetlink-legacy.yaml b/Documentation/netlink/genetlink-legacy.yaml index d8f132114308..ac4350498f5e 100644 --- a/Documentation/netlink/genetlink-legacy.yaml +++ b/Documentation/netlink/genetlink-legacy.yaml @@ -127,6 +127,9 @@ properties: doc: description: Documentation for the struct member attribute. type: string + enum: + description: Name of the enum type used for the attribute. + type: string # End genetlink-legacy attribute-sets: diff --git a/tools/net/ynl/lib/nlspec.py b/tools/net/ynl/lib/nlspec.py index c624cdfde223..ada22b073aa2 100644 --- a/tools/net/ynl/lib/nlspec.py +++ b/tools/net/ynl/lib/nlspec.py @@ -228,11 +228,13 @@ class SpecStructMember(SpecElement): Attributes: type string, type of the member attribute byte_order string or None for native byte order + enum string, name of the enum definition """ def __init__(self, family, yaml): super().__init__(family, yaml) self.type = yaml['type'] self.byte_order = yaml.get('byte-order') + self.enum = yaml.get('enum') class SpecStruct(SpecElement): diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index 85ee6a4bee72..0692293447ad 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -412,7 +412,11 @@ class YnlFamily(SpecFamily): def _decode_binary(self, attr, attr_spec): if attr_spec.struct_name: - decoded = attr.as_struct(self.consts[attr_spec.struct_name]) + members = self.consts[attr_spec.struct_name] + decoded = attr.as_struct(members) + for m in members: + if m.enum: + self._decode_enum(decoded, m) elif attr_spec.sub_type: decoded = attr.as_c_array(attr_spec.sub_type) else: -- cgit v1.2.3 From d8eea68d913c20aabb3a97bc1e9ba61407a9e872 Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Fri, 23 Jun 2023 21:19:27 +0100 Subject: tools: ynl: add display-hint support to ynl Add support to the ynl tool for rendering output based on display-hint properties. Signed-off-by: Donald Hunter Link: https://lore.kernel.org/r/20230623201928.14275-3-donald.hunter@gmail.com Signed-off-by: Jakub Kicinski --- tools/net/ynl/lib/nlspec.py | 10 ++++++++++ tools/net/ynl/lib/ynl.py | 34 +++++++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 5 deletions(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/tools/net/ynl/lib/nlspec.py b/tools/net/ynl/lib/nlspec.py index 1ba572cae27b..0ff0d18666b2 100644 --- a/tools/net/ynl/lib/nlspec.py +++ b/tools/net/ynl/lib/nlspec.py @@ -154,6 +154,9 @@ class SpecAttr(SpecElement): is_multi bool, attr may repeat multiple times struct_name string, name of struct definition sub_type string, name of sub type + len integer, optional byte length of binary types + display_hint string, hint to help choose format specifier + when displaying the value """ def __init__(self, family, attr_set, yaml, value): super().__init__(family, yaml) @@ -164,6 +167,8 @@ class SpecAttr(SpecElement): self.struct_name = yaml.get('struct') self.sub_type = yaml.get('sub-type') self.byte_order = yaml.get('byte-order') + self.len = yaml.get('len') + self.display_hint = yaml.get('display-hint') class SpecAttrSet(SpecElement): @@ -229,12 +234,17 @@ class SpecStructMember(SpecElement): type string, type of the member attribute byte_order string or None for native byte order enum string, name of the enum definition + len integer, optional byte length of binary types + display_hint string, hint to help choose format specifier + when displaying the value """ def __init__(self, family, yaml): super().__init__(family, yaml) self.type = yaml['type'] self.byte_order = yaml.get('byte-order') self.enum = yaml.get('enum') + self.len = yaml.get('len') + self.display_hint = yaml.get('display-hint') class SpecStruct(SpecElement): diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index 3b343d6cbbc0..1b3a36fbb1c3 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -8,6 +8,8 @@ import socket import struct from struct import Struct import yaml +import ipaddress +import uuid from .nlspec import SpecFamily @@ -105,6 +107,20 @@ class NlAttr: else format.little return format.native + @classmethod + def formatted_string(cls, raw, display_hint): + if display_hint == 'mac': + formatted = ':'.join('%02x' % b for b in raw) + elif display_hint == 'hex': + formatted = bytes.hex(raw, ' ') + elif display_hint in [ 'ipv4', 'ipv6' ]: + formatted = format(ipaddress.ip_address(raw)) + elif display_hint == 'uuid': + formatted = str(uuid.UUID(bytes=raw)) + else: + formatted = raw + return formatted + def as_scalar(self, attr_type, byte_order=None): format = self.get_format(attr_type, byte_order) return format.unpack(self.raw)[0] @@ -124,10 +140,16 @@ class NlAttr: offset = 0 for m in members: # TODO: handle non-scalar members - format = self.get_format(m.type, m.byte_order) - decoded = format.unpack_from(self.raw, offset) - offset += format.size - value[m.name] = decoded[0] + if m.type == 'binary': + decoded = self.raw[offset:offset+m['len']] + offset += m['len'] + elif m.type in NlAttr.type_formats: + format = self.get_format(m.type, m.byte_order) + [ decoded ] = format.unpack_from(self.raw, offset) + offset += format.size + if m.display_hint: + decoded = self.formatted_string(decoded, m.display_hint) + value[m.name] = decoded return value def __repr__(self): @@ -385,7 +407,7 @@ class YnlFamily(SpecFamily): elif attr["type"] == 'string': attr_payload = str(value).encode('ascii') + b'\x00' elif attr["type"] == 'binary': - attr_payload = value + attr_payload = bytes.fromhex(value) elif attr['type'] in NlAttr.type_formats: format = NlAttr.get_format(attr['type'], attr.byte_order) attr_payload = format.pack(int(value)) @@ -421,6 +443,8 @@ class YnlFamily(SpecFamily): decoded = attr.as_c_array(attr_spec.sub_type) else: decoded = attr.as_bin() + if attr_spec.display_hint: + decoded = NlAttr.formatted_string(decoded, attr_spec.display_hint) return decoded def _decode(self, attrs, space): -- cgit v1.2.3