From d0efb5810b097ddd5463a9ce33f61e6b459dd7c0 Mon Sep 17 00:00:00 2001 From: sebaszm Date: Wed, 11 Mar 2026 13:49:13 +0100 Subject: [PATCH 1/5] [StubGen] Optimize MACAddress and fixed arrays --- ProxyStubGenerator/StubGenerator.py | 79 ++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/ProxyStubGenerator/StubGenerator.py b/ProxyStubGenerator/StubGenerator.py index f96948d4..6b565a28 100755 --- a/ProxyStubGenerator/StubGenerator.py +++ b/ProxyStubGenerator/StubGenerator.py @@ -413,8 +413,14 @@ def Convert(paramtype, retval, vars, hresult=False, allow_ptr = True): length_value = parsed[1] if isinstance(p, CppParser.Integer) and p.size == "char": - # char[] is sent as a buffer - lua_type.append_type(("BUFFER" + parsed[0]), param, paramtype) + if length_value: + element = Convert(paramtype, None, None, allow_ptr=False) + lua_type.append_type("ARRAY", param, paramtype) + lua_type.append("element", "{ %s }" % element.join()) + lua_type.append("count", length_value) + else: + # char[] is sent as a buffer + lua_type.append_type(("BUFFER" + parsed[0]), param, paramtype) else: element = Convert(paramtype, None, None, allow_ptr=False) @@ -891,6 +897,7 @@ def _FindLength(length_name, variable_name): self.length = _FindLength(self.identifier.meta.length, (name[1:] + "_Len")) self.max_length = _FindLength(self.identifier.meta.maxlength, (name[1:] + "_MaxLen")) self.is_buffer = ((self.length or self.max_length) and is_buffer and not self.is_array) + self.is_fixed_buffer = False if ((self.length or self.max_length) and is_array_pointer and not is_buffer): raise TypenameError(self.identifier, "'%s': variable-length arrays are not supported (use std::vector instead)" % self.trace_proto) @@ -915,6 +922,7 @@ def _FindLength(length_name, variable_name): # array of bytes, let's make it a buffer then self.is_array = False self.is_buffer = True + self.is_fixed_buffer = True self.max_length = self.length @@ -1034,11 +1042,11 @@ def _FindLength(length_name, variable_name): raise TypenameError(identifier, "'%s': output parameter must not be const" % self.trace_proto) if (not self.is_buffer and not self.is_array) and isinstance(self.kind, (CppParser.Integer, CppParser.BuiltinInteger)): - if not self.kind.IsFixed(): + if not self.kind.fixed and not self.kind.char: log.WarnLine(self.identifier, "'%s': integer is not fixed-width, use a stdint type" % self.trace_proto) if isinstance(self.kind, CppParser.Enum): - if not self.kind.type.Type().IsFixed(): + if not self.kind.type.Type().fixed: log.WarnLine(self.identifier, "'%s': underlying type of enumeration is not fixed-width integer, use a stdint type" % self.trace_proto) # Lastly handle restrict @@ -1242,12 +1250,15 @@ def read_rpc_type(self): # Raw buffers elif self.is_buffer: assert self.length or self.max_length, "Invalid type for buffer" - if self.max_length: - return "Buffer<%s>(%s, %s)" % (self.max_length.type_name, self.max_length.as_rvalue, self.as_lvalue) - elif self.length: - return "Buffer<%s>(%s, %s)" % (self.length.type_name, self.length.as_rvalue, self.as_lvalue) + if self.is_fixed_buffer: + return "Copy(%s, %s)" % (self.length.as_rvalue, self.as_lvalue) else: - Unreachable() + if self.max_length: + return "Buffer<%s>(%s, %s)" % (self.max_length.type_name, self.max_length.as_rvalue, self.as_lvalue) + elif self.length: + return "Buffer<%s>(%s, %s)" % (self.length.type_name, self.length.as_rvalue, self.as_lvalue) + else: + Unreachable() # Strings elif self.is_string: @@ -1255,7 +1266,7 @@ def read_rpc_type(self): # MacAddress elif isinstance(self.kind, CppParser.MacAddress): - return "Buffer(6, %s)" % self.as_lvalue + return "Copy(6, %s)" % self.as_lvalue # The integral types elif isinstance(self.kind, (CppParser.Integer, CppParser.Int24, CppParser.BuiltinInteger, CppParser.Enum)): @@ -1290,7 +1301,10 @@ def write_rpc_type(self): # Raw buffers elif self.is_buffer: assert self.max_length, "Invalid type for buffer " + self.name - return "Buffer<%s>(%s, %s)" % (self.length.type_name, self.length.as_rvalue, self.as_rvalue) + if self.is_fixed_buffer: + return "Copy(%s, %s)" % (self.length.as_rvalue, self.as_rvalue) + else: + return "Buffer<%s>(%s, %s)" % (self.length.type_name, self.length.as_rvalue, self.as_rvalue) # Strings elif self.is_string: @@ -1298,7 +1312,7 @@ def write_rpc_type(self): # MacAddress elif isinstance(self.kind, CppParser.MacAddress): - return "Buffer(6, %s)" % self.as_lvalue + return "Copy(6, %s)" % self.as_lvalue # The integral types elif isinstance(self.kind, (CppParser.Integer, CppParser.Int24, CppParser.Enum, CppParser.BuiltinInteger)): @@ -1659,24 +1673,39 @@ def _EmitAssignment(p): _name = p.as_rvalue buffer_param = "const_cast(%s)" % (p.type_name, _name) if not p.identifier_type.IsPointerToConst() else _name - emit.Line("%s = %s.LockBuffer<%s>(%s);" % (p.length.temporary, vars["reader"], p.length.type_name, buffer_param)) - emit.Line("%s.UnlockBuffer(%s);" % (vars["reader"], p.length.name)) + + if not p.suppress_type: + # we may reuse the wire buffer to skip a copy + if p.is_fixed_buffer: + assert p.length.value + emit.Line("%s = %s.LockFixedBuffer(%s, %s);" % (p.length.temporary, vars["reader"], buffer_param, p.length.value)) + emit.Line("ASSERT(%s == %s);" % (p.length.name, p.length.value)) + else: + emit.Line("%s = %s.LockBuffer<%s>(%s);" % (p.length.temporary, vars["reader"], p.length.type_name, buffer_param)) + + emit.Line("%s.UnlockBuffer(%s);" % (vars["reader"], p.length.name)) + else: + # inside a struct, have to copy :( + emit.Line("%s.%s;" % (vars["reader"], p.read_rpc_type)) if p.is_array: emit.Line("ASSERT(%s == (%s * sizeof(%s)));" % (p.length.name, p.identifier.array, p.type_name)) elif isinstance(p.kind, CppParser.MacAddress): - if not p.suppress_type: - length = AuxIdentifier(CppParser.Integer("uint8_t"), CppParser.Ref.VALUE | CppParser.Ref.CONST, (p.name[1:] + "Length")) - buffer = AuxIdentifier(CppParser.Integer("uint8_t"), CppParser.Ref.POINTER | CppParser.Ref.POINTER_TO_CONST, (p.name[1:] + "Buffer")) - emit.Line("%s{};" % buffer.temporary) + length = AuxIdentifier(CppParser.Integer("uint8_t"), CppParser.Ref.VALUE | CppParser.Ref.CONST, (p.name[1:] + "Length")) + buffer = AuxIdentifier(CppParser.Integer("uint8_t"), CppParser.Ref.POINTER | CppParser.Ref.POINTER_TO_CONST, (p.name[1:] + "Buffer")) + emit.Line("%s{};" % buffer.temporary) - emit.Line("%s = %s.LockBuffer(%s);" % (length.temporary, vars["reader"], buffer.name)) - emit.Line("ASSERT(%s == 6);" % (length.name)) - emit.Line("%s.UnlockBuffer(%s);" % (vars["reader"], length.name)) + # construct the mac directly from the wire buffer + emit.Line("%s = %s.LockFixedBuffer(%s, 6);" % (length.temporary, vars["reader"], buffer.name)) + emit.Line("ASSERT(%s == 6);" % (length.name)) + emit.Line("%s.UnlockBuffer(%s);" % (vars["reader"], length.name)) + if not p.suppress_type: emit.Line("%s{%s};" % (p.temporary_no_cv, buffer.as_rvalue)) + else: + emit.Line("%s = Core::MACAddress(%s);" % (p.name, buffer.as_rvalue)) else: _EmitAssignment(p) @@ -2507,7 +2536,7 @@ def EmitProxyMethod(index, method, interface_name, interface, prepared_params): EmitProxyMethodImplementation(index, method, interface_name, interface, retval, params, \ input_params, output_params, proxy_params, return_proxy_params) else: - EmitProxyMethodStubbed(index, methods, interface_name, retval) + EmitProxyMethodStubbed(index, method, interface_name, retval) emit.IndentDec() emit.Line("}") @@ -2579,10 +2608,10 @@ def EmitRegistration(announce_list): emit.Line("{") emit.IndentInc() - if EMIT_TRACES: - emit.Line("fprintf(stderr, \"*** Announcing %s interface methods...\\n\");" % Flatten(interface.obj.type, ns)) - if announce_list: + if EMIT_TRACES: + emit.Line("fprintf(stderr, \"*** Announcing %s interface methods...\\n\");" % Flatten(interface.obj.type, ns)) + security_options = [] security_var = "" From d32bea557182891f60e408baab084fa91e4d35ab Mon Sep 17 00:00:00 2001 From: sebaszm Date: Fri, 13 Mar 2026 08:50:10 +0100 Subject: [PATCH 2/5] also consider buffers with const length tag --- ProxyStubGenerator/StubGenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProxyStubGenerator/StubGenerator.py b/ProxyStubGenerator/StubGenerator.py index 6b565a28..5e9050e3 100755 --- a/ProxyStubGenerator/StubGenerator.py +++ b/ProxyStubGenerator/StubGenerator.py @@ -897,7 +897,7 @@ def _FindLength(length_name, variable_name): self.length = _FindLength(self.identifier.meta.length, (name[1:] + "_Len")) self.max_length = _FindLength(self.identifier.meta.maxlength, (name[1:] + "_MaxLen")) self.is_buffer = ((self.length or self.max_length) and is_buffer and not self.is_array) - self.is_fixed_buffer = False + self.is_fixed_buffer = self.is_buffer and ((self.length and self.length.value != None) or (not self.length and (self.max_length and self.max_length.value != None))) if ((self.length or self.max_length) and is_array_pointer and not is_buffer): raise TypenameError(self.identifier, "'%s': variable-length arrays are not supported (use std::vector instead)" % self.trace_proto) From 38b1bf277fe052d1775d380a4c88adf080b07b16 Mon Sep 17 00:00:00 2001 From: sebaszm Date: Mon, 16 Mar 2026 14:09:26 +0100 Subject: [PATCH 3/5] fix fixed buffer detection --- ProxyStubGenerator/StubGenerator.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ProxyStubGenerator/StubGenerator.py b/ProxyStubGenerator/StubGenerator.py index 5e9050e3..4f5a5889 100755 --- a/ProxyStubGenerator/StubGenerator.py +++ b/ProxyStubGenerator/StubGenerator.py @@ -897,7 +897,15 @@ def _FindLength(length_name, variable_name): self.length = _FindLength(self.identifier.meta.length, (name[1:] + "_Len")) self.max_length = _FindLength(self.identifier.meta.maxlength, (name[1:] + "_MaxLen")) self.is_buffer = ((self.length or self.max_length) and is_buffer and not self.is_array) - self.is_fixed_buffer = self.is_buffer and ((self.length and self.length.value != None) or (not self.length and (self.max_length and self.max_length.value != None))) + + try: + self.length_numeric_value = eval(self.length.value) if self.length else None + self.maxlength_numeric_value = eval(self.maxlength.value) if self.maxlength else None + except: + self.length_numeric_value = None + self.maxlength_numeric_value = None + + self.is_fixed_buffer = self.is_buffer and ((self.length_numeric_value != None) or (not self.length and (self.length_numeric_value!= None))) if ((self.length or self.max_length) and is_array_pointer and not is_buffer): raise TypenameError(self.identifier, "'%s': variable-length arrays are not supported (use std::vector instead)" % self.trace_proto) From d79b065c41eebff2ce0b51e83fa41f0f8e739d7a Mon Sep 17 00:00:00 2001 From: sebaszm Date: Tue, 17 Mar 2026 18:24:50 +0100 Subject: [PATCH 4/5] include more arrays for optimization --- ProxyStubGenerator/StubGenerator.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/ProxyStubGenerator/StubGenerator.py b/ProxyStubGenerator/StubGenerator.py index 4f5a5889..651f0171 100755 --- a/ProxyStubGenerator/StubGenerator.py +++ b/ProxyStubGenerator/StubGenerator.py @@ -314,6 +314,7 @@ def _Convert(size): value = 0 expr = "".join(length) + try: value = eval(expr) if value > 0xFF: @@ -776,9 +777,9 @@ def _FindLength(length_name, variable_name): matches = [v for v in self.identifier.parent.vars if v.name == length_name[2]] if matches: - return EmitIdentifier(interface, CppParser.Temporary(self.identifier.parent, ("%s %s" % (length_type, variable_name)).split(), ["sizeof(_%s)" % matches[0].name]), variable_name) + return EmitIdentifier(interface, CppParser.Temporary(self.identifier.parent, ("%s %s" % ("uint8_t", variable_name)).split(), ["sizeof(_%s)" % matches[0].name]), variable_name) else: - return EmitIdentifier(interface, CppParser.Temporary(self.identifier.parent, ("%s %s" % (length_type, variable_name)).split(), ["".join(length_name)]), variable_name) + return EmitIdentifier(interface, CppParser.Temporary(self.identifier.parent, ("%s %s" % ("uint8_t", variable_name)).split(), ["".join(length_name)]), variable_name) matches = [v for v in self.identifier.parent.vars if v.name == "".join(length_name)] @@ -889,8 +890,6 @@ def _FindLength(length_name, variable_name): self.interface_id = _FindLength(self.identifier.meta.interface, (name[1:] + "IntefaceId")) - # Is it a buffer? - is_array_pointer = (self.type.IsPointer() and not is_interface and not self.interface_id) is_buffer = is_array_pointer and isinstance(self.identifier_kind, CppParser.Integer) and (self.identifier_kind.size == "char") self.is_array = (self.identifier.array != None) @@ -898,14 +897,13 @@ def _FindLength(length_name, variable_name): self.max_length = _FindLength(self.identifier.meta.maxlength, (name[1:] + "_MaxLen")) self.is_buffer = ((self.length or self.max_length) and is_buffer and not self.is_array) - try: - self.length_numeric_value = eval(self.length.value) if self.length else None - self.maxlength_numeric_value = eval(self.maxlength.value) if self.maxlength else None - except: - self.length_numeric_value = None - self.maxlength_numeric_value = None + self.length_numeric_value = self.length.value if self.length and isinstance(self.length.value, int) else None + self.max_length_numeric_value = self.max_length.value if self.max_length and isinstance(self.max_length.value, int) else None + + numeric_length = (self.length_numeric_value and (not self.max_length or self.max_length_numeric_value)) \ + or (self.max_length_numeric_value and (self.length or self.length_numeric_value)) - self.is_fixed_buffer = self.is_buffer and ((self.length_numeric_value != None) or (not self.length and (self.length_numeric_value!= None))) + self.is_fixed_buffer = self.is_buffer and numeric_length if ((self.length or self.max_length) and is_array_pointer and not is_buffer): raise TypenameError(self.identifier, "'%s': variable-length arrays are not supported (use std::vector instead)" % self.trace_proto) From c181e85f67d45fe1d56d63f39911eae0982f2889 Mon Sep 17 00:00:00 2001 From: sebaszm Date: Wed, 18 Mar 2026 14:17:12 +0100 Subject: [PATCH 5/5] fix int24, out fixed arrays --- JsonGenerator/source/header_loader.py | 4 ++-- ProxyStubGenerator/StubGenerator.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/JsonGenerator/source/header_loader.py b/JsonGenerator/source/header_loader.py index 683d0098..0790d4fd 100644 --- a/JsonGenerator/source/header_loader.py +++ b/JsonGenerator/source/header_loader.py @@ -544,7 +544,7 @@ def ConvertType(var, quiet=False, meta=None, no_array=False): # C-style buffers converted to JSON arrays (no encode tag) elif isinstance(cppType, CppParser.Integer) and (cppType.size == "char") and not var.array: props = {} - props["items"] = ConvertParameter(var, is_member=True, quiet=quiet, no_array=True) + props["items"] = ConvertParameter(var, quiet=quiet, no_array=True) props["@length"] = " ".join(meta.length) props["@proto"] = var.ProtoFmt() @@ -562,7 +562,7 @@ def ConvertType(var, quiet=False, meta=None, no_array=False): log.WarnLine(var, "%s: large array size (%s elements)" % (var.name, var.array)) props = {} - props["items"] = ConvertParameter(var, quiet=quiet, is_member=True, no_array=True) + props["items"] = ConvertParameter(var, quiet=quiet, no_array=True) props["@arraysize"] = var.array props["@proto"] = var.ProtoFmt() props["range"] = [var.array, var.array] diff --git a/ProxyStubGenerator/StubGenerator.py b/ProxyStubGenerator/StubGenerator.py index 651f0171..fdbe5304 100755 --- a/ProxyStubGenerator/StubGenerator.py +++ b/ProxyStubGenerator/StubGenerator.py @@ -1275,11 +1275,11 @@ def read_rpc_type(self): return "Copy(6, %s)" % self.as_lvalue # The integral types - elif isinstance(self.kind, (CppParser.Integer, CppParser.Int24, CppParser.BuiltinInteger, CppParser.Enum)): + elif isinstance(self.kind, (CppParser.Integer, CppParser.BuiltinInteger, CppParser.Enum)): return "Number<%s>()" % self.type_name elif isinstance(self.kind, CppParser.Bool): return "Boolean()" - elif isinstance(self.kind, CppParser.Time): + elif isinstance(self.kind, (CppParser.Time, CppParser.Int24)): return "Number<%s>()" % self.target_type_name # Floating point types @@ -1321,8 +1321,10 @@ def write_rpc_type(self): return "Copy(6, %s)" % self.as_lvalue # The integral types - elif isinstance(self.kind, (CppParser.Integer, CppParser.Int24, CppParser.Enum, CppParser.BuiltinInteger)): + elif isinstance(self.kind, (CppParser.Integer, CppParser.Enum, CppParser.BuiltinInteger)): return "Number<%s>(%s)" % (self.type_name, self.as_rvalue) + elif isinstance(self.kind, (CppParser.Int24)): + return "Number<%s>(%s)" % (self.target_type_name, self.as_rvalue) elif isinstance(self.kind, CppParser.Bool): return "Boolean(%s)" % self.as_rvalue elif isinstance(self.kind, CppParser.Time):