From 3eb9b62b39b00ed15fef5b258213f5f46ffa7c50 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Sat, 27 Jun 2026 10:12:26 +0100 Subject: [PATCH 1/3] Add Python device interface generator Add a PythonGenerator class that produces a Python device interface module from device metadata, alongside the existing C# and firmware generators. The generated module targets the pyharp register and payload DSL and emits IntEnum and IntFlag types for group and bit masks, payload classes for structured and converter-backed registers, register classes, and an address-to-class REGISTER_MAP. Application devices merge the core register map, and member and field names follow canonical Python casing through the shared firmware naming convention. Add expected-output tests for the core and device metadata and a cross-stack interop test that writes Harp binary frames through the generated C# interface, then reads them back through the generated Python interface to verify the two stacks agree. --- .gitattributes | 1 + .github/workflows/Harp.Generators.yml | 13 + .gitignore | 5 + src/PyDevice.tt | 193 ++++++++++ src/Python.cs | 519 ++++++++++++++++++++++++++ src/PythonGenerator.cs | 70 ++++ tests/CompilerTestHelper.cs | 16 +- tests/ExpectedOutput/core.py | 250 +++++++++++++ tests/ExpectedOutput/device.py | 246 ++++++++++++ tests/Harp.Generators.Tests.csproj | 1 + tests/Python/converters.py | 41 ++ tests/Python/pyproject.toml | 13 + tests/Python/test_interop.py | 102 +++++ tests/PythonGeneratorTests.cs | 44 +++ tests/PythonInteropTests.cs | 285 ++++++++++++++ 15 files changed, 1798 insertions(+), 1 deletion(-) create mode 100644 src/PyDevice.tt create mode 100644 src/Python.cs create mode 100644 src/PythonGenerator.cs create mode 100644 tests/ExpectedOutput/core.py create mode 100644 tests/ExpectedOutput/device.py create mode 100644 tests/Python/converters.py create mode 100644 tests/Python/pyproject.toml create mode 100644 tests/Python/test_interop.py create mode 100644 tests/PythonGeneratorTests.cs create mode 100644 tests/PythonInteropTests.cs diff --git a/.gitattributes b/.gitattributes index 601a4e2..8f96242 100644 --- a/.gitattributes +++ b/.gitattributes @@ -23,6 +23,7 @@ *.yml text # Code +*.py text *.manifest text *.vsixmanifest text *.vstemplate text diff --git a/.github/workflows/Harp.Generators.yml b/.github/workflows/Harp.Generators.yml index 031d5d4..13a6cdb 100644 --- a/.github/workflows/Harp.Generators.yml +++ b/.github/workflows/Harp.Generators.yml @@ -92,6 +92,19 @@ jobs: - name: Test .NET 8 run: dotnet test --no-restore --no-build --configuration ${{matrix.configuration}} --verbosity normal --framework net8.0 + # ----------------------------------------------------------------------- Python interop test + # The .NET test run generates Harp binaries from the auto-generated C# interface; this step + # reads them back through the auto-generated Python interface to verify the two stacks agree. + - name: Set up uv + if: matrix.configuration == 'release' + uses: astral-sh/setup-uv@v5 + + - name: Test Python interop + if: matrix.configuration == 'release' + env: + HARP_INTEROP_OUTPUT: ${{github.workspace}}/artifacts/python-interop + run: uv run --project tests/Python pytest tests/Python/test_interop.py + # ----------------------------------------------------------------------- Collect artifacts - name: Collect NuGet packages uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index 8c2e811..e23fe8e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ .vs/ /artifacts/ + +# Python +__pycache__/ +*.py[cod] +uv.lock diff --git a/src/PyDevice.tt b/src/PyDevice.tt new file mode 100644 index 0000000..e2a66f1 --- /dev/null +++ b/src/PyDevice.tt @@ -0,0 +1,193 @@ +<#@ template debug="true" hostspecific="false" visibility="internal" language="C#" inherits="TemplateBase" #> +<#@ parameter name="DeviceMetadata" type="DeviceInfo" #> +<#@ import namespace="Harp.Generators" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Collections.Generic" #> +<#@ output extension=".py" #> +<# +var module = TemplateHelper.BuildPythonModule(DeviceMetadata); +var usesNumpy = module.Payloads.Count > 0; +#> +# This file was automatically generated and should not be edited directly. +# To make changes, edit the device metadata and regenerate the interface. + +<# +if (module.Enums.Count > 0) +{ +#> +import enum +<# +} +#> +from typing import Any, ClassVar +<# +if (usesNumpy) +{ +#> + +import numpy as np +<# +} +if (module.UsesNDArray) +{ +#> +from numpy.typing import NDArray +<# +} +#> +from harp.protocol import ( +<# +foreach (var import in module.ProtocolImports) +{ +#> + <#= import #>, +<# +} +#> +) +<# +if (module.IsApplicationDevice) +{ +#> +from harp.device import REGISTER_MAP as _CORE_REGISTER_MAP +<# +} +if (module.ExtensionImports.Count > 0) +{ +#> + +from .converters import ( +<# + foreach (var import in module.ExtensionImports) + { +#> + <#= import #>, +<# + } +#> +) +<# +} +#> +<# +foreach (var pythonEnum in module.Enums) +{ +#> + + +class <#= pythonEnum.Name #>(enum.<#= pythonEnum.IsFlag ? "IntFlag" : "IntEnum" #>): +<# + if (!string.IsNullOrEmpty(pythonEnum.Description)) + { +#> + """<#= pythonEnum.Description #>""" + +<# + } + foreach (var member in pythonEnum.Members) + { +#> + <#= member.Name #> = <#= member.Value #> +<# + if (!string.IsNullOrEmpty(member.Description)) + { +#> + """<#= member.Description #>""" +<# + } + } +} +#> +<# +foreach (var payload in module.Payloads) +{ + var length = payload.Length.HasValue ? $", length={payload.Length.Value}" : string.Empty; +#> + + +<# + if (!string.IsNullOrEmpty(payload.Converter)) + { +#> +class <#= payload.Name #>(AnonymousPayload, converter=<#= payload.Converter #>): +<# + } + else + { +#> +class <#= payload.Name #>(StructPayload[<#= payload.ElementType #>]<#= length #>): +<# + } + if (!string.IsNullOrEmpty(payload.Description)) + { +#> + """<#= payload.Description #>""" +<# + if (payload.Fields.Count > 0) WriteLine(""); + } + foreach (var field in payload.Fields) + { +#> + <#= field.Name #>: <#= field.Annotation #> = <#= field.Descriptor #> +<# + if (!string.IsNullOrEmpty(field.Description)) + { +#> + """<#= field.Description #>""" +<# + } + } +} +#> +<# +foreach (var register in module.Registers) +{ +#> + + +class <#= register.Name #>(<#= register.BaseClass #>): +<# + if (!string.IsNullOrEmpty(register.Description)) + { +#> + """<#= register.Description #>""" + +<# + } +#> + address: ClassVar[int] = <#= register.Address #> +<# + if (register.Kind == PythonRegisterKind.Array) + { +#> + length: ClassVar[int] = <#= register.Length #> +<# + } + else if (register.Kind == PythonRegisterKind.Struct) + { +#> + payload_type: ClassVar[PayloadType] = <#= register.PayloadType #> + payload_class = <#= register.PayloadClass #> +<# + } +} +#> + + +REGISTER_MAP: dict[int, type[RegisterBase[Any]]] = { +<# +if (module.IsApplicationDevice) +{ +#> + **_CORE_REGISTER_MAP, +<# +} +foreach (var register in module.Registers) +{ +#> + <#= register.Address #>: <#= register.Name #>, +<# +} +#> +} + diff --git a/src/Python.cs b/src/Python.cs new file mode 100644 index 0000000..c549f23 --- /dev/null +++ b/src/Python.cs @@ -0,0 +1,519 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Bonsai.Harp; + +namespace Harp.Generators; + +internal sealed class PythonModule +{ + public string Device = ""; + public SortedSet ProtocolImports = []; + public SortedSet ExtensionImports = []; + public bool UsesNDArray; + public bool IsApplicationDevice; + public List Enums = []; + public List Payloads = []; + public List Registers = []; +} + +internal sealed class PythonEnum +{ + public string Name = ""; + public bool IsFlag; + public string Description = ""; + public List Members = []; +} + +internal sealed class PythonEnumMember +{ + public string Name = ""; + public string Value = ""; + public string Description = ""; +} + +internal sealed class PythonPayload +{ + public string Name = ""; + public string Description = ""; + public string ElementType = ""; + public int? Length; + public string Converter = ""; + public string UnwrapType = ""; + public List Fields = []; +} + +internal sealed class PythonField +{ + public string Name = ""; + public string Annotation = ""; + public string Descriptor = ""; + public string Description = ""; +} + +internal enum PythonRegisterKind +{ + Scalar, + Array, + Struct +} + +internal sealed class PythonRegister +{ + public string Name = ""; + public int Address; + public PythonRegisterKind Kind; + public string BaseClass = ""; + public int Length; + public string PayloadType = ""; + public string PayloadClass = ""; + public string Description = ""; +} + +internal static partial class TemplateHelper +{ + static readonly Dictionary PrimitiveNumpyTypes = new() + { + { "byte", "np.uint8" }, + { "sbyte", "np.int8" }, + { "short", "np.int16" }, + { "ushort", "np.uint16" }, + { "int", "np.int32" }, + { "uint", "np.uint32" }, + { "long", "np.int64" }, + { "ulong", "np.uint64" }, + { "float", "np.float32" }, + }; + + static readonly Dictionary PrimitiveSizes = new() + { + { "byte", 1 }, { "sbyte", 1 }, + { "short", 2 }, { "ushort", 2 }, + { "int", 4 }, { "uint", 4 }, + { "long", 8 }, { "ulong", 8 }, + { "float", 4 }, + }; + + static readonly HashSet StandardConverterInterfaceTypes = ["HarpVersion"]; + + const int CoreRegisterAddressLimit = 32; + + static void AddConverterImports(PythonModule module, string domainType, string domainConverter) + { + if (StandardConverterInterfaceTypes.Contains(domainType)) + { + module.ProtocolImports.Add(domainType); + module.ProtocolImports.Add(domainConverter); + } + else + { + module.ExtensionImports.Add(domainType); + module.ExtensionImports.Add(domainConverter); + } + } + + public static string GetPythonScalarType(PayloadType payloadType) + { + return payloadType switch + { + PayloadType.U8 => "U8", + PayloadType.S8 => "S8", + PayloadType.U16 => "U16", + PayloadType.S16 => "S16", + PayloadType.U32 => "U32", + PayloadType.S32 => "S32", + PayloadType.U64 => "U64", + PayloadType.S64 => "S64", + PayloadType.Float => "Float", + _ => throw new ArgumentOutOfRangeException(nameof(payloadType)), + }; + } + + public static string GetPythonNumpyType(PayloadType payloadType) + { + return payloadType switch + { + PayloadType.U8 => "np.uint8", + PayloadType.S8 => "np.int8", + PayloadType.U16 => "np.uint16", + PayloadType.S16 => "np.int16", + PayloadType.U32 => "np.uint32", + PayloadType.S32 => "np.int32", + PayloadType.U64 => "np.uint64", + PayloadType.S64 => "np.int64", + PayloadType.Float => "np.float32", + _ => throw new ArgumentOutOfRangeException(nameof(payloadType)), + }; + } + + static bool IsPrimitiveInterfaceType(string interfaceType) + { + return PrimitiveNumpyTypes.ContainsKey(interfaceType); + } + + static string GetMemberTypeName(PayloadMemberInfo member) + { + if (!string.IsNullOrEmpty(member.InterfaceType)) return member.InterfaceType; + else if (!string.IsNullOrEmpty(member.MaskType)) return member.MaskType; + else return string.Empty; + } + + public static string GetPythonFieldName(string name) + { + return FirmwareNamingConvention.Instance.Apply(name).ToLowerInvariant(); + } + + public static PythonModule BuildPythonModule(DeviceInfo deviceMetadata) + { + var module = new PythonModule { Device = deviceMetadata.Device }; + + foreach (var bitMask in deviceMetadata.BitMasks) + module.Enums.Add(BuildPythonEnum(bitMask.Key, bitMask.Value.Description, bitMask.Value.Bits, isFlag: true)); + foreach (var groupMask in deviceMetadata.GroupMasks) + module.Enums.Add(BuildPythonEnum(groupMask.Key, groupMask.Value.Description, groupMask.Value.Values, isFlag: false)); + + foreach (var registerMetadata in deviceMetadata.Registers) + BuildPythonRegister(module, registerMetadata.Key, registerMetadata.Value, deviceMetadata); + + module.ProtocolImports.Add("RegisterBase"); + module.IsApplicationDevice = deviceMetadata.Registers.Values.All(register => register.Address >= CoreRegisterAddressLimit); + return module; + } + + static PythonEnum BuildPythonEnum(string name, string description, Dictionary values, bool isFlag) + { + var pythonEnum = new PythonEnum { Name = name, IsFlag = isFlag, Description = description }; + foreach (var value in values) + { + if (isFlag && value.Value.Value == 0) continue; + pythonEnum.Members.Add(new PythonEnumMember + { + Name = FirmwareNamingConvention.Instance.Apply(value.Key), + Value = isFlag + ? $"0x{value.Value.Value:X}" + : value.Value.Value.ToString(CultureInfo.InvariantCulture), + Description = value.Value.Description + }); + } + return pythonEnum; + } + + static void BuildPythonRegister(PythonModule module, string name, RegisterInfo register, DeviceInfo deviceMetadata) + { + var hasMask = !string.IsNullOrEmpty(register.MaskType); + var isString = register.InterfaceType == "string"; + var isCustom = register.HasConverter + || (!string.IsNullOrEmpty(register.InterfaceType) && !IsPrimitiveInterfaceType(register.InterfaceType) && !isString); + var className = register.Visibility == RegisterVisibility.Private ? $"_{name}" : name; + + if (register.PayloadSpec == null && !hasMask && !isString && !isCustom) + { + var isArray = register.Length > 0; + module.ProtocolImports.Add(isArray ? $"Register{GetPythonScalarType(register.Type)}Array" : $"Register{GetPythonScalarType(register.Type)}"); + module.Registers.Add(new PythonRegister + { + Name = className, + Address = register.Address, + Kind = isArray ? PythonRegisterKind.Array : PythonRegisterKind.Scalar, + BaseClass = isArray ? $"Register{GetPythonScalarType(register.Type)}Array" : $"Register{GetPythonScalarType(register.Type)}", + Length = register.Length, + Description = register.Description + }); + return; + } + + module.ProtocolImports.Add("RegisterBase"); + module.ProtocolImports.Add("PayloadType"); + + var payloadName = register.PayloadSpec != null && !string.IsNullOrEmpty(register.InterfaceType) + ? register.InterfaceType + : $"{name}Payload"; + var payload = module.Payloads.Find(existing => existing.Name == payloadName); + if (payload == null) + { + payload = BuildPythonPayload(module, payloadName, register, deviceMetadata); + module.Payloads.Add(payload); + } + + var typeParameter = !string.IsNullOrEmpty(payload.UnwrapType) ? payload.UnwrapType : payload.Name; + module.Registers.Add(new PythonRegister + { + Name = className, + Address = register.Address, + Kind = PythonRegisterKind.Struct, + BaseClass = $"RegisterBase[{typeParameter}]", + PayloadType = $"PayloadType.{GetPythonScalarType(register.Type)}", + PayloadClass = payload.Name, + Description = register.Description + }); + } + + static PythonPayload BuildPythonPayload(PythonModule module, string payloadName, RegisterInfo register, DeviceInfo deviceMetadata) + { + var elementType = GetPythonNumpyType(register.Type); + var elementSize = GetPayloadTypeSize(register.Type); + var payload = new PythonPayload + { + Name = payloadName, + Description = $"Represents the payload of the {RemoveSuffix(payloadName, "Payload")} register.", + ElementType = elementType, + Length = register.Length > 0 ? register.Length : null + }; + + if (register.PayloadSpec != null) + { + module.ProtocolImports.Add("StructPayload"); + foreach (var member in register.PayloadSpec) + { + var field = BuildPythonField(module, member.Key, member.Value, register, deviceMetadata, elementType, elementSize); + field.Description = member.Value.Description; + payload.Fields.Add(field); + } + return payload; + } + + if (!string.IsNullOrEmpty(register.MaskType)) + { + module.ProtocolImports.Add("StructPayload"); + BuildMaskRegisterFields(module, payload, register, deviceMetadata, elementType, elementSize); + } + else if (register.InterfaceType == "string") + { + module.ProtocolImports.Add("AnonymousPayload"); + module.ProtocolImports.Add("StringConverter"); + payload.Converter = $"StringConverter({Math.Max(1, register.Length) * elementSize})"; + payload.UnwrapType = "str"; + } + else + { + module.ProtocolImports.Add("AnonymousPayload"); + var domainType = register.InterfaceType; + var domainConverter = $"{domainType}Converter"; + AddConverterImports(module, domainType, domainConverter); + payload.Converter = $"{domainConverter}({elementType})"; + payload.UnwrapType = domainType; + } + + return payload; + } + + static void BuildMaskRegisterFields( + PythonModule module, + PythonPayload payload, + RegisterInfo register, + DeviceInfo deviceMetadata, + string elementType, + int elementSize) + { + var elementMask = (1L << (elementSize * 8)) - 1; + if (deviceMetadata.GroupMasks.ContainsKey(register.MaskType)) + { + module.ProtocolImports.Add("GroupMask"); + payload.Fields.Add(new PythonField + { + Name = "value", + Annotation = register.MaskType, + Descriptor = $"GroupMask(enum={register.MaskType}, mask=0x{elementMask:X})" + }); + } + else if (deviceMetadata.BitMasks.TryGetValue(register.MaskType, out var bitMask)) + { + module.ProtocolImports.Add("BitFlag"); + foreach (var bit in bitMask.Bits) + { + if (bit.Value.Value == 0 || bit.Value.Value > elementMask) continue; + payload.Fields.Add(new PythonField + { + Name = GetPythonFieldName(bit.Key), + Annotation = "bool", + Descriptor = $"BitFlag(mask=0x{bit.Value.Value:X})", + Description = bit.Value.Description + }); + } + } + } + + static PythonField BuildPythonField( + PythonModule module, + string name, + PayloadMemberInfo member, + RegisterInfo register, + DeviceInfo deviceMetadata, + string elementType, + int elementSize) + { + var offset = member.Offset.GetValueOrDefault(0); + var offsetArgument = offset > 0 ? $", offset={offset}" : string.Empty; + var typeName = GetMemberTypeName(member); + var defaultArgument = GetPythonDefaultArgument(member, typeName, register, deviceMetadata); + var converterBaseName = name; + name = GetPythonFieldName(name); + + if (deviceMetadata.GroupMasks.ContainsKey(typeName)) + { + module.ProtocolImports.Add("GroupMask"); + var elementMask = (1L << (elementSize * 8)) - 1; + var enumMask = member.Mask.GetValueOrDefault((int)elementMask); + return new PythonField + { + Name = name, + Annotation = typeName, + Descriptor = $"GroupMask(enum={typeName}, mask=0x{enumMask:X}{offsetArgument}{defaultArgument})" + }; + } + + if (member.Mask.HasValue) + { + var mask = member.Mask.GetValueOrDefault(); + if (member.InterfaceType == "bool") + { + module.ProtocolImports.Add("BitFlag"); + return new PythonField + { + Name = name, + Annotation = "bool", + Descriptor = $"BitFlag(mask=0x{mask:X}{offsetArgument}{defaultArgument})" + }; + } + + module.ProtocolImports.Add("Field"); + module.ProtocolImports.Add("IdentityConverter"); + var numpyType = GetMemberNumpyType(member, register); + return new PythonField + { + Name = name, + Annotation = numpyType, + Descriptor = $"Field(IdentityConverter({numpyType}), mask=0x{mask:X}{offsetArgument}{defaultArgument})" + }; + } + + module.ProtocolImports.Add("Field"); + var memberLength = member.Length.GetValueOrDefault(0); + var span = Math.Max(1, memberLength) * elementSize; + + if (member.InterfaceType == "string") + { + module.ProtocolImports.Add("StringConverter"); + return new PythonField + { + Name = name, + Annotation = "str", + Descriptor = $"Field(StringConverter({span}){offsetArgument}{defaultArgument})" + }; + } + + if (member.InterfaceType == "bool") + { + module.ProtocolImports.Add("BoolConverter"); + return new PythonField + { + Name = name, + Annotation = "bool", + Descriptor = $"Field(BoolConverter(){offsetArgument}{defaultArgument})" + }; + } + + if (string.IsNullOrEmpty(member.InterfaceType)) + { + module.ProtocolImports.Add("IdentityConverter"); + if (memberLength > 1) + { + module.UsesNDArray = true; + return new PythonField + { + Name = name, + Annotation = $"NDArray[{elementType}]", + Descriptor = $"Field(IdentityConverter(np.dtype(({elementType}, ({memberLength},)))){offsetArgument}{defaultArgument})" + }; + } + + return new PythonField + { + Name = name, + Annotation = elementType, + Descriptor = $"Field(IdentityConverter({elementType}){offsetArgument}{defaultArgument})" + }; + } + + if (IsPrimitiveInterfaceType(member.InterfaceType) && PrimitiveSizes[member.InterfaceType] == span) + { + module.ProtocolImports.Add("IdentityConverter"); + var numpyType = PrimitiveNumpyTypes[member.InterfaceType]; + return new PythonField + { + Name = name, + Annotation = numpyType, + Descriptor = $"Field(IdentityConverter({numpyType}){offsetArgument}{defaultArgument})" + }; + } + + if (!IsPrimitiveInterfaceType(member.InterfaceType)) + { + var domainType = member.InterfaceType; + var domainConverter = $"{domainType}Converter"; + AddConverterImports(module, domainType, domainConverter); + return new PythonField + { + Name = name, + Annotation = domainType, + Descriptor = $"Field({domainConverter}({elementType}){offsetArgument}{defaultArgument})" + }; + } + + var converterName = $"{converterBaseName}Converter"; + module.ExtensionImports.Add(converterName); + return new PythonField + { + Name = name, + Annotation = PrimitiveNumpyTypes[member.InterfaceType], + Descriptor = $"Field({converterName}(){offsetArgument}{defaultArgument})" + }; + } + + static string GetMemberNumpyType(PayloadMemberInfo member, RegisterInfo register) + { + if (!string.IsNullOrEmpty(member.InterfaceType) && PrimitiveNumpyTypes.TryGetValue(member.InterfaceType, out var numpyType)) + return numpyType; + return GetPythonNumpyType(register.Type); + } + + static string GetMemberAnnotation(PayloadMemberInfo member, RegisterInfo register) + { + return GetMemberNumpyType(member, register); + } + + static string GetPythonDefaultArgument(PayloadMemberInfo member, string typeName, RegisterInfo register, DeviceInfo deviceMetadata) + { + var defaultValue = member.DefaultValue ?? member.MinValue; + if (!defaultValue.HasValue || member.Length.GetValueOrDefault() > 1) + return string.Empty; + + var value = defaultValue.GetValueOrDefault(); + if (deviceMetadata.GroupMasks.TryGetValue(typeName, out var groupMask)) + { + foreach (var entry in groupMask.Values) + { + if (entry.Value.Value == (int)value) + return $", default={typeName}.{FirmwareNamingConvention.Instance.Apply(entry.Key)}"; + } + return $", default={(long)value}"; + } + + if (member.InterfaceType == "bool") + return $", default={(value != 0 ? "True" : "False")}"; + + if (member.InterfaceType == "string" || member.HasConverter || + (!string.IsNullOrEmpty(member.InterfaceType) && !IsPrimitiveInterfaceType(member.InterfaceType))) + return string.Empty; + + return $", default={GetMemberNumpyType(member, register)}({FormatNumericLiteral(value)})"; + } + + static string FormatNumericLiteral(float value) + { + return value == Math.Floor(value) + ? ((long)value).ToString(CultureInfo.InvariantCulture) + : value.ToString(CultureInfo.InvariantCulture); + } +} diff --git a/src/PythonGenerator.cs b/src/PythonGenerator.cs new file mode 100644 index 0000000..c716cb7 --- /dev/null +++ b/src/PythonGenerator.cs @@ -0,0 +1,70 @@ +using System.CodeDom.Compiler; +using System.Collections; +using System.Collections.Generic; + +namespace Harp.Generators; + +/// +/// Provides automatic generation of Python device interface implementations. +/// +public sealed class PythonGenerator +{ + readonly PyDevice _deviceTemplate = new(); + readonly CompilerErrorCollection errors = []; + + /// + /// Initializes a new instance of the class with the + /// specified device metadata. + /// + /// The device metadata object. + public PythonGenerator(DeviceInfo deviceMetadata) + { + var session = new Dictionary + { + { "DeviceMetadata", deviceMetadata } + }; + _deviceTemplate.Initialize(PythonImplementation.DeviceFileName, errors, session); + } + + /// + /// Gets the collection of errors emitted during the code generation process. + /// + public CompilerErrorCollection Errors => errors; + + /// + /// Generates a Python device interface implementation complying with the specified metadata file. + /// + /// The generated device interface implementation. + public PythonImplementation GenerateImplementation() => + new(Device: _deviceTemplate.TransformText()); +} + +/// +/// Represents the generated Python device interface implementation. +/// +/// The generated source code implementing the device register interface. +public record struct PythonImplementation(string Device) + : IEnumerable> +{ + /// + /// Represents the default name for the file storing the device register interface source code. + /// + public const string DeviceFileName = "device.py"; + + /// + /// Returns an enumerator that iterates through all the source code files in the + /// generated implementation. + /// + /// + /// An enumerator that can be used to iterate through the generated implementation files. + /// + public readonly IEnumerator> GetEnumerator() + { + yield return new(DeviceFileName, Device); + } + + readonly IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } +} diff --git a/tests/CompilerTestHelper.cs b/tests/CompilerTestHelper.cs index 4d3409d..b0eadad 100644 --- a/tests/CompilerTestHelper.cs +++ b/tests/CompilerTestHelper.cs @@ -1,4 +1,5 @@ -using Basic.Reference.Assemblies; +using System.Reflection; +using Basic.Reference.Assemblies; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -8,6 +9,17 @@ namespace Harp.Generators.Tests; internal static class CompilerTestHelper { public static void CompileFromSource(params string[] code) + { + Compile(code); + } + + public static Assembly CompileAndLoadFromSource(params string[] code) + { + var bytes = Compile(code); + return Assembly.Load(bytes); + } + + static byte[] Compile(string[] code) { var options = CSharpParseOptions.Default; var syntaxTrees = Array.ConvertAll(code, text => CSharpSyntaxTree.ParseText(text, options)); @@ -38,5 +50,7 @@ public static void CompileFromSource(params string[] code) .ToList(); Assert.Fail(string.Join(Environment.NewLine, errorMessages)); } + + return memoryStream.ToArray(); } } diff --git a/tests/ExpectedOutput/core.py b/tests/ExpectedOutput/core.py new file mode 100644 index 0000000..d184dbf --- /dev/null +++ b/tests/ExpectedOutput/core.py @@ -0,0 +1,250 @@ +# This file was automatically generated and should not be edited directly. +# To make changes, edit the device metadata and regenerate the interface. + +import enum +from typing import Any, ClassVar + +import numpy as np +from harp.protocol import ( + AnonymousPayload, + BitFlag, + GroupMask, + PayloadType, + RegisterBase, + RegisterU16, + RegisterU32, + RegisterU8, + StringConverter, + StructPayload, +) + + +class ResetFlags(enum.IntFlag): + """Specifies the behavior of the non-volatile registers when resetting the device.""" + + RESTORE_DEFAULT = 0x1 + """The device will boot with all the registers reset to their default factory values.""" + RESTORE_EEPROM = 0x2 + """The device will boot and restore all the registers to the values stored in non-volatile memory.""" + SAVE = 0x4 + """The device will boot and save all the current register values to non-volatile memory.""" + RESTORE_NAME = 0x8 + """The device will boot with the default device name.""" + UPDATE_FIRMWARE = 0x20 + """The device will enter firmware update mode.""" + BOOT_FROM_DEFAULT = 0x40 + """Specifies that the device has booted from default factory values.""" + BOOT_FROM_EEPROM = 0x80 + """Specifies that the device has booted from non-volatile values stored in EEPROM.""" + + +class ClockConfigurationFlags(enum.IntFlag): + """Specifies configuration flags for the device synchronization clock.""" + + CLOCK_REPEATER = 0x1 + """The device will repeat the clock synchronization signal to the clock output connector, if available.""" + CLOCK_GENERATOR = 0x2 + """The device resets and generates the clock synchronization signal on the clock output connector, if available.""" + REPEATER_CAPABILITY = 0x8 + """Specifies the device has the capability to repeat the clock synchronization signal to the clock output connector.""" + GENERATOR_CAPABILITY = 0x10 + """Specifies the device has the capability to generate the clock synchronization signal to the clock output connector.""" + CLOCK_UNLOCK = 0x40 + """The device will unlock the timestamp register counter and will accept commands to set new timestamp values.""" + CLOCK_LOCK = 0x80 + """The device will lock the timestamp register counter and will not accept commands to set new timestamp values.""" + + +class OperationMode(enum.IntEnum): + """Specifies the operation mode of the device.""" + + STANDBY = 0 + """Disable all event reporting on the device.""" + ACTIVE = 1 + """Event detection is enabled. Only enabled events are reported by the device.""" + SPEED = 3 + """The device enters speed mode.""" + + +class EnableFlag(enum.IntEnum): + """Specifies whether a specific register flag is enabled or disabled.""" + + DISABLED = 0 + """Specifies that the flag is disabled.""" + ENABLED = 1 + """Specifies that the flag is enabled.""" + + +class OperationControlPayload(StructPayload[np.uint8]): + """Represents the payload of the OperationControl register.""" + + operation_mode: OperationMode = GroupMask(enum=OperationMode, mask=0x3) + """Specifies the operation mode of the device.""" + dump_registers: bool = BitFlag(mask=0x8) + """Specifies whether the device should report the content of all registers on initialization.""" + mute_replies: bool = BitFlag(mask=0x10) + """Specifies whether the replies to all commands will be muted, i.e. not sent by the device.""" + visual_indicators: EnableFlag = GroupMask(enum=EnableFlag, mask=0x20) + """Specifies the state of all visual indicators on the device.""" + operation_led: EnableFlag = GroupMask(enum=EnableFlag, mask=0x40) + """Specifies whether the device state LED should report the operation mode of the device.""" + heartbeat: EnableFlag = GroupMask(enum=EnableFlag, mask=0x80) + """Specifies whether the device should report the content of the seconds register each second.""" + + +class ResetDevicePayload(StructPayload[np.uint8]): + """Represents the payload of the ResetDevice register.""" + + restore_default: bool = BitFlag(mask=0x1) + """The device will boot with all the registers reset to their default factory values.""" + restore_eeprom: bool = BitFlag(mask=0x2) + """The device will boot and restore all the registers to the values stored in non-volatile memory.""" + save: bool = BitFlag(mask=0x4) + """The device will boot and save all the current register values to non-volatile memory.""" + restore_name: bool = BitFlag(mask=0x8) + """The device will boot with the default device name.""" + update_firmware: bool = BitFlag(mask=0x20) + """The device will enter firmware update mode.""" + boot_from_default: bool = BitFlag(mask=0x40) + """Specifies that the device has booted from default factory values.""" + boot_from_eeprom: bool = BitFlag(mask=0x80) + """Specifies that the device has booted from non-volatile values stored in EEPROM.""" + + +class DeviceNamePayload(AnonymousPayload, converter=StringConverter(25)): + """Represents the payload of the DeviceName register.""" + + +class ClockConfigurationPayload(StructPayload[np.uint8]): + """Represents the payload of the ClockConfiguration register.""" + + clock_repeater: bool = BitFlag(mask=0x1) + """The device will repeat the clock synchronization signal to the clock output connector, if available.""" + clock_generator: bool = BitFlag(mask=0x2) + """The device resets and generates the clock synchronization signal on the clock output connector, if available.""" + repeater_capability: bool = BitFlag(mask=0x8) + """Specifies the device has the capability to repeat the clock synchronization signal to the clock output connector.""" + generator_capability: bool = BitFlag(mask=0x10) + """Specifies the device has the capability to generate the clock synchronization signal to the clock output connector.""" + clock_unlock: bool = BitFlag(mask=0x40) + """The device will unlock the timestamp register counter and will accept commands to set new timestamp values.""" + clock_lock: bool = BitFlag(mask=0x80) + """The device will lock the timestamp register counter and will not accept commands to set new timestamp values.""" + + +class WhoAmI(RegisterU16): + """Specifies the identity class of the device.""" + + address: ClassVar[int] = 0 + + +class HardwareVersionHigh(RegisterU8): + """Specifies the major hardware version of the device.""" + + address: ClassVar[int] = 1 + + +class HardwareVersionLow(RegisterU8): + """Specifies the minor hardware version of the device.""" + + address: ClassVar[int] = 2 + + +class AssemblyVersion(RegisterU8): + """Specifies the version of the assembled components in the device.""" + + address: ClassVar[int] = 3 + + +class CoreVersionHigh(RegisterU8): + """Specifies the major version of the Harp core implemented by the device.""" + + address: ClassVar[int] = 4 + + +class CoreVersionLow(RegisterU8): + """Specifies the minor version of the Harp core implemented by the device.""" + + address: ClassVar[int] = 5 + + +class FirmwareVersionHigh(RegisterU8): + """Specifies the major version of the Harp core implemented by the device.""" + + address: ClassVar[int] = 6 + + +class FirmwareVersionLow(RegisterU8): + """Specifies the minor version of the Harp core implemented by the device.""" + + address: ClassVar[int] = 7 + + +class TimestampSeconds(RegisterU32): + """Stores the integral part of the system timestamp, in seconds.""" + + address: ClassVar[int] = 8 + + +class TimestampMicroseconds(RegisterU16): + """Stores the fractional part of the system timestamp, in microseconds.""" + + address: ClassVar[int] = 9 + + +class OperationControl(RegisterBase[OperationControlPayload]): + """Stores the configuration mode of the device.""" + + address: ClassVar[int] = 10 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = OperationControlPayload + + +class ResetDevice(RegisterBase[ResetDevicePayload]): + """Resets the device and saves non-volatile registers.""" + + address: ClassVar[int] = 11 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = ResetDevicePayload + + +class DeviceName(RegisterBase[str]): + """Stores the user-specified device name.""" + + address: ClassVar[int] = 12 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = DeviceNamePayload + + +class SerialNumber(RegisterU16): + """Specifies the unique serial number of the device.""" + + address: ClassVar[int] = 13 + + +class ClockConfiguration(RegisterBase[ClockConfigurationPayload]): + """Specifies the configuration for the device synchronization clock.""" + + address: ClassVar[int] = 14 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = ClockConfigurationPayload + + +REGISTER_MAP: dict[int, type[RegisterBase[Any]]] = { + 0: WhoAmI, + 1: HardwareVersionHigh, + 2: HardwareVersionLow, + 3: AssemblyVersion, + 4: CoreVersionHigh, + 5: CoreVersionLow, + 6: FirmwareVersionHigh, + 7: FirmwareVersionLow, + 8: TimestampSeconds, + 9: TimestampMicroseconds, + 10: OperationControl, + 11: ResetDevice, + 12: DeviceName, + 13: SerialNumber, + 14: ClockConfiguration, +} + diff --git a/tests/ExpectedOutput/device.py b/tests/ExpectedOutput/device.py new file mode 100644 index 0000000..64760bf --- /dev/null +++ b/tests/ExpectedOutput/device.py @@ -0,0 +1,246 @@ +# This file was automatically generated and should not be edited directly. +# To make changes, edit the device metadata and regenerate the interface. + +import enum +from typing import Any, ClassVar + +import numpy as np +from numpy.typing import NDArray +from harp.protocol import ( + AnonymousPayload, + BitFlag, + BoolConverter, + Field, + GroupMask, + HarpVersion, + HarpVersionConverter, + IdentityConverter, + PayloadType, + RegisterBase, + RegisterS32, + RegisterU16, + RegisterU8, + StringConverter, + StructPayload, +) +from harp.device import REGISTER_MAP as _CORE_REGISTER_MAP + +from .converters import ( + DataConverter, +) + + +class PortDigitalIOS(enum.IntFlag): + DIO0 = 0x1 + DIO1 = 0x2 + DIO2 = 0x4 + DIO3 = 0x8 + DI_PORT0 = 0x100 + TEST_DI_PORT1 = 0x200 + SUPPLY_PORT0 = 0x400 + PORT_DIO1 = 0x800 + + +class PwmPort(enum.IntEnum): + PWM0 = 1 + PWM1 = 2 + PWM2 = 4 + PWM3 = 10 + + +class EncoderModeMask(enum.IntEnum): + """Specifies the type of encoder mode.""" + + POSITION = 0 + DISPLACEMENT = 1 + + +class AnalogDataPayload(StructPayload[np.float32], length=6): + """Represents the payload of the AnalogData register.""" + + analog0: np.float32 = Field(IdentityConverter(np.float32)) + analog1: np.float32 = Field(IdentityConverter(np.float32), offset=1) + analog2: np.float32 = Field(IdentityConverter(np.float32), offset=2) + accelerometer: NDArray[np.float32] = Field(IdentityConverter(np.dtype((np.float32, (3,)))), offset=3) + + +class ComplexConfigurationPayload(StructPayload[np.uint8], length=17): + """Represents the payload of the ComplexConfiguration register.""" + + pwm_port: PwmPort = GroupMask(enum=PwmPort, mask=0xFF) + duty_cycle: np.float32 = Field(IdentityConverter(np.float32), offset=4) + frequency: np.float32 = Field(IdentityConverter(np.float32), offset=8) + events_enabled: bool = Field(BoolConverter(), offset=12) + delta: np.uint32 = Field(IdentityConverter(np.uint32), offset=13) + + +class VersionPayload(StructPayload[np.uint8], length=32): + """Represents the payload of the Version register.""" + + protocol_version: HarpVersion = Field(HarpVersionConverter(np.uint8)) + firmware_version: HarpVersion = Field(HarpVersionConverter(np.uint8), offset=3) + hardware_version: HarpVersion = Field(HarpVersionConverter(np.uint8), offset=6) + core_id: str = Field(StringConverter(3), offset=9) + interface_hash: NDArray[np.uint8] = Field(IdentityConverter(np.dtype((np.uint8, (20,)))), offset=12) + + +class CustomPayloadPayload(AnonymousPayload, converter=HarpVersionConverter(np.uint32)): + """Represents the payload of the CustomPayload register.""" + + +class CustomRawPayloadPayload(AnonymousPayload, converter=HarpVersionConverter(np.uint32)): + """Represents the payload of the CustomRawPayload register.""" + + +class CustomMemberConverterPayload(StructPayload[np.uint8], length=3): + """Represents the payload of the CustomMemberConverter register.""" + + header: np.uint8 = Field(IdentityConverter(np.uint8)) + data: np.int32 = Field(DataConverter(), offset=1) + + +class BitmaskSplitterPayload(StructPayload[np.uint8]): + """Represents the payload of the BitmaskSplitter register.""" + + low: np.int32 = Field(IdentityConverter(np.int32), mask=0xF) + high: np.int32 = Field(IdentityConverter(np.int32), mask=0xF0) + + +class PortDIOSetPayload(StructPayload[np.uint8]): + """Represents the payload of the PortDIOSet register.""" + + dio0: bool = BitFlag(mask=0x1) + dio1: bool = BitFlag(mask=0x2) + dio2: bool = BitFlag(mask=0x4) + dio3: bool = BitFlag(mask=0x8) + + +class StartPulsePayload(StructPayload[np.uint16]): + """Represents the payload of the StartPulse register.""" + + digital_output: PwmPort = GroupMask(enum=PwmPort, mask=0xC00) + pulse_width: np.uint16 = Field(IdentityConverter(np.uint16), mask=0x3FF) + + +class StartPulseTrainPayload(StructPayload[np.uint16], length=2): + """Represents the payload of the StartPulseTrain register.""" + + digital_output: PwmPort = GroupMask(enum=PwmPort, mask=0xC00) + pulse_width: np.uint16 = Field(IdentityConverter(np.uint16), mask=0x3FF) + frequency: np.uint8 = Field(IdentityConverter(np.uint8), mask=0xFF00, offset=1, default=np.uint8(1)) + pulse_count: np.uint8 = Field(IdentityConverter(np.uint8), mask=0xFF, offset=1) + + +class EncoderModePayload(StructPayload[np.uint8]): + """Represents the payload of the EncoderMode register.""" + + value: EncoderModeMask = GroupMask(enum=EncoderModeMask, mask=0xFF) + + +class DigitalInputs(RegisterU8): + address: ClassVar[int] = 32 + + +class AnalogData(RegisterBase[AnalogDataPayload]): + address: ClassVar[int] = 33 + payload_type: ClassVar[PayloadType] = PayloadType.Float + payload_class = AnalogDataPayload + + +class ComplexConfiguration(RegisterBase[ComplexConfigurationPayload]): + address: ClassVar[int] = 34 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = ComplexConfigurationPayload + + +class Version(RegisterBase[VersionPayload]): + address: ClassVar[int] = 35 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = VersionPayload + + +class CustomPayload(RegisterBase[HarpVersion]): + address: ClassVar[int] = 36 + payload_type: ClassVar[PayloadType] = PayloadType.U32 + payload_class = CustomPayloadPayload + + +class CustomRawPayload(RegisterBase[HarpVersion]): + address: ClassVar[int] = 37 + payload_type: ClassVar[PayloadType] = PayloadType.U32 + payload_class = CustomRawPayloadPayload + + +class CustomMemberConverter(RegisterBase[CustomMemberConverterPayload]): + address: ClassVar[int] = 38 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = CustomMemberConverterPayload + + +class BitmaskSplitter(RegisterBase[BitmaskSplitterPayload]): + address: ClassVar[int] = 39 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = BitmaskSplitterPayload + + +class Counter0(RegisterS32): + address: ClassVar[int] = 40 + + +class PortDIOSet(RegisterBase[PortDIOSetPayload]): + address: ClassVar[int] = 41 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = PortDIOSetPayload + + +class PulseDOPort0(RegisterU16): + address: ClassVar[int] = 42 + + +class PulseDO0(RegisterU16): + address: ClassVar[int] = 43 + + +class StartPulse(RegisterBase[StartPulsePayload]): + """Starts a PWM pulse.""" + + address: ClassVar[int] = 100 + payload_type: ClassVar[PayloadType] = PayloadType.U16 + payload_class = StartPulsePayload + + +class StartPulseTrain(RegisterBase[StartPulseTrainPayload]): + """Starts a PWM pulse train.""" + + address: ClassVar[int] = 101 + payload_type: ClassVar[PayloadType] = PayloadType.U16 + payload_class = StartPulseTrainPayload + + +class EncoderMode(RegisterBase[EncoderModePayload]): + """Configures the operation mode of the encoder.""" + + address: ClassVar[int] = 103 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = EncoderModePayload + + +REGISTER_MAP: dict[int, type[RegisterBase[Any]]] = { + **_CORE_REGISTER_MAP, + 32: DigitalInputs, + 33: AnalogData, + 34: ComplexConfiguration, + 35: Version, + 36: CustomPayload, + 37: CustomRawPayload, + 38: CustomMemberConverter, + 39: BitmaskSplitter, + 40: Counter0, + 41: PortDIOSet, + 42: PulseDOPort0, + 43: PulseDO0, + 100: StartPulse, + 101: StartPulseTrain, + 103: EncoderMode, +} + diff --git a/tests/Harp.Generators.Tests.csproj b/tests/Harp.Generators.Tests.csproj index 6cf4bf8..e37d1b8 100644 --- a/tests/Harp.Generators.Tests.csproj +++ b/tests/Harp.Generators.Tests.csproj @@ -22,6 +22,7 @@ + diff --git a/tests/Python/converters.py b/tests/Python/converters.py new file mode 100644 index 0000000..1c372e8 --- /dev/null +++ b/tests/Python/converters.py @@ -0,0 +1,41 @@ +"""User-supplied converters for the generated Tests device module. + +This is the Python counterpart to tests/EmbeddedSources/device.cs: the generator emits +references to these symbols for the custom payloads it cannot synthesize from device.yml, +and the device package author provides the implementations here in the companion module the +generated code imports from. +""" + +from typing import Any + +import numpy as np +from numpy.typing import NDArray + +from harp.protocol import Converter + + +class DataConverter(Converter[int]): + """Maps two raw little-endian signed bytes to and from a Python int. + + Models interfaceType: int over a two-byte sub-region of the CustomMemberConverter payload. + """ + + init_kwarg_type = int + + def __init__(self) -> None: + self._length = 2 + self.dtype = np.dtype((np.uint8, (self._length,))) + + def decode_scalar(self, view: np.generic) -> int: + return int.from_bytes(bytes(np.asarray(view).tolist()), "little", signed=True) + + def decode_batch(self, view: NDArray[np.generic]) -> Any: + return np.array( + [int.from_bytes(bytes(np.asarray(r).tolist()), "little", signed=True) for r in np.atleast_2d(view)], + dtype=object, + ) + + def encode_into(self, view: NDArray[np.generic], value: int) -> None: + view[...] = np.frombuffer( + int(value).to_bytes(self._length, "little", signed=True), dtype=np.uint8 + ) diff --git a/tests/Python/pyproject.toml b/tests/Python/pyproject.toml new file mode 100644 index 0000000..2d3e54f --- /dev/null +++ b/tests/Python/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "harp-generators-interop" +version = "0" +description = "Cross-stack interop test for the generated Python device interface." +requires-python = ">=3.11" +# harp-protocol and harp-device are pinned to a pyharp unified-API PR commit until published to +# PyPI. This commit moved the packages under src/packages and added harp.device.REGISTER_MAP. +dependencies = [ + "harp-protocol @ git+https://github.com/harp-tech/pyharp.git@0f0365a3fb1ebd68c4f3a8c421375bfce22aeb37#subdirectory=src/packages/harp-protocol", + "harp-device @ git+https://github.com/harp-tech/pyharp.git@0f0365a3fb1ebd68c4f3a8c421375bfce22aeb37#subdirectory=src/packages/harp-device", + "harp-data @ git+https://github.com/harp-tech/pyharp.git@0f0365a3fb1ebd68c4f3a8c421375bfce22aeb37#subdirectory=src/packages/harp-data", + "pytest>=8", +] diff --git a/tests/Python/test_interop.py b/tests/Python/test_interop.py new file mode 100644 index 0000000..607fd57 --- /dev/null +++ b/tests/Python/test_interop.py @@ -0,0 +1,102 @@ +"""Cross-stack interop test. + +The .NET test ``PythonInteropTests.GenerateInteropFixtures`` drives the auto-generated C# +device interface to produce, for each register, a standard Harp binary file of frames plus a +manifest of the expected decoded values. This test reads those binaries back through the +auto-generated Python interface and asserts that every register the C# interface produces is +consumed correctly by the Python interface, closing the loop across the acquisition to +analysis stack. +""" + +import enum +import json +import os +import sys +from dataclasses import is_dataclass +from pathlib import Path + +import numpy as np +import pytest + +from harp.data import read_dataframe +from harp.protocol import HarpMessage +from harp.protocol._payload import PayloadBase + + +def _output_directory() -> Path: + configured = os.environ.get("HARP_INTEROP_OUTPUT") + if configured: + return Path(configured) + repository_root = Path(__file__).resolve().parents[2] + return repository_root / "artifacts" / "python-interop" + + +OUTPUT_DIRECTORY = _output_directory() +sys.path.insert(0, str(OUTPUT_DIRECTORY)) + + +def _load_manifest() -> list: + manifest_path = OUTPUT_DIRECTORY / "manifest.json" + if not manifest_path.exists(): + pytest.skip(f"Interop fixtures not found at {OUTPUT_DIRECTORY}; run the .NET fixture test first.") + with open(manifest_path, encoding="utf-8") as manifest_file: + return json.load(manifest_file) + + +def _split_frames(buffer: bytes) -> list: + frames = [] + offset = 0 + while offset < len(buffer): + # Harp framing: byte 1 is the length of everything after the first two bytes. + length = buffer[offset + 1] + 2 + frames.append(buffer[offset:offset + length]) + offset += length + return frames + + +def _canonical(value) -> object: + if isinstance(value, PayloadBase): + return {name: _canonical(getattr(value, name)) for name in type(value)._repr_fields} + if isinstance(value, (enum.IntFlag, enum.IntEnum)): + return int(value) + if isinstance(value, bool): + return 1 if value else 0 + if isinstance(value, np.ndarray): + return [_canonical(element) for element in value.tolist()] + if isinstance(value, (np.floating, float)): + return round(float(value), 3) + if isinstance(value, (np.integer, int)): + return int(value) + if isinstance(value, (bytes, bytearray)): + return bytes(value).rstrip(b"\x00").decode("ascii") + if isinstance(value, str): + return value + if is_dataclass(value): + return [int(getattr(value, "major")), int(getattr(value, "minor"))] + raise TypeError(f"Unhandled value type for canonicalization: {type(value)!r}") + + +@pytest.fixture(scope="module") +def device_module(): + import harp_device.device as device + return device + + +@pytest.mark.parametrize("entry", _load_manifest(), ids=lambda entry: entry["name"]) +def test_register_round_trips_from_csharp(device_module, entry): + register = getattr(device_module, entry["name"]) + binary_path = OUTPUT_DIRECTORY / "data" / f"{entry['name']}.bin" + buffer = binary_path.read_bytes() + frames = _split_frames(buffer) + + assert len(frames) == entry["frames"] + + # Bulk analysis path: the whole binary parses into one DataFrame row per frame. + dataframe = read_dataframe(register, buffer, timestamp=False) + assert len(dataframe) == entry["frames"] + + # Single message path: each frame decodes to the value the C# interface encoded. + for frame, expected in zip(frames, entry["expected"]): + message = HarpMessage.parse(frame) + parsed = register.parse(message) + assert _canonical(parsed) == expected diff --git a/tests/PythonGeneratorTests.cs b/tests/PythonGeneratorTests.cs new file mode 100644 index 0000000..0e37301 --- /dev/null +++ b/tests/PythonGeneratorTests.cs @@ -0,0 +1,44 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Harp.Generators.Tests; + +[TestClass] +public sealed class PythonGeneratorTests +{ + DirectoryInfo? outputDirectory; + + [TestInitialize] + public void Initialize() + { + outputDirectory = Directory.CreateDirectory("PythonOutput"); + try { Directory.Delete(outputDirectory.FullName, recursive: true); } + catch { } // best effort + } + + [DataTestMethod] + [DataRow("core.yml")] + [DataRow("device.yml")] + public void DeviceTemplate_GenerateMatchesExpectedOutput(string metadataFileName) + { + metadataFileName = TestHelper.GetMetadataPath(metadataFileName); + var deviceMetadata = TestHelper.ReadDeviceMetadata(metadataFileName); + var generator = new PythonGenerator(deviceMetadata); + var implementation = generator.GenerateImplementation(); + TestHelper.AssertNoGeneratorErrors(generator.Errors); + + var outputFileName = $"{Path.GetFileNameWithoutExtension(metadataFileName)}.py"; + try + { + TestHelper.AssertExpectedOutput(implementation.Device, outputFileName); + } + catch (AssertFailedException) + { + if (outputDirectory is not null) + { + outputDirectory.Create(); + File.WriteAllText(Path.Combine(outputDirectory.FullName, outputFileName), implementation.Device); + } + throw; + } + } +} diff --git a/tests/PythonInteropTests.cs b/tests/PythonInteropTests.cs new file mode 100644 index 0000000..c5c56dd --- /dev/null +++ b/tests/PythonInteropTests.cs @@ -0,0 +1,285 @@ +using System.Globalization; +using System.Reflection; +using System.Text; +using Bonsai.Harp; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Harp.Generators.Tests; + +[TestClass] +public sealed class PythonInteropTests +{ + const string DeviceMetadataFileName = "device.yml"; + const int FrameCount = 3; + + static string ResolveOutputDirectory() + { + var configured = Environment.GetEnvironmentVariable("HARP_INTEROP_OUTPUT"); + if (!string.IsNullOrEmpty(configured)) + return configured; + + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "Harp.Generators.sln"))) + directory = directory.Parent; + var root = directory?.FullName ?? AppContext.BaseDirectory; + return Path.Combine(root, "artifacts", "python-interop"); + } + + [TestMethod] + public void GenerateInteropFixtures() + { + var metadataPath = TestHelper.GetMetadataPath(DeviceMetadataFileName); + var deviceMetadata = TestHelper.ReadDeviceMetadata(metadataPath); + + var interfaceImplementation = new InterfaceGenerator(deviceMetadata, typeof(PythonInteropTests).Namespace ?? "").GenerateImplementation(); + var pythonImplementation = new PythonGenerator(deviceMetadata).GenerateImplementation(); + var payloadExtensions = TestHelper.GetManifestResourceText("PayloadMarshal.cs"); + var customImplementation = TestHelper.GetManifestResourceText("EmbeddedSources.device.cs"); + var assembly = CompilerTestHelper.CompileAndLoadFromSource( + interfaceImplementation.Device, + interfaceImplementation.AsyncDevice, + payloadExtensions, + customImplementation); + + var outputDirectory = ResolveOutputDirectory(); + var packageDirectory = Path.Combine(outputDirectory, "harp_device"); + var dataDirectory = Path.Combine(outputDirectory, "data"); + Directory.CreateDirectory(packageDirectory); + Directory.CreateDirectory(dataDirectory); + + File.WriteAllText(Path.Combine(packageDirectory, "device.py"), pythonImplementation.Device); + File.WriteAllText(Path.Combine(packageDirectory, "__init__.py"), string.Empty); + File.WriteAllText(Path.Combine(packageDirectory, "converters.py"), TestHelper.GetManifestResourceText("Python.converters.py")); + File.WriteAllText(Path.Combine(outputDirectory, DeviceMetadataFileName), File.ReadAllText(metadataPath)); + + var manifest = new StringBuilder(); + manifest.Append("[\n"); + var first = true; + foreach (var registerMetadata in deviceMetadata.Registers) + { + var register = registerMetadata.Value; + if (register.Visibility != RegisterVisibility.Public) + continue; + + if (register.PayloadSpec == null && register.HasConverter) + continue; + + var registerType = assembly.GetType($"{typeof(PythonInteropTests).Namespace}.{registerMetadata.Key}") + ?? throw new InvalidOperationException($"Generated register type not found: {registerMetadata.Key}"); + var fromPayload = FindFromPayloadMethod(registerType); + var valueType = fromPayload.GetParameters()[1].ParameterType; + + var expected = new List(); + using (var binaryStream = new FileStream(Path.Combine(dataDirectory, $"{registerMetadata.Key}.bin"), FileMode.Create)) + { + for (int seed = 1; seed <= FrameCount; seed++) + { + var value = InteropValue.Build(valueType, register, seed); + var message = (HarpMessage)fromPayload.Invoke(null, [MessageType.Write, value])!; + binaryStream.Write(message.MessageBytes, 0, message.MessageBytes.Length); + expected.Add(InteropValue.Canonicalize(value, register, deviceMetadata)); + } + } + + if (!first) manifest.Append(",\n"); + first = false; + manifest.Append(" {\n"); + manifest.Append($" \"name\": \"{registerMetadata.Key}\",\n"); + manifest.Append($" \"address\": {register.Address},\n"); + manifest.Append($" \"frames\": {FrameCount},\n"); + manifest.Append(" \"expected\": [").Append(string.Join(", ", expected)).Append("]\n"); + manifest.Append(" }"); + } + manifest.Append("\n]\n"); + File.WriteAllText(Path.Combine(outputDirectory, "manifest.json"), manifest.ToString()); + + Assert.IsTrue(Directory.GetFiles(dataDirectory, "*.bin").Length > 0); + } + + static MethodInfo FindFromPayloadMethod(Type registerType) + { + foreach (var method in registerType.GetMethods(BindingFlags.Public | BindingFlags.Static)) + { + if (method.Name != "FromPayload") continue; + var parameters = method.GetParameters(); + if (parameters.Length == 2 && parameters[0].ParameterType == typeof(MessageType)) + return method; + } + throw new InvalidOperationException($"No FromPayload(MessageType, value) method on {registerType.Name}."); + } +} + +static class InteropValue +{ + public static object Build(Type valueType, RegisterInfo register, int seed) + { + if (register.PayloadSpec != null) + { + var instance = Activator.CreateInstance(valueType)!; + foreach (var member in register.PayloadSpec) + { + var field = valueType.GetField(member.Key) + ?? throw new InvalidOperationException($"Field not found on payload struct: {member.Key}"); + field.SetValue(instance, BuildField(field.FieldType, member.Value, seed)); + } + return instance; + } + + return BuildScalar(valueType, Math.Max(1, register.Length), GetElementRange(register.Type), seed); + } + + static object BuildField(Type fieldType, PayloadMemberInfo member, int seed) + { + if (fieldType.IsArray) + return BuildArray(fieldType.GetElementType()!, member.Length.GetValueOrDefault(1), seed); + if (fieldType.IsEnum) + return SmallestFittingEnumValue(fieldType, GetFieldRange(member)); + return BuildScalar(fieldType, member.Length.GetValueOrDefault(1), GetFieldRange(member), seed); + } + + static long GetElementRange(PayloadType payloadType) + { + var size = (int)payloadType & 0xF; + return size >= 8 ? long.MaxValue : (1L << (size * 8)) - 1; + } + + static long GetFieldRange(PayloadMemberInfo member) + { + if (!member.Mask.HasValue) return long.MaxValue; + var mask = member.Mask.GetValueOrDefault(); + var shift = 0; + while (((mask >> shift) & 1) == 0 && shift < 32) shift++; + return mask >> shift; + } + + static object BuildScalar(Type type, int length, long range, int seed) + { + if (type.IsArray) + return BuildArray(type.GetElementType()!, length, seed); + if (type == typeof(string)) + return new string('A', Math.Min(length > 1 ? length - 1 : 3, 8)); + if (type == typeof(bool)) + return seed % 2 == 1; + if (type.IsEnum) + return SmallestFittingEnumValue(type, range); + if (type.FullName == "Bonsai.Harp.HarpVersion") + return BuildHarpVersion(type, seed); + if (type == typeof(float)) + return seed + 0.5f; + if (type == typeof(double)) + return seed + 0.5d; + var clamped = range >= long.MaxValue ? seed : (int)Math.Min(seed, range); + return Convert.ChangeType(clamped, type, CultureInfo.InvariantCulture); + } + + static object BuildArray(Type elementType, int length, int seed) + { + var array = Array.CreateInstance(elementType, Math.Max(1, length)); + for (int i = 0; i < array.Length; i++) + array.SetValue(BuildScalar(elementType, 1, long.MaxValue, seed + i), i); + return array; + } + + static object SmallestFittingEnumValue(Type type, long fieldRange) + { + object? chosen = null; + long chosenValue = long.MaxValue; + foreach (var value in Enum.GetValues(type)) + { + var numeric = Convert.ToInt64(value, CultureInfo.InvariantCulture); + if (numeric <= 0 || numeric > fieldRange) continue; + if (numeric < chosenValue) + { + chosenValue = numeric; + chosen = value; + } + } + return chosen ?? Enum.GetValues(type).GetValue(0)!; + } + + static object BuildHarpVersion(Type type, int seed) + { + var constructor = type.GetConstructors() + .OrderByDescending(c => c.GetParameters().Length) + .First(c => c.GetParameters().Length >= 2); + var parameters = constructor.GetParameters(); + var arguments = new object?[parameters.Length]; + for (int i = 0; i < parameters.Length; i++) + { + var parameterType = Nullable.GetUnderlyingType(parameters[i].ParameterType) ?? parameters[i].ParameterType; + if (parameterType.IsValueType && i < 2) + arguments[i] = Convert.ChangeType(seed + i, parameterType, CultureInfo.InvariantCulture); + else + arguments[i] = parameters[i].HasDefaultValue ? parameters[i].DefaultValue : null; + } + return constructor.Invoke(arguments); + } + + public static string Canonicalize(object value, RegisterInfo register, DeviceInfo deviceMetadata) + { + if (register.PayloadSpec != null) + { + var type = value.GetType(); + var members = register.PayloadSpec.Select(member => + { + var field = type.GetField(member.Key)!; + return $"\"{ToSnakeCase(member.Key)}\": {CanonicalizeLeaf(field.GetValue(value)!)}"; + }); + return $"{{{string.Join(", ", members)}}}"; + } + + if (deviceMetadata.GroupMasks.ContainsKey(register.MaskType)) + return $"{{\"value\": {CanonicalizeLeaf(value)}}}"; + + if (deviceMetadata.BitMasks.TryGetValue(register.MaskType, out var bitMask)) + { + var elementRange = GetElementRange(register.Type); + var flags = Convert.ToInt64(value, CultureInfo.InvariantCulture); + var bits = bitMask.Bits + .Where(bit => bit.Value.Value != 0 && bit.Value.Value <= elementRange) + .Select(bit => $"\"{ToSnakeCase(bit.Key)}\": {((flags & bit.Value.Value) != 0 ? 1 : 0)}"); + return $"{{{string.Join(", ", bits)}}}"; + } + + return CanonicalizeLeaf(value); + } + + static string CanonicalizeLeaf(object value) + { + var type = value.GetType(); + if (type == typeof(bool)) return (bool)value ? "1" : "0"; + if (type == typeof(string)) return $"\"{value}\""; + if (type.IsEnum) return Convert.ToInt64(value, CultureInfo.InvariantCulture).ToString(CultureInfo.InvariantCulture); + if (type.FullName == "Bonsai.Harp.HarpVersion") return HarpVersionToJson(value); + if (type == typeof(float) || type == typeof(double)) + return Convert.ToDouble(value, CultureInfo.InvariantCulture).ToString("0.000", CultureInfo.InvariantCulture); + if (type.IsArray) + { + var array = (Array)value; + var elements = new string[array.Length]; + for (int i = 0; i < array.Length; i++) + elements[i] = CanonicalizeLeaf(array.GetValue(i)!); + return $"[{string.Join(", ", elements)}]"; + } + return Convert.ToInt64(value, CultureInfo.InvariantCulture).ToString(CultureInfo.InvariantCulture); + } + + static string HarpVersionToJson(object value) + { + var type = value.GetType(); + var major = type.GetProperty("Major")?.GetValue(value); + var minor = type.GetProperty("Minor")?.GetValue(value); + return $"[{ConvertComponent(major)}, {ConvertComponent(minor)}]"; + } + + static string ConvertComponent(object? component) + { + if (component is null) return "0"; + return Convert.ToInt64(component, CultureInfo.InvariantCulture).ToString(CultureInfo.InvariantCulture); + } + + static string ToSnakeCase(string name) + { + return FirmwareNamingConvention.Instance.Apply(name).ToLowerInvariant(); + } +} From a2c4797e9e30125415c613cc41fe0eedd0f3a25c Mon Sep 17 00:00:00 2001 From: glopesdev Date: Sun, 28 Jun 2026 10:01:19 +0100 Subject: [PATCH 2/3] Drop harp-data dependency from interop test Switch the interop test bulk-parse check from read_dataframe in harp-data to RegisterBase.parse_bulk in harp-protocol, and drop the harp-data dependency from the test project. The per-frame parse and value assertions are unchanged. --- tests/Python/pyproject.toml | 1 - tests/Python/test_interop.py | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/Python/pyproject.toml b/tests/Python/pyproject.toml index 2d3e54f..60e37c6 100644 --- a/tests/Python/pyproject.toml +++ b/tests/Python/pyproject.toml @@ -8,6 +8,5 @@ requires-python = ">=3.11" dependencies = [ "harp-protocol @ git+https://github.com/harp-tech/pyharp.git@0f0365a3fb1ebd68c4f3a8c421375bfce22aeb37#subdirectory=src/packages/harp-protocol", "harp-device @ git+https://github.com/harp-tech/pyharp.git@0f0365a3fb1ebd68c4f3a8c421375bfce22aeb37#subdirectory=src/packages/harp-device", - "harp-data @ git+https://github.com/harp-tech/pyharp.git@0f0365a3fb1ebd68c4f3a8c421375bfce22aeb37#subdirectory=src/packages/harp-data", "pytest>=8", ] diff --git a/tests/Python/test_interop.py b/tests/Python/test_interop.py index 607fd57..c946be0 100644 --- a/tests/Python/test_interop.py +++ b/tests/Python/test_interop.py @@ -18,7 +18,6 @@ import numpy as np import pytest -from harp.data import read_dataframe from harp.protocol import HarpMessage from harp.protocol._payload import PayloadBase @@ -91,9 +90,9 @@ def test_register_round_trips_from_csharp(device_module, entry): assert len(frames) == entry["frames"] - # Bulk analysis path: the whole binary parses into one DataFrame row per frame. - dataframe = read_dataframe(register, buffer, timestamp=False) - assert len(dataframe) == entry["frames"] + # Bulk parse path: the whole binary parses into one payload record per frame. + _, _, _, payload = register.parse_bulk(buffer, parse_timestamp=False) + assert len(payload) == entry["frames"] # Single message path: each frame decodes to the value the C# interface encoded. for frame, expected in zip(frames, entry["expected"]): From 0504a0a4294ccefecef981d0642b8dffbad384c7 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 1 Jul 2026 13:29:36 +0100 Subject: [PATCH 3/3] Update Python generator for new pyharp API Regenerate the Python interface against the redesigned pyharp register and payload DSL. Whole-register bit and group masks now unwrap to their enum type: the payload is an AnonymousPayload with a single __value__ field and the register parses to the IntFlag or IntEnum directly, replacing the previous struct of boolean fields. Masked boolean sub-fields now use a BoolConverter field with a mask, since BitFlag was removed upstream, and string and converter registers adopt the same AnonymousPayload __value__ form. Bump the interop test pin to the matching pyharp commit and simplify the cross-stack comparison, which now checks mask registers as a plain integer on both sides. --- src/PyDevice.tt | 6 +-- src/Python.cs | 87 ++++++++++++++++------------------ tests/ExpectedOutput/core.py | 48 ++++++------------- tests/ExpectedOutput/device.py | 25 +++++----- tests/Python/pyproject.toml | 7 +-- tests/PythonInteropTests.cs | 17 +------ 6 files changed, 78 insertions(+), 112 deletions(-) diff --git a/src/PyDevice.tt b/src/PyDevice.tt index e2a66f1..a02f6e2 100644 --- a/src/PyDevice.tt +++ b/src/PyDevice.tt @@ -101,19 +101,19 @@ class <#= pythonEnum.Name #>(enum.<#= pythonEnum.IsFlag ? "IntFlag" : "IntEnum" <# foreach (var payload in module.Payloads) { - var length = payload.Length.HasValue ? $", length={payload.Length.Value}" : string.Empty; #> <# - if (!string.IsNullOrEmpty(payload.Converter)) + if (payload.IsAnonymous) { #> -class <#= payload.Name #>(AnonymousPayload, converter=<#= payload.Converter #>): +class <#= payload.Name #>(AnonymousPayload[<#= payload.ElementType #>]): <# } else { + var length = payload.Length.HasValue ? $", length={payload.Length.Value}" : string.Empty; #> class <#= payload.Name #>(StructPayload[<#= payload.ElementType #>]<#= length #>): <# diff --git a/src/Python.cs b/src/Python.cs index c549f23..b31229f 100644 --- a/src/Python.cs +++ b/src/Python.cs @@ -39,7 +39,7 @@ internal sealed class PythonPayload public string Description = ""; public string ElementType = ""; public int? Length; - public string Converter = ""; + public bool IsAnonymous; public string UnwrapType = ""; public List Fields = []; } @@ -273,65 +273,61 @@ static PythonPayload BuildPythonPayload(PythonModule module, string payloadName, return payload; } - if (!string.IsNullOrEmpty(register.MaskType)) + module.ProtocolImports.Add("AnonymousPayload"); + payload.IsAnonymous = true; + payload.Length = null; + + if (deviceMetadata.GroupMasks.ContainsKey(register.MaskType)) { - module.ProtocolImports.Add("StructPayload"); - BuildMaskRegisterFields(module, payload, register, deviceMetadata, elementType, elementSize); + module.ProtocolImports.Add("GroupMask"); + var elementMask = (1L << (elementSize * 8)) - 1; + payload.UnwrapType = register.MaskType; + payload.Fields.Add(new PythonField + { + Name = "__value__", + Annotation = register.MaskType, + Descriptor = $"GroupMask(enum={register.MaskType}, mask=0x{elementMask:X})" + }); + } + else if (deviceMetadata.BitMasks.ContainsKey(register.MaskType)) + { + module.ProtocolImports.Add("BitMask"); + payload.UnwrapType = register.MaskType; + payload.Fields.Add(new PythonField + { + Name = "__value__", + Annotation = register.MaskType, + Descriptor = $"BitMask(enum={register.MaskType})" + }); } else if (register.InterfaceType == "string") { - module.ProtocolImports.Add("AnonymousPayload"); + module.ProtocolImports.Add("Field"); module.ProtocolImports.Add("StringConverter"); - payload.Converter = $"StringConverter({Math.Max(1, register.Length) * elementSize})"; payload.UnwrapType = "str"; + payload.Fields.Add(new PythonField + { + Name = "__value__", + Annotation = "str", + Descriptor = $"Field(StringConverter({Math.Max(1, register.Length) * elementSize}))" + }); } else { - module.ProtocolImports.Add("AnonymousPayload"); + module.ProtocolImports.Add("Field"); var domainType = register.InterfaceType; var domainConverter = $"{domainType}Converter"; AddConverterImports(module, domainType, domainConverter); - payload.Converter = $"{domainConverter}({elementType})"; payload.UnwrapType = domainType; - } - - return payload; - } - - static void BuildMaskRegisterFields( - PythonModule module, - PythonPayload payload, - RegisterInfo register, - DeviceInfo deviceMetadata, - string elementType, - int elementSize) - { - var elementMask = (1L << (elementSize * 8)) - 1; - if (deviceMetadata.GroupMasks.ContainsKey(register.MaskType)) - { - module.ProtocolImports.Add("GroupMask"); payload.Fields.Add(new PythonField { - Name = "value", - Annotation = register.MaskType, - Descriptor = $"GroupMask(enum={register.MaskType}, mask=0x{elementMask:X})" + Name = "__value__", + Annotation = domainType, + Descriptor = $"Field({domainConverter}({elementType}))" }); } - else if (deviceMetadata.BitMasks.TryGetValue(register.MaskType, out var bitMask)) - { - module.ProtocolImports.Add("BitFlag"); - foreach (var bit in bitMask.Bits) - { - if (bit.Value.Value == 0 || bit.Value.Value > elementMask) continue; - payload.Fields.Add(new PythonField - { - Name = GetPythonFieldName(bit.Key), - Annotation = "bool", - Descriptor = $"BitFlag(mask=0x{bit.Value.Value:X})", - Description = bit.Value.Description - }); - } - } + + return payload; } static PythonField BuildPythonField( @@ -368,12 +364,13 @@ static PythonField BuildPythonField( var mask = member.Mask.GetValueOrDefault(); if (member.InterfaceType == "bool") { - module.ProtocolImports.Add("BitFlag"); + module.ProtocolImports.Add("Field"); + module.ProtocolImports.Add("BoolConverter"); return new PythonField { Name = name, Annotation = "bool", - Descriptor = $"BitFlag(mask=0x{mask:X}{offsetArgument}{defaultArgument})" + Descriptor = $"Field(BoolConverter(), mask=0x{mask:X}{offsetArgument}{defaultArgument})" }; } diff --git a/tests/ExpectedOutput/core.py b/tests/ExpectedOutput/core.py index d184dbf..ae746b7 100644 --- a/tests/ExpectedOutput/core.py +++ b/tests/ExpectedOutput/core.py @@ -7,7 +7,9 @@ import numpy as np from harp.protocol import ( AnonymousPayload, - BitFlag, + BitMask, + BoolConverter, + Field, GroupMask, PayloadType, RegisterBase, @@ -80,9 +82,9 @@ class OperationControlPayload(StructPayload[np.uint8]): operation_mode: OperationMode = GroupMask(enum=OperationMode, mask=0x3) """Specifies the operation mode of the device.""" - dump_registers: bool = BitFlag(mask=0x8) + dump_registers: bool = Field(BoolConverter(), mask=0x8) """Specifies whether the device should report the content of all registers on initialization.""" - mute_replies: bool = BitFlag(mask=0x10) + mute_replies: bool = Field(BoolConverter(), mask=0x10) """Specifies whether the replies to all commands will be muted, i.e. not sent by the device.""" visual_indicators: EnableFlag = GroupMask(enum=EnableFlag, mask=0x20) """Specifies the state of all visual indicators on the device.""" @@ -92,44 +94,22 @@ class OperationControlPayload(StructPayload[np.uint8]): """Specifies whether the device should report the content of the seconds register each second.""" -class ResetDevicePayload(StructPayload[np.uint8]): +class ResetDevicePayload(AnonymousPayload[np.uint8]): """Represents the payload of the ResetDevice register.""" - restore_default: bool = BitFlag(mask=0x1) - """The device will boot with all the registers reset to their default factory values.""" - restore_eeprom: bool = BitFlag(mask=0x2) - """The device will boot and restore all the registers to the values stored in non-volatile memory.""" - save: bool = BitFlag(mask=0x4) - """The device will boot and save all the current register values to non-volatile memory.""" - restore_name: bool = BitFlag(mask=0x8) - """The device will boot with the default device name.""" - update_firmware: bool = BitFlag(mask=0x20) - """The device will enter firmware update mode.""" - boot_from_default: bool = BitFlag(mask=0x40) - """Specifies that the device has booted from default factory values.""" - boot_from_eeprom: bool = BitFlag(mask=0x80) - """Specifies that the device has booted from non-volatile values stored in EEPROM.""" + __value__: ResetFlags = BitMask(enum=ResetFlags) -class DeviceNamePayload(AnonymousPayload, converter=StringConverter(25)): +class DeviceNamePayload(AnonymousPayload[np.uint8]): """Represents the payload of the DeviceName register.""" + __value__: str = Field(StringConverter(25)) -class ClockConfigurationPayload(StructPayload[np.uint8]): + +class ClockConfigurationPayload(AnonymousPayload[np.uint8]): """Represents the payload of the ClockConfiguration register.""" - clock_repeater: bool = BitFlag(mask=0x1) - """The device will repeat the clock synchronization signal to the clock output connector, if available.""" - clock_generator: bool = BitFlag(mask=0x2) - """The device resets and generates the clock synchronization signal on the clock output connector, if available.""" - repeater_capability: bool = BitFlag(mask=0x8) - """Specifies the device has the capability to repeat the clock synchronization signal to the clock output connector.""" - generator_capability: bool = BitFlag(mask=0x10) - """Specifies the device has the capability to generate the clock synchronization signal to the clock output connector.""" - clock_unlock: bool = BitFlag(mask=0x40) - """The device will unlock the timestamp register counter and will accept commands to set new timestamp values.""" - clock_lock: bool = BitFlag(mask=0x80) - """The device will lock the timestamp register counter and will not accept commands to set new timestamp values.""" + __value__: ClockConfigurationFlags = BitMask(enum=ClockConfigurationFlags) class WhoAmI(RegisterU16): @@ -200,7 +180,7 @@ class OperationControl(RegisterBase[OperationControlPayload]): payload_class = OperationControlPayload -class ResetDevice(RegisterBase[ResetDevicePayload]): +class ResetDevice(RegisterBase[ResetFlags]): """Resets the device and saves non-volatile registers.""" address: ClassVar[int] = 11 @@ -222,7 +202,7 @@ class SerialNumber(RegisterU16): address: ClassVar[int] = 13 -class ClockConfiguration(RegisterBase[ClockConfigurationPayload]): +class ClockConfiguration(RegisterBase[ClockConfigurationFlags]): """Specifies the configuration for the device synchronization clock.""" address: ClassVar[int] = 14 diff --git a/tests/ExpectedOutput/device.py b/tests/ExpectedOutput/device.py index 64760bf..1ea9e22 100644 --- a/tests/ExpectedOutput/device.py +++ b/tests/ExpectedOutput/device.py @@ -8,7 +8,7 @@ from numpy.typing import NDArray from harp.protocol import ( AnonymousPayload, - BitFlag, + BitMask, BoolConverter, Field, GroupMask, @@ -84,13 +84,17 @@ class VersionPayload(StructPayload[np.uint8], length=32): interface_hash: NDArray[np.uint8] = Field(IdentityConverter(np.dtype((np.uint8, (20,)))), offset=12) -class CustomPayloadPayload(AnonymousPayload, converter=HarpVersionConverter(np.uint32)): +class CustomPayloadPayload(AnonymousPayload[np.uint32]): """Represents the payload of the CustomPayload register.""" + __value__: HarpVersion = Field(HarpVersionConverter(np.uint32)) -class CustomRawPayloadPayload(AnonymousPayload, converter=HarpVersionConverter(np.uint32)): + +class CustomRawPayloadPayload(AnonymousPayload[np.uint32]): """Represents the payload of the CustomRawPayload register.""" + __value__: HarpVersion = Field(HarpVersionConverter(np.uint32)) + class CustomMemberConverterPayload(StructPayload[np.uint8], length=3): """Represents the payload of the CustomMemberConverter register.""" @@ -106,13 +110,10 @@ class BitmaskSplitterPayload(StructPayload[np.uint8]): high: np.int32 = Field(IdentityConverter(np.int32), mask=0xF0) -class PortDIOSetPayload(StructPayload[np.uint8]): +class PortDIOSetPayload(AnonymousPayload[np.uint8]): """Represents the payload of the PortDIOSet register.""" - dio0: bool = BitFlag(mask=0x1) - dio1: bool = BitFlag(mask=0x2) - dio2: bool = BitFlag(mask=0x4) - dio3: bool = BitFlag(mask=0x8) + __value__: PortDigitalIOS = BitMask(enum=PortDigitalIOS) class StartPulsePayload(StructPayload[np.uint16]): @@ -131,10 +132,10 @@ class StartPulseTrainPayload(StructPayload[np.uint16], length=2): pulse_count: np.uint8 = Field(IdentityConverter(np.uint8), mask=0xFF, offset=1) -class EncoderModePayload(StructPayload[np.uint8]): +class EncoderModePayload(AnonymousPayload[np.uint8]): """Represents the payload of the EncoderMode register.""" - value: EncoderModeMask = GroupMask(enum=EncoderModeMask, mask=0xFF) + __value__: EncoderModeMask = GroupMask(enum=EncoderModeMask, mask=0xFF) class DigitalInputs(RegisterU8): @@ -187,7 +188,7 @@ class Counter0(RegisterS32): address: ClassVar[int] = 40 -class PortDIOSet(RegisterBase[PortDIOSetPayload]): +class PortDIOSet(RegisterBase[PortDigitalIOS]): address: ClassVar[int] = 41 payload_type: ClassVar[PayloadType] = PayloadType.U8 payload_class = PortDIOSetPayload @@ -217,7 +218,7 @@ class StartPulseTrain(RegisterBase[StartPulseTrainPayload]): payload_class = StartPulseTrainPayload -class EncoderMode(RegisterBase[EncoderModePayload]): +class EncoderMode(RegisterBase[EncoderModeMask]): """Configures the operation mode of the encoder.""" address: ClassVar[int] = 103 diff --git a/tests/Python/pyproject.toml b/tests/Python/pyproject.toml index 60e37c6..1b7ef3f 100644 --- a/tests/Python/pyproject.toml +++ b/tests/Python/pyproject.toml @@ -4,9 +4,10 @@ version = "0" description = "Cross-stack interop test for the generated Python device interface." requires-python = ">=3.11" # harp-protocol and harp-device are pinned to a pyharp unified-API PR commit until published to -# PyPI. This commit moved the packages under src/packages and added harp.device.REGISTER_MAP. +# PyPI. This revision replaced BitFlag with the BitMask notation and gave AnonymousPayload a +# single __value__ field carrying the register interface type. dependencies = [ - "harp-protocol @ git+https://github.com/harp-tech/pyharp.git@0f0365a3fb1ebd68c4f3a8c421375bfce22aeb37#subdirectory=src/packages/harp-protocol", - "harp-device @ git+https://github.com/harp-tech/pyharp.git@0f0365a3fb1ebd68c4f3a8c421375bfce22aeb37#subdirectory=src/packages/harp-device", + "harp-protocol @ git+https://github.com/harp-tech/pyharp.git@5716d5352637e0844212c4b33d4c52e599bc7571#subdirectory=src/packages/harp-protocol", + "harp-device @ git+https://github.com/harp-tech/pyharp.git@5716d5352637e0844212c4b33d4c52e599bc7571#subdirectory=src/packages/harp-device", "pytest>=8", ] diff --git a/tests/PythonInteropTests.cs b/tests/PythonInteropTests.cs index c5c56dd..004e9c7 100644 --- a/tests/PythonInteropTests.cs +++ b/tests/PythonInteropTests.cs @@ -77,7 +77,7 @@ public void GenerateInteropFixtures() var value = InteropValue.Build(valueType, register, seed); var message = (HarpMessage)fromPayload.Invoke(null, [MessageType.Write, value])!; binaryStream.Write(message.MessageBytes, 0, message.MessageBytes.Length); - expected.Add(InteropValue.Canonicalize(value, register, deviceMetadata)); + expected.Add(InteropValue.Canonicalize(value, register)); } } @@ -215,7 +215,7 @@ static object BuildHarpVersion(Type type, int seed) return constructor.Invoke(arguments); } - public static string Canonicalize(object value, RegisterInfo register, DeviceInfo deviceMetadata) + public static string Canonicalize(object value, RegisterInfo register) { if (register.PayloadSpec != null) { @@ -228,19 +228,6 @@ public static string Canonicalize(object value, RegisterInfo register, DeviceInf return $"{{{string.Join(", ", members)}}}"; } - if (deviceMetadata.GroupMasks.ContainsKey(register.MaskType)) - return $"{{\"value\": {CanonicalizeLeaf(value)}}}"; - - if (deviceMetadata.BitMasks.TryGetValue(register.MaskType, out var bitMask)) - { - var elementRange = GetElementRange(register.Type); - var flags = Convert.ToInt64(value, CultureInfo.InvariantCulture); - var bits = bitMask.Bits - .Where(bit => bit.Value.Value != 0 && bit.Value.Value <= elementRange) - .Select(bit => $"\"{ToSnakeCase(bit.Key)}\": {((flags & bit.Value.Value) != 0 ? 1 : 0)}"); - return $"{{{string.Join(", ", bits)}}}"; - } - return CanonicalizeLeaf(value); }