diff --git a/.gitattributes b/.gitattributes index 601a4e2..8f96242 100644 --- a/.gitattributes +++ b/.gitattributes @@ -23,6 +23,7 @@ *.yml text # Code +*.py text *.manifest text *.vsixmanifest text *.vstemplate text diff --git a/.github/workflows/Harp.Generators.yml b/.github/workflows/Harp.Generators.yml index 031d5d4..13a6cdb 100644 --- a/.github/workflows/Harp.Generators.yml +++ b/.github/workflows/Harp.Generators.yml @@ -92,6 +92,19 @@ jobs: - name: Test .NET 8 run: dotnet test --no-restore --no-build --configuration ${{matrix.configuration}} --verbosity normal --framework net8.0 + # ----------------------------------------------------------------------- Python interop test + # The .NET test run generates Harp binaries from the auto-generated C# interface; this step + # reads them back through the auto-generated Python interface to verify the two stacks agree. + - name: Set up uv + if: matrix.configuration == 'release' + uses: astral-sh/setup-uv@v5 + + - name: Test Python interop + if: matrix.configuration == 'release' + env: + HARP_INTEROP_OUTPUT: ${{github.workspace}}/artifacts/python-interop + run: uv run --project tests/Python pytest tests/Python/test_interop.py + # ----------------------------------------------------------------------- Collect artifacts - name: Collect NuGet packages uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index 8c2e811..e23fe8e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ .vs/ /artifacts/ + +# Python +__pycache__/ +*.py[cod] +uv.lock diff --git a/src/PyDevice.tt b/src/PyDevice.tt new file mode 100644 index 0000000..a02f6e2 --- /dev/null +++ b/src/PyDevice.tt @@ -0,0 +1,193 @@ +<#@ template debug="true" hostspecific="false" visibility="internal" language="C#" inherits="TemplateBase" #> +<#@ parameter name="DeviceMetadata" type="DeviceInfo" #> +<#@ import namespace="Harp.Generators" #> +<#@ import namespace="System.Linq" #> +<#@ import namespace="System.Collections.Generic" #> +<#@ output extension=".py" #> +<# +var module = TemplateHelper.BuildPythonModule(DeviceMetadata); +var usesNumpy = module.Payloads.Count > 0; +#> +# This file was automatically generated and should not be edited directly. +# To make changes, edit the device metadata and regenerate the interface. + +<# +if (module.Enums.Count > 0) +{ +#> +import enum +<# +} +#> +from typing import Any, ClassVar +<# +if (usesNumpy) +{ +#> + +import numpy as np +<# +} +if (module.UsesNDArray) +{ +#> +from numpy.typing import NDArray +<# +} +#> +from harp.protocol import ( +<# +foreach (var import in module.ProtocolImports) +{ +#> + <#= import #>, +<# +} +#> +) +<# +if (module.IsApplicationDevice) +{ +#> +from harp.device import REGISTER_MAP as _CORE_REGISTER_MAP +<# +} +if (module.ExtensionImports.Count > 0) +{ +#> + +from .converters import ( +<# + foreach (var import in module.ExtensionImports) + { +#> + <#= import #>, +<# + } +#> +) +<# +} +#> +<# +foreach (var pythonEnum in module.Enums) +{ +#> + + +class <#= pythonEnum.Name #>(enum.<#= pythonEnum.IsFlag ? "IntFlag" : "IntEnum" #>): +<# + if (!string.IsNullOrEmpty(pythonEnum.Description)) + { +#> + """<#= pythonEnum.Description #>""" + +<# + } + foreach (var member in pythonEnum.Members) + { +#> + <#= member.Name #> = <#= member.Value #> +<# + if (!string.IsNullOrEmpty(member.Description)) + { +#> + """<#= member.Description #>""" +<# + } + } +} +#> +<# +foreach (var payload in module.Payloads) +{ +#> + + +<# + if (payload.IsAnonymous) + { +#> +class <#= payload.Name #>(AnonymousPayload[<#= payload.ElementType #>]): +<# + } + else + { + var length = payload.Length.HasValue ? $", length={payload.Length.Value}" : string.Empty; +#> +class <#= payload.Name #>(StructPayload[<#= payload.ElementType #>]<#= length #>): +<# + } + if (!string.IsNullOrEmpty(payload.Description)) + { +#> + """<#= payload.Description #>""" +<# + if (payload.Fields.Count > 0) WriteLine(""); + } + foreach (var field in payload.Fields) + { +#> + <#= field.Name #>: <#= field.Annotation #> = <#= field.Descriptor #> +<# + if (!string.IsNullOrEmpty(field.Description)) + { +#> + """<#= field.Description #>""" +<# + } + } +} +#> +<# +foreach (var register in module.Registers) +{ +#> + + +class <#= register.Name #>(<#= register.BaseClass #>): +<# + if (!string.IsNullOrEmpty(register.Description)) + { +#> + """<#= register.Description #>""" + +<# + } +#> + address: ClassVar[int] = <#= register.Address #> +<# + if (register.Kind == PythonRegisterKind.Array) + { +#> + length: ClassVar[int] = <#= register.Length #> +<# + } + else if (register.Kind == PythonRegisterKind.Struct) + { +#> + payload_type: ClassVar[PayloadType] = <#= register.PayloadType #> + payload_class = <#= register.PayloadClass #> +<# + } +} +#> + + +REGISTER_MAP: dict[int, type[RegisterBase[Any]]] = { +<# +if (module.IsApplicationDevice) +{ +#> + **_CORE_REGISTER_MAP, +<# +} +foreach (var register in module.Registers) +{ +#> + <#= register.Address #>: <#= register.Name #>, +<# +} +#> +} + diff --git a/src/Python.cs b/src/Python.cs new file mode 100644 index 0000000..b31229f --- /dev/null +++ b/src/Python.cs @@ -0,0 +1,516 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Bonsai.Harp; + +namespace Harp.Generators; + +internal sealed class PythonModule +{ + public string Device = ""; + public SortedSet ProtocolImports = []; + public SortedSet ExtensionImports = []; + public bool UsesNDArray; + public bool IsApplicationDevice; + public List Enums = []; + public List Payloads = []; + public List Registers = []; +} + +internal sealed class PythonEnum +{ + public string Name = ""; + public bool IsFlag; + public string Description = ""; + public List Members = []; +} + +internal sealed class PythonEnumMember +{ + public string Name = ""; + public string Value = ""; + public string Description = ""; +} + +internal sealed class PythonPayload +{ + public string Name = ""; + public string Description = ""; + public string ElementType = ""; + public int? Length; + public bool IsAnonymous; + public string UnwrapType = ""; + public List Fields = []; +} + +internal sealed class PythonField +{ + public string Name = ""; + public string Annotation = ""; + public string Descriptor = ""; + public string Description = ""; +} + +internal enum PythonRegisterKind +{ + Scalar, + Array, + Struct +} + +internal sealed class PythonRegister +{ + public string Name = ""; + public int Address; + public PythonRegisterKind Kind; + public string BaseClass = ""; + public int Length; + public string PayloadType = ""; + public string PayloadClass = ""; + public string Description = ""; +} + +internal static partial class TemplateHelper +{ + static readonly Dictionary PrimitiveNumpyTypes = new() + { + { "byte", "np.uint8" }, + { "sbyte", "np.int8" }, + { "short", "np.int16" }, + { "ushort", "np.uint16" }, + { "int", "np.int32" }, + { "uint", "np.uint32" }, + { "long", "np.int64" }, + { "ulong", "np.uint64" }, + { "float", "np.float32" }, + }; + + static readonly Dictionary PrimitiveSizes = new() + { + { "byte", 1 }, { "sbyte", 1 }, + { "short", 2 }, { "ushort", 2 }, + { "int", 4 }, { "uint", 4 }, + { "long", 8 }, { "ulong", 8 }, + { "float", 4 }, + }; + + static readonly HashSet StandardConverterInterfaceTypes = ["HarpVersion"]; + + const int CoreRegisterAddressLimit = 32; + + static void AddConverterImports(PythonModule module, string domainType, string domainConverter) + { + if (StandardConverterInterfaceTypes.Contains(domainType)) + { + module.ProtocolImports.Add(domainType); + module.ProtocolImports.Add(domainConverter); + } + else + { + module.ExtensionImports.Add(domainType); + module.ExtensionImports.Add(domainConverter); + } + } + + public static string GetPythonScalarType(PayloadType payloadType) + { + return payloadType switch + { + PayloadType.U8 => "U8", + PayloadType.S8 => "S8", + PayloadType.U16 => "U16", + PayloadType.S16 => "S16", + PayloadType.U32 => "U32", + PayloadType.S32 => "S32", + PayloadType.U64 => "U64", + PayloadType.S64 => "S64", + PayloadType.Float => "Float", + _ => throw new ArgumentOutOfRangeException(nameof(payloadType)), + }; + } + + public static string GetPythonNumpyType(PayloadType payloadType) + { + return payloadType switch + { + PayloadType.U8 => "np.uint8", + PayloadType.S8 => "np.int8", + PayloadType.U16 => "np.uint16", + PayloadType.S16 => "np.int16", + PayloadType.U32 => "np.uint32", + PayloadType.S32 => "np.int32", + PayloadType.U64 => "np.uint64", + PayloadType.S64 => "np.int64", + PayloadType.Float => "np.float32", + _ => throw new ArgumentOutOfRangeException(nameof(payloadType)), + }; + } + + static bool IsPrimitiveInterfaceType(string interfaceType) + { + return PrimitiveNumpyTypes.ContainsKey(interfaceType); + } + + static string GetMemberTypeName(PayloadMemberInfo member) + { + if (!string.IsNullOrEmpty(member.InterfaceType)) return member.InterfaceType; + else if (!string.IsNullOrEmpty(member.MaskType)) return member.MaskType; + else return string.Empty; + } + + public static string GetPythonFieldName(string name) + { + return FirmwareNamingConvention.Instance.Apply(name).ToLowerInvariant(); + } + + public static PythonModule BuildPythonModule(DeviceInfo deviceMetadata) + { + var module = new PythonModule { Device = deviceMetadata.Device }; + + foreach (var bitMask in deviceMetadata.BitMasks) + module.Enums.Add(BuildPythonEnum(bitMask.Key, bitMask.Value.Description, bitMask.Value.Bits, isFlag: true)); + foreach (var groupMask in deviceMetadata.GroupMasks) + module.Enums.Add(BuildPythonEnum(groupMask.Key, groupMask.Value.Description, groupMask.Value.Values, isFlag: false)); + + foreach (var registerMetadata in deviceMetadata.Registers) + BuildPythonRegister(module, registerMetadata.Key, registerMetadata.Value, deviceMetadata); + + module.ProtocolImports.Add("RegisterBase"); + module.IsApplicationDevice = deviceMetadata.Registers.Values.All(register => register.Address >= CoreRegisterAddressLimit); + return module; + } + + static PythonEnum BuildPythonEnum(string name, string description, Dictionary values, bool isFlag) + { + var pythonEnum = new PythonEnum { Name = name, IsFlag = isFlag, Description = description }; + foreach (var value in values) + { + if (isFlag && value.Value.Value == 0) continue; + pythonEnum.Members.Add(new PythonEnumMember + { + Name = FirmwareNamingConvention.Instance.Apply(value.Key), + Value = isFlag + ? $"0x{value.Value.Value:X}" + : value.Value.Value.ToString(CultureInfo.InvariantCulture), + Description = value.Value.Description + }); + } + return pythonEnum; + } + + static void BuildPythonRegister(PythonModule module, string name, RegisterInfo register, DeviceInfo deviceMetadata) + { + var hasMask = !string.IsNullOrEmpty(register.MaskType); + var isString = register.InterfaceType == "string"; + var isCustom = register.HasConverter + || (!string.IsNullOrEmpty(register.InterfaceType) && !IsPrimitiveInterfaceType(register.InterfaceType) && !isString); + var className = register.Visibility == RegisterVisibility.Private ? $"_{name}" : name; + + if (register.PayloadSpec == null && !hasMask && !isString && !isCustom) + { + var isArray = register.Length > 0; + module.ProtocolImports.Add(isArray ? $"Register{GetPythonScalarType(register.Type)}Array" : $"Register{GetPythonScalarType(register.Type)}"); + module.Registers.Add(new PythonRegister + { + Name = className, + Address = register.Address, + Kind = isArray ? PythonRegisterKind.Array : PythonRegisterKind.Scalar, + BaseClass = isArray ? $"Register{GetPythonScalarType(register.Type)}Array" : $"Register{GetPythonScalarType(register.Type)}", + Length = register.Length, + Description = register.Description + }); + return; + } + + module.ProtocolImports.Add("RegisterBase"); + module.ProtocolImports.Add("PayloadType"); + + var payloadName = register.PayloadSpec != null && !string.IsNullOrEmpty(register.InterfaceType) + ? register.InterfaceType + : $"{name}Payload"; + var payload = module.Payloads.Find(existing => existing.Name == payloadName); + if (payload == null) + { + payload = BuildPythonPayload(module, payloadName, register, deviceMetadata); + module.Payloads.Add(payload); + } + + var typeParameter = !string.IsNullOrEmpty(payload.UnwrapType) ? payload.UnwrapType : payload.Name; + module.Registers.Add(new PythonRegister + { + Name = className, + Address = register.Address, + Kind = PythonRegisterKind.Struct, + BaseClass = $"RegisterBase[{typeParameter}]", + PayloadType = $"PayloadType.{GetPythonScalarType(register.Type)}", + PayloadClass = payload.Name, + Description = register.Description + }); + } + + static PythonPayload BuildPythonPayload(PythonModule module, string payloadName, RegisterInfo register, DeviceInfo deviceMetadata) + { + var elementType = GetPythonNumpyType(register.Type); + var elementSize = GetPayloadTypeSize(register.Type); + var payload = new PythonPayload + { + Name = payloadName, + Description = $"Represents the payload of the {RemoveSuffix(payloadName, "Payload")} register.", + ElementType = elementType, + Length = register.Length > 0 ? register.Length : null + }; + + if (register.PayloadSpec != null) + { + module.ProtocolImports.Add("StructPayload"); + foreach (var member in register.PayloadSpec) + { + var field = BuildPythonField(module, member.Key, member.Value, register, deviceMetadata, elementType, elementSize); + field.Description = member.Value.Description; + payload.Fields.Add(field); + } + return payload; + } + + module.ProtocolImports.Add("AnonymousPayload"); + payload.IsAnonymous = true; + payload.Length = null; + + if (deviceMetadata.GroupMasks.ContainsKey(register.MaskType)) + { + module.ProtocolImports.Add("GroupMask"); + var elementMask = (1L << (elementSize * 8)) - 1; + payload.UnwrapType = register.MaskType; + payload.Fields.Add(new PythonField + { + Name = "__value__", + Annotation = register.MaskType, + Descriptor = $"GroupMask(enum={register.MaskType}, mask=0x{elementMask:X})" + }); + } + else if (deviceMetadata.BitMasks.ContainsKey(register.MaskType)) + { + module.ProtocolImports.Add("BitMask"); + payload.UnwrapType = register.MaskType; + payload.Fields.Add(new PythonField + { + Name = "__value__", + Annotation = register.MaskType, + Descriptor = $"BitMask(enum={register.MaskType})" + }); + } + else if (register.InterfaceType == "string") + { + module.ProtocolImports.Add("Field"); + module.ProtocolImports.Add("StringConverter"); + payload.UnwrapType = "str"; + payload.Fields.Add(new PythonField + { + Name = "__value__", + Annotation = "str", + Descriptor = $"Field(StringConverter({Math.Max(1, register.Length) * elementSize}))" + }); + } + else + { + module.ProtocolImports.Add("Field"); + var domainType = register.InterfaceType; + var domainConverter = $"{domainType}Converter"; + AddConverterImports(module, domainType, domainConverter); + payload.UnwrapType = domainType; + payload.Fields.Add(new PythonField + { + Name = "__value__", + Annotation = domainType, + Descriptor = $"Field({domainConverter}({elementType}))" + }); + } + + return payload; + } + + static PythonField BuildPythonField( + PythonModule module, + string name, + PayloadMemberInfo member, + RegisterInfo register, + DeviceInfo deviceMetadata, + string elementType, + int elementSize) + { + var offset = member.Offset.GetValueOrDefault(0); + var offsetArgument = offset > 0 ? $", offset={offset}" : string.Empty; + var typeName = GetMemberTypeName(member); + var defaultArgument = GetPythonDefaultArgument(member, typeName, register, deviceMetadata); + var converterBaseName = name; + name = GetPythonFieldName(name); + + if (deviceMetadata.GroupMasks.ContainsKey(typeName)) + { + module.ProtocolImports.Add("GroupMask"); + var elementMask = (1L << (elementSize * 8)) - 1; + var enumMask = member.Mask.GetValueOrDefault((int)elementMask); + return new PythonField + { + Name = name, + Annotation = typeName, + Descriptor = $"GroupMask(enum={typeName}, mask=0x{enumMask:X}{offsetArgument}{defaultArgument})" + }; + } + + if (member.Mask.HasValue) + { + var mask = member.Mask.GetValueOrDefault(); + if (member.InterfaceType == "bool") + { + module.ProtocolImports.Add("Field"); + module.ProtocolImports.Add("BoolConverter"); + return new PythonField + { + Name = name, + Annotation = "bool", + Descriptor = $"Field(BoolConverter(), mask=0x{mask:X}{offsetArgument}{defaultArgument})" + }; + } + + module.ProtocolImports.Add("Field"); + module.ProtocolImports.Add("IdentityConverter"); + var numpyType = GetMemberNumpyType(member, register); + return new PythonField + { + Name = name, + Annotation = numpyType, + Descriptor = $"Field(IdentityConverter({numpyType}), mask=0x{mask:X}{offsetArgument}{defaultArgument})" + }; + } + + module.ProtocolImports.Add("Field"); + var memberLength = member.Length.GetValueOrDefault(0); + var span = Math.Max(1, memberLength) * elementSize; + + if (member.InterfaceType == "string") + { + module.ProtocolImports.Add("StringConverter"); + return new PythonField + { + Name = name, + Annotation = "str", + Descriptor = $"Field(StringConverter({span}){offsetArgument}{defaultArgument})" + }; + } + + if (member.InterfaceType == "bool") + { + module.ProtocolImports.Add("BoolConverter"); + return new PythonField + { + Name = name, + Annotation = "bool", + Descriptor = $"Field(BoolConverter(){offsetArgument}{defaultArgument})" + }; + } + + if (string.IsNullOrEmpty(member.InterfaceType)) + { + module.ProtocolImports.Add("IdentityConverter"); + if (memberLength > 1) + { + module.UsesNDArray = true; + return new PythonField + { + Name = name, + Annotation = $"NDArray[{elementType}]", + Descriptor = $"Field(IdentityConverter(np.dtype(({elementType}, ({memberLength},)))){offsetArgument}{defaultArgument})" + }; + } + + return new PythonField + { + Name = name, + Annotation = elementType, + Descriptor = $"Field(IdentityConverter({elementType}){offsetArgument}{defaultArgument})" + }; + } + + if (IsPrimitiveInterfaceType(member.InterfaceType) && PrimitiveSizes[member.InterfaceType] == span) + { + module.ProtocolImports.Add("IdentityConverter"); + var numpyType = PrimitiveNumpyTypes[member.InterfaceType]; + return new PythonField + { + Name = name, + Annotation = numpyType, + Descriptor = $"Field(IdentityConverter({numpyType}){offsetArgument}{defaultArgument})" + }; + } + + if (!IsPrimitiveInterfaceType(member.InterfaceType)) + { + var domainType = member.InterfaceType; + var domainConverter = $"{domainType}Converter"; + AddConverterImports(module, domainType, domainConverter); + return new PythonField + { + Name = name, + Annotation = domainType, + Descriptor = $"Field({domainConverter}({elementType}){offsetArgument}{defaultArgument})" + }; + } + + var converterName = $"{converterBaseName}Converter"; + module.ExtensionImports.Add(converterName); + return new PythonField + { + Name = name, + Annotation = PrimitiveNumpyTypes[member.InterfaceType], + Descriptor = $"Field({converterName}(){offsetArgument}{defaultArgument})" + }; + } + + static string GetMemberNumpyType(PayloadMemberInfo member, RegisterInfo register) + { + if (!string.IsNullOrEmpty(member.InterfaceType) && PrimitiveNumpyTypes.TryGetValue(member.InterfaceType, out var numpyType)) + return numpyType; + return GetPythonNumpyType(register.Type); + } + + static string GetMemberAnnotation(PayloadMemberInfo member, RegisterInfo register) + { + return GetMemberNumpyType(member, register); + } + + static string GetPythonDefaultArgument(PayloadMemberInfo member, string typeName, RegisterInfo register, DeviceInfo deviceMetadata) + { + var defaultValue = member.DefaultValue ?? member.MinValue; + if (!defaultValue.HasValue || member.Length.GetValueOrDefault() > 1) + return string.Empty; + + var value = defaultValue.GetValueOrDefault(); + if (deviceMetadata.GroupMasks.TryGetValue(typeName, out var groupMask)) + { + foreach (var entry in groupMask.Values) + { + if (entry.Value.Value == (int)value) + return $", default={typeName}.{FirmwareNamingConvention.Instance.Apply(entry.Key)}"; + } + return $", default={(long)value}"; + } + + if (member.InterfaceType == "bool") + return $", default={(value != 0 ? "True" : "False")}"; + + if (member.InterfaceType == "string" || member.HasConverter || + (!string.IsNullOrEmpty(member.InterfaceType) && !IsPrimitiveInterfaceType(member.InterfaceType))) + return string.Empty; + + return $", default={GetMemberNumpyType(member, register)}({FormatNumericLiteral(value)})"; + } + + static string FormatNumericLiteral(float value) + { + return value == Math.Floor(value) + ? ((long)value).ToString(CultureInfo.InvariantCulture) + : value.ToString(CultureInfo.InvariantCulture); + } +} diff --git a/src/PythonGenerator.cs b/src/PythonGenerator.cs new file mode 100644 index 0000000..c716cb7 --- /dev/null +++ b/src/PythonGenerator.cs @@ -0,0 +1,70 @@ +using System.CodeDom.Compiler; +using System.Collections; +using System.Collections.Generic; + +namespace Harp.Generators; + +/// +/// Provides automatic generation of Python device interface implementations. +/// +public sealed class PythonGenerator +{ + readonly PyDevice _deviceTemplate = new(); + readonly CompilerErrorCollection errors = []; + + /// + /// Initializes a new instance of the class with the + /// specified device metadata. + /// + /// The device metadata object. + public PythonGenerator(DeviceInfo deviceMetadata) + { + var session = new Dictionary + { + { "DeviceMetadata", deviceMetadata } + }; + _deviceTemplate.Initialize(PythonImplementation.DeviceFileName, errors, session); + } + + /// + /// Gets the collection of errors emitted during the code generation process. + /// + public CompilerErrorCollection Errors => errors; + + /// + /// Generates a Python device interface implementation complying with the specified metadata file. + /// + /// The generated device interface implementation. + public PythonImplementation GenerateImplementation() => + new(Device: _deviceTemplate.TransformText()); +} + +/// +/// Represents the generated Python device interface implementation. +/// +/// The generated source code implementing the device register interface. +public record struct PythonImplementation(string Device) + : IEnumerable> +{ + /// + /// Represents the default name for the file storing the device register interface source code. + /// + public const string DeviceFileName = "device.py"; + + /// + /// Returns an enumerator that iterates through all the source code files in the + /// generated implementation. + /// + /// + /// An enumerator that can be used to iterate through the generated implementation files. + /// + public readonly IEnumerator> GetEnumerator() + { + yield return new(DeviceFileName, Device); + } + + readonly IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } +} diff --git a/tests/CompilerTestHelper.cs b/tests/CompilerTestHelper.cs index 4d3409d..b0eadad 100644 --- a/tests/CompilerTestHelper.cs +++ b/tests/CompilerTestHelper.cs @@ -1,4 +1,5 @@ -using Basic.Reference.Assemblies; +using System.Reflection; +using Basic.Reference.Assemblies; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -8,6 +9,17 @@ namespace Harp.Generators.Tests; internal static class CompilerTestHelper { public static void CompileFromSource(params string[] code) + { + Compile(code); + } + + public static Assembly CompileAndLoadFromSource(params string[] code) + { + var bytes = Compile(code); + return Assembly.Load(bytes); + } + + static byte[] Compile(string[] code) { var options = CSharpParseOptions.Default; var syntaxTrees = Array.ConvertAll(code, text => CSharpSyntaxTree.ParseText(text, options)); @@ -38,5 +50,7 @@ public static void CompileFromSource(params string[] code) .ToList(); Assert.Fail(string.Join(Environment.NewLine, errorMessages)); } + + return memoryStream.ToArray(); } } diff --git a/tests/ExpectedOutput/core.py b/tests/ExpectedOutput/core.py new file mode 100644 index 0000000..ae746b7 --- /dev/null +++ b/tests/ExpectedOutput/core.py @@ -0,0 +1,230 @@ +# This file was automatically generated and should not be edited directly. +# To make changes, edit the device metadata and regenerate the interface. + +import enum +from typing import Any, ClassVar + +import numpy as np +from harp.protocol import ( + AnonymousPayload, + BitMask, + BoolConverter, + Field, + GroupMask, + PayloadType, + RegisterBase, + RegisterU16, + RegisterU32, + RegisterU8, + StringConverter, + StructPayload, +) + + +class ResetFlags(enum.IntFlag): + """Specifies the behavior of the non-volatile registers when resetting the device.""" + + RESTORE_DEFAULT = 0x1 + """The device will boot with all the registers reset to their default factory values.""" + RESTORE_EEPROM = 0x2 + """The device will boot and restore all the registers to the values stored in non-volatile memory.""" + SAVE = 0x4 + """The device will boot and save all the current register values to non-volatile memory.""" + RESTORE_NAME = 0x8 + """The device will boot with the default device name.""" + UPDATE_FIRMWARE = 0x20 + """The device will enter firmware update mode.""" + BOOT_FROM_DEFAULT = 0x40 + """Specifies that the device has booted from default factory values.""" + BOOT_FROM_EEPROM = 0x80 + """Specifies that the device has booted from non-volatile values stored in EEPROM.""" + + +class ClockConfigurationFlags(enum.IntFlag): + """Specifies configuration flags for the device synchronization clock.""" + + CLOCK_REPEATER = 0x1 + """The device will repeat the clock synchronization signal to the clock output connector, if available.""" + CLOCK_GENERATOR = 0x2 + """The device resets and generates the clock synchronization signal on the clock output connector, if available.""" + REPEATER_CAPABILITY = 0x8 + """Specifies the device has the capability to repeat the clock synchronization signal to the clock output connector.""" + GENERATOR_CAPABILITY = 0x10 + """Specifies the device has the capability to generate the clock synchronization signal to the clock output connector.""" + CLOCK_UNLOCK = 0x40 + """The device will unlock the timestamp register counter and will accept commands to set new timestamp values.""" + CLOCK_LOCK = 0x80 + """The device will lock the timestamp register counter and will not accept commands to set new timestamp values.""" + + +class OperationMode(enum.IntEnum): + """Specifies the operation mode of the device.""" + + STANDBY = 0 + """Disable all event reporting on the device.""" + ACTIVE = 1 + """Event detection is enabled. Only enabled events are reported by the device.""" + SPEED = 3 + """The device enters speed mode.""" + + +class EnableFlag(enum.IntEnum): + """Specifies whether a specific register flag is enabled or disabled.""" + + DISABLED = 0 + """Specifies that the flag is disabled.""" + ENABLED = 1 + """Specifies that the flag is enabled.""" + + +class OperationControlPayload(StructPayload[np.uint8]): + """Represents the payload of the OperationControl register.""" + + operation_mode: OperationMode = GroupMask(enum=OperationMode, mask=0x3) + """Specifies the operation mode of the device.""" + dump_registers: bool = Field(BoolConverter(), mask=0x8) + """Specifies whether the device should report the content of all registers on initialization.""" + mute_replies: bool = Field(BoolConverter(), mask=0x10) + """Specifies whether the replies to all commands will be muted, i.e. not sent by the device.""" + visual_indicators: EnableFlag = GroupMask(enum=EnableFlag, mask=0x20) + """Specifies the state of all visual indicators on the device.""" + operation_led: EnableFlag = GroupMask(enum=EnableFlag, mask=0x40) + """Specifies whether the device state LED should report the operation mode of the device.""" + heartbeat: EnableFlag = GroupMask(enum=EnableFlag, mask=0x80) + """Specifies whether the device should report the content of the seconds register each second.""" + + +class ResetDevicePayload(AnonymousPayload[np.uint8]): + """Represents the payload of the ResetDevice register.""" + + __value__: ResetFlags = BitMask(enum=ResetFlags) + + +class DeviceNamePayload(AnonymousPayload[np.uint8]): + """Represents the payload of the DeviceName register.""" + + __value__: str = Field(StringConverter(25)) + + +class ClockConfigurationPayload(AnonymousPayload[np.uint8]): + """Represents the payload of the ClockConfiguration register.""" + + __value__: ClockConfigurationFlags = BitMask(enum=ClockConfigurationFlags) + + +class WhoAmI(RegisterU16): + """Specifies the identity class of the device.""" + + address: ClassVar[int] = 0 + + +class HardwareVersionHigh(RegisterU8): + """Specifies the major hardware version of the device.""" + + address: ClassVar[int] = 1 + + +class HardwareVersionLow(RegisterU8): + """Specifies the minor hardware version of the device.""" + + address: ClassVar[int] = 2 + + +class AssemblyVersion(RegisterU8): + """Specifies the version of the assembled components in the device.""" + + address: ClassVar[int] = 3 + + +class CoreVersionHigh(RegisterU8): + """Specifies the major version of the Harp core implemented by the device.""" + + address: ClassVar[int] = 4 + + +class CoreVersionLow(RegisterU8): + """Specifies the minor version of the Harp core implemented by the device.""" + + address: ClassVar[int] = 5 + + +class FirmwareVersionHigh(RegisterU8): + """Specifies the major version of the Harp core implemented by the device.""" + + address: ClassVar[int] = 6 + + +class FirmwareVersionLow(RegisterU8): + """Specifies the minor version of the Harp core implemented by the device.""" + + address: ClassVar[int] = 7 + + +class TimestampSeconds(RegisterU32): + """Stores the integral part of the system timestamp, in seconds.""" + + address: ClassVar[int] = 8 + + +class TimestampMicroseconds(RegisterU16): + """Stores the fractional part of the system timestamp, in microseconds.""" + + address: ClassVar[int] = 9 + + +class OperationControl(RegisterBase[OperationControlPayload]): + """Stores the configuration mode of the device.""" + + address: ClassVar[int] = 10 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = OperationControlPayload + + +class ResetDevice(RegisterBase[ResetFlags]): + """Resets the device and saves non-volatile registers.""" + + address: ClassVar[int] = 11 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = ResetDevicePayload + + +class DeviceName(RegisterBase[str]): + """Stores the user-specified device name.""" + + address: ClassVar[int] = 12 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = DeviceNamePayload + + +class SerialNumber(RegisterU16): + """Specifies the unique serial number of the device.""" + + address: ClassVar[int] = 13 + + +class ClockConfiguration(RegisterBase[ClockConfigurationFlags]): + """Specifies the configuration for the device synchronization clock.""" + + address: ClassVar[int] = 14 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = ClockConfigurationPayload + + +REGISTER_MAP: dict[int, type[RegisterBase[Any]]] = { + 0: WhoAmI, + 1: HardwareVersionHigh, + 2: HardwareVersionLow, + 3: AssemblyVersion, + 4: CoreVersionHigh, + 5: CoreVersionLow, + 6: FirmwareVersionHigh, + 7: FirmwareVersionLow, + 8: TimestampSeconds, + 9: TimestampMicroseconds, + 10: OperationControl, + 11: ResetDevice, + 12: DeviceName, + 13: SerialNumber, + 14: ClockConfiguration, +} + diff --git a/tests/ExpectedOutput/device.py b/tests/ExpectedOutput/device.py new file mode 100644 index 0000000..1ea9e22 --- /dev/null +++ b/tests/ExpectedOutput/device.py @@ -0,0 +1,247 @@ +# This file was automatically generated and should not be edited directly. +# To make changes, edit the device metadata and regenerate the interface. + +import enum +from typing import Any, ClassVar + +import numpy as np +from numpy.typing import NDArray +from harp.protocol import ( + AnonymousPayload, + BitMask, + BoolConverter, + Field, + GroupMask, + HarpVersion, + HarpVersionConverter, + IdentityConverter, + PayloadType, + RegisterBase, + RegisterS32, + RegisterU16, + RegisterU8, + StringConverter, + StructPayload, +) +from harp.device import REGISTER_MAP as _CORE_REGISTER_MAP + +from .converters import ( + DataConverter, +) + + +class PortDigitalIOS(enum.IntFlag): + DIO0 = 0x1 + DIO1 = 0x2 + DIO2 = 0x4 + DIO3 = 0x8 + DI_PORT0 = 0x100 + TEST_DI_PORT1 = 0x200 + SUPPLY_PORT0 = 0x400 + PORT_DIO1 = 0x800 + + +class PwmPort(enum.IntEnum): + PWM0 = 1 + PWM1 = 2 + PWM2 = 4 + PWM3 = 10 + + +class EncoderModeMask(enum.IntEnum): + """Specifies the type of encoder mode.""" + + POSITION = 0 + DISPLACEMENT = 1 + + +class AnalogDataPayload(StructPayload[np.float32], length=6): + """Represents the payload of the AnalogData register.""" + + analog0: np.float32 = Field(IdentityConverter(np.float32)) + analog1: np.float32 = Field(IdentityConverter(np.float32), offset=1) + analog2: np.float32 = Field(IdentityConverter(np.float32), offset=2) + accelerometer: NDArray[np.float32] = Field(IdentityConverter(np.dtype((np.float32, (3,)))), offset=3) + + +class ComplexConfigurationPayload(StructPayload[np.uint8], length=17): + """Represents the payload of the ComplexConfiguration register.""" + + pwm_port: PwmPort = GroupMask(enum=PwmPort, mask=0xFF) + duty_cycle: np.float32 = Field(IdentityConverter(np.float32), offset=4) + frequency: np.float32 = Field(IdentityConverter(np.float32), offset=8) + events_enabled: bool = Field(BoolConverter(), offset=12) + delta: np.uint32 = Field(IdentityConverter(np.uint32), offset=13) + + +class VersionPayload(StructPayload[np.uint8], length=32): + """Represents the payload of the Version register.""" + + protocol_version: HarpVersion = Field(HarpVersionConverter(np.uint8)) + firmware_version: HarpVersion = Field(HarpVersionConverter(np.uint8), offset=3) + hardware_version: HarpVersion = Field(HarpVersionConverter(np.uint8), offset=6) + core_id: str = Field(StringConverter(3), offset=9) + interface_hash: NDArray[np.uint8] = Field(IdentityConverter(np.dtype((np.uint8, (20,)))), offset=12) + + +class CustomPayloadPayload(AnonymousPayload[np.uint32]): + """Represents the payload of the CustomPayload register.""" + + __value__: HarpVersion = Field(HarpVersionConverter(np.uint32)) + + +class CustomRawPayloadPayload(AnonymousPayload[np.uint32]): + """Represents the payload of the CustomRawPayload register.""" + + __value__: HarpVersion = Field(HarpVersionConverter(np.uint32)) + + +class CustomMemberConverterPayload(StructPayload[np.uint8], length=3): + """Represents the payload of the CustomMemberConverter register.""" + + header: np.uint8 = Field(IdentityConverter(np.uint8)) + data: np.int32 = Field(DataConverter(), offset=1) + + +class BitmaskSplitterPayload(StructPayload[np.uint8]): + """Represents the payload of the BitmaskSplitter register.""" + + low: np.int32 = Field(IdentityConverter(np.int32), mask=0xF) + high: np.int32 = Field(IdentityConverter(np.int32), mask=0xF0) + + +class PortDIOSetPayload(AnonymousPayload[np.uint8]): + """Represents the payload of the PortDIOSet register.""" + + __value__: PortDigitalIOS = BitMask(enum=PortDigitalIOS) + + +class StartPulsePayload(StructPayload[np.uint16]): + """Represents the payload of the StartPulse register.""" + + digital_output: PwmPort = GroupMask(enum=PwmPort, mask=0xC00) + pulse_width: np.uint16 = Field(IdentityConverter(np.uint16), mask=0x3FF) + + +class StartPulseTrainPayload(StructPayload[np.uint16], length=2): + """Represents the payload of the StartPulseTrain register.""" + + digital_output: PwmPort = GroupMask(enum=PwmPort, mask=0xC00) + pulse_width: np.uint16 = Field(IdentityConverter(np.uint16), mask=0x3FF) + frequency: np.uint8 = Field(IdentityConverter(np.uint8), mask=0xFF00, offset=1, default=np.uint8(1)) + pulse_count: np.uint8 = Field(IdentityConverter(np.uint8), mask=0xFF, offset=1) + + +class EncoderModePayload(AnonymousPayload[np.uint8]): + """Represents the payload of the EncoderMode register.""" + + __value__: EncoderModeMask = GroupMask(enum=EncoderModeMask, mask=0xFF) + + +class DigitalInputs(RegisterU8): + address: ClassVar[int] = 32 + + +class AnalogData(RegisterBase[AnalogDataPayload]): + address: ClassVar[int] = 33 + payload_type: ClassVar[PayloadType] = PayloadType.Float + payload_class = AnalogDataPayload + + +class ComplexConfiguration(RegisterBase[ComplexConfigurationPayload]): + address: ClassVar[int] = 34 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = ComplexConfigurationPayload + + +class Version(RegisterBase[VersionPayload]): + address: ClassVar[int] = 35 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = VersionPayload + + +class CustomPayload(RegisterBase[HarpVersion]): + address: ClassVar[int] = 36 + payload_type: ClassVar[PayloadType] = PayloadType.U32 + payload_class = CustomPayloadPayload + + +class CustomRawPayload(RegisterBase[HarpVersion]): + address: ClassVar[int] = 37 + payload_type: ClassVar[PayloadType] = PayloadType.U32 + payload_class = CustomRawPayloadPayload + + +class CustomMemberConverter(RegisterBase[CustomMemberConverterPayload]): + address: ClassVar[int] = 38 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = CustomMemberConverterPayload + + +class BitmaskSplitter(RegisterBase[BitmaskSplitterPayload]): + address: ClassVar[int] = 39 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = BitmaskSplitterPayload + + +class Counter0(RegisterS32): + address: ClassVar[int] = 40 + + +class PortDIOSet(RegisterBase[PortDigitalIOS]): + address: ClassVar[int] = 41 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = PortDIOSetPayload + + +class PulseDOPort0(RegisterU16): + address: ClassVar[int] = 42 + + +class PulseDO0(RegisterU16): + address: ClassVar[int] = 43 + + +class StartPulse(RegisterBase[StartPulsePayload]): + """Starts a PWM pulse.""" + + address: ClassVar[int] = 100 + payload_type: ClassVar[PayloadType] = PayloadType.U16 + payload_class = StartPulsePayload + + +class StartPulseTrain(RegisterBase[StartPulseTrainPayload]): + """Starts a PWM pulse train.""" + + address: ClassVar[int] = 101 + payload_type: ClassVar[PayloadType] = PayloadType.U16 + payload_class = StartPulseTrainPayload + + +class EncoderMode(RegisterBase[EncoderModeMask]): + """Configures the operation mode of the encoder.""" + + address: ClassVar[int] = 103 + payload_type: ClassVar[PayloadType] = PayloadType.U8 + payload_class = EncoderModePayload + + +REGISTER_MAP: dict[int, type[RegisterBase[Any]]] = { + **_CORE_REGISTER_MAP, + 32: DigitalInputs, + 33: AnalogData, + 34: ComplexConfiguration, + 35: Version, + 36: CustomPayload, + 37: CustomRawPayload, + 38: CustomMemberConverter, + 39: BitmaskSplitter, + 40: Counter0, + 41: PortDIOSet, + 42: PulseDOPort0, + 43: PulseDO0, + 100: StartPulse, + 101: StartPulseTrain, + 103: EncoderMode, +} + diff --git a/tests/Harp.Generators.Tests.csproj b/tests/Harp.Generators.Tests.csproj index 6cf4bf8..e37d1b8 100644 --- a/tests/Harp.Generators.Tests.csproj +++ b/tests/Harp.Generators.Tests.csproj @@ -22,6 +22,7 @@ + diff --git a/tests/Python/converters.py b/tests/Python/converters.py new file mode 100644 index 0000000..1c372e8 --- /dev/null +++ b/tests/Python/converters.py @@ -0,0 +1,41 @@ +"""User-supplied converters for the generated Tests device module. + +This is the Python counterpart to tests/EmbeddedSources/device.cs: the generator emits +references to these symbols for the custom payloads it cannot synthesize from device.yml, +and the device package author provides the implementations here in the companion module the +generated code imports from. +""" + +from typing import Any + +import numpy as np +from numpy.typing import NDArray + +from harp.protocol import Converter + + +class DataConverter(Converter[int]): + """Maps two raw little-endian signed bytes to and from a Python int. + + Models interfaceType: int over a two-byte sub-region of the CustomMemberConverter payload. + """ + + init_kwarg_type = int + + def __init__(self) -> None: + self._length = 2 + self.dtype = np.dtype((np.uint8, (self._length,))) + + def decode_scalar(self, view: np.generic) -> int: + return int.from_bytes(bytes(np.asarray(view).tolist()), "little", signed=True) + + def decode_batch(self, view: NDArray[np.generic]) -> Any: + return np.array( + [int.from_bytes(bytes(np.asarray(r).tolist()), "little", signed=True) for r in np.atleast_2d(view)], + dtype=object, + ) + + def encode_into(self, view: NDArray[np.generic], value: int) -> None: + view[...] = np.frombuffer( + int(value).to_bytes(self._length, "little", signed=True), dtype=np.uint8 + ) diff --git a/tests/Python/pyproject.toml b/tests/Python/pyproject.toml new file mode 100644 index 0000000..1b7ef3f --- /dev/null +++ b/tests/Python/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "harp-generators-interop" +version = "0" +description = "Cross-stack interop test for the generated Python device interface." +requires-python = ">=3.11" +# harp-protocol and harp-device are pinned to a pyharp unified-API PR commit until published to +# PyPI. This revision replaced BitFlag with the BitMask notation and gave AnonymousPayload a +# single __value__ field carrying the register interface type. +dependencies = [ + "harp-protocol @ git+https://github.com/harp-tech/pyharp.git@5716d5352637e0844212c4b33d4c52e599bc7571#subdirectory=src/packages/harp-protocol", + "harp-device @ git+https://github.com/harp-tech/pyharp.git@5716d5352637e0844212c4b33d4c52e599bc7571#subdirectory=src/packages/harp-device", + "pytest>=8", +] diff --git a/tests/Python/test_interop.py b/tests/Python/test_interop.py new file mode 100644 index 0000000..c946be0 --- /dev/null +++ b/tests/Python/test_interop.py @@ -0,0 +1,101 @@ +"""Cross-stack interop test. + +The .NET test ``PythonInteropTests.GenerateInteropFixtures`` drives the auto-generated C# +device interface to produce, for each register, a standard Harp binary file of frames plus a +manifest of the expected decoded values. This test reads those binaries back through the +auto-generated Python interface and asserts that every register the C# interface produces is +consumed correctly by the Python interface, closing the loop across the acquisition to +analysis stack. +""" + +import enum +import json +import os +import sys +from dataclasses import is_dataclass +from pathlib import Path + +import numpy as np +import pytest + +from harp.protocol import HarpMessage +from harp.protocol._payload import PayloadBase + + +def _output_directory() -> Path: + configured = os.environ.get("HARP_INTEROP_OUTPUT") + if configured: + return Path(configured) + repository_root = Path(__file__).resolve().parents[2] + return repository_root / "artifacts" / "python-interop" + + +OUTPUT_DIRECTORY = _output_directory() +sys.path.insert(0, str(OUTPUT_DIRECTORY)) + + +def _load_manifest() -> list: + manifest_path = OUTPUT_DIRECTORY / "manifest.json" + if not manifest_path.exists(): + pytest.skip(f"Interop fixtures not found at {OUTPUT_DIRECTORY}; run the .NET fixture test first.") + with open(manifest_path, encoding="utf-8") as manifest_file: + return json.load(manifest_file) + + +def _split_frames(buffer: bytes) -> list: + frames = [] + offset = 0 + while offset < len(buffer): + # Harp framing: byte 1 is the length of everything after the first two bytes. + length = buffer[offset + 1] + 2 + frames.append(buffer[offset:offset + length]) + offset += length + return frames + + +def _canonical(value) -> object: + if isinstance(value, PayloadBase): + return {name: _canonical(getattr(value, name)) for name in type(value)._repr_fields} + if isinstance(value, (enum.IntFlag, enum.IntEnum)): + return int(value) + if isinstance(value, bool): + return 1 if value else 0 + if isinstance(value, np.ndarray): + return [_canonical(element) for element in value.tolist()] + if isinstance(value, (np.floating, float)): + return round(float(value), 3) + if isinstance(value, (np.integer, int)): + return int(value) + if isinstance(value, (bytes, bytearray)): + return bytes(value).rstrip(b"\x00").decode("ascii") + if isinstance(value, str): + return value + if is_dataclass(value): + return [int(getattr(value, "major")), int(getattr(value, "minor"))] + raise TypeError(f"Unhandled value type for canonicalization: {type(value)!r}") + + +@pytest.fixture(scope="module") +def device_module(): + import harp_device.device as device + return device + + +@pytest.mark.parametrize("entry", _load_manifest(), ids=lambda entry: entry["name"]) +def test_register_round_trips_from_csharp(device_module, entry): + register = getattr(device_module, entry["name"]) + binary_path = OUTPUT_DIRECTORY / "data" / f"{entry['name']}.bin" + buffer = binary_path.read_bytes() + frames = _split_frames(buffer) + + assert len(frames) == entry["frames"] + + # Bulk parse path: the whole binary parses into one payload record per frame. + _, _, _, payload = register.parse_bulk(buffer, parse_timestamp=False) + assert len(payload) == entry["frames"] + + # Single message path: each frame decodes to the value the C# interface encoded. + for frame, expected in zip(frames, entry["expected"]): + message = HarpMessage.parse(frame) + parsed = register.parse(message) + assert _canonical(parsed) == expected diff --git a/tests/PythonGeneratorTests.cs b/tests/PythonGeneratorTests.cs new file mode 100644 index 0000000..0e37301 --- /dev/null +++ b/tests/PythonGeneratorTests.cs @@ -0,0 +1,44 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Harp.Generators.Tests; + +[TestClass] +public sealed class PythonGeneratorTests +{ + DirectoryInfo? outputDirectory; + + [TestInitialize] + public void Initialize() + { + outputDirectory = Directory.CreateDirectory("PythonOutput"); + try { Directory.Delete(outputDirectory.FullName, recursive: true); } + catch { } // best effort + } + + [DataTestMethod] + [DataRow("core.yml")] + [DataRow("device.yml")] + public void DeviceTemplate_GenerateMatchesExpectedOutput(string metadataFileName) + { + metadataFileName = TestHelper.GetMetadataPath(metadataFileName); + var deviceMetadata = TestHelper.ReadDeviceMetadata(metadataFileName); + var generator = new PythonGenerator(deviceMetadata); + var implementation = generator.GenerateImplementation(); + TestHelper.AssertNoGeneratorErrors(generator.Errors); + + var outputFileName = $"{Path.GetFileNameWithoutExtension(metadataFileName)}.py"; + try + { + TestHelper.AssertExpectedOutput(implementation.Device, outputFileName); + } + catch (AssertFailedException) + { + if (outputDirectory is not null) + { + outputDirectory.Create(); + File.WriteAllText(Path.Combine(outputDirectory.FullName, outputFileName), implementation.Device); + } + throw; + } + } +} diff --git a/tests/PythonInteropTests.cs b/tests/PythonInteropTests.cs new file mode 100644 index 0000000..004e9c7 --- /dev/null +++ b/tests/PythonInteropTests.cs @@ -0,0 +1,272 @@ +using System.Globalization; +using System.Reflection; +using System.Text; +using Bonsai.Harp; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Harp.Generators.Tests; + +[TestClass] +public sealed class PythonInteropTests +{ + const string DeviceMetadataFileName = "device.yml"; + const int FrameCount = 3; + + static string ResolveOutputDirectory() + { + var configured = Environment.GetEnvironmentVariable("HARP_INTEROP_OUTPUT"); + if (!string.IsNullOrEmpty(configured)) + return configured; + + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "Harp.Generators.sln"))) + directory = directory.Parent; + var root = directory?.FullName ?? AppContext.BaseDirectory; + return Path.Combine(root, "artifacts", "python-interop"); + } + + [TestMethod] + public void GenerateInteropFixtures() + { + var metadataPath = TestHelper.GetMetadataPath(DeviceMetadataFileName); + var deviceMetadata = TestHelper.ReadDeviceMetadata(metadataPath); + + var interfaceImplementation = new InterfaceGenerator(deviceMetadata, typeof(PythonInteropTests).Namespace ?? "").GenerateImplementation(); + var pythonImplementation = new PythonGenerator(deviceMetadata).GenerateImplementation(); + var payloadExtensions = TestHelper.GetManifestResourceText("PayloadMarshal.cs"); + var customImplementation = TestHelper.GetManifestResourceText("EmbeddedSources.device.cs"); + var assembly = CompilerTestHelper.CompileAndLoadFromSource( + interfaceImplementation.Device, + interfaceImplementation.AsyncDevice, + payloadExtensions, + customImplementation); + + var outputDirectory = ResolveOutputDirectory(); + var packageDirectory = Path.Combine(outputDirectory, "harp_device"); + var dataDirectory = Path.Combine(outputDirectory, "data"); + Directory.CreateDirectory(packageDirectory); + Directory.CreateDirectory(dataDirectory); + + File.WriteAllText(Path.Combine(packageDirectory, "device.py"), pythonImplementation.Device); + File.WriteAllText(Path.Combine(packageDirectory, "__init__.py"), string.Empty); + File.WriteAllText(Path.Combine(packageDirectory, "converters.py"), TestHelper.GetManifestResourceText("Python.converters.py")); + File.WriteAllText(Path.Combine(outputDirectory, DeviceMetadataFileName), File.ReadAllText(metadataPath)); + + var manifest = new StringBuilder(); + manifest.Append("[\n"); + var first = true; + foreach (var registerMetadata in deviceMetadata.Registers) + { + var register = registerMetadata.Value; + if (register.Visibility != RegisterVisibility.Public) + continue; + + if (register.PayloadSpec == null && register.HasConverter) + continue; + + var registerType = assembly.GetType($"{typeof(PythonInteropTests).Namespace}.{registerMetadata.Key}") + ?? throw new InvalidOperationException($"Generated register type not found: {registerMetadata.Key}"); + var fromPayload = FindFromPayloadMethod(registerType); + var valueType = fromPayload.GetParameters()[1].ParameterType; + + var expected = new List(); + using (var binaryStream = new FileStream(Path.Combine(dataDirectory, $"{registerMetadata.Key}.bin"), FileMode.Create)) + { + for (int seed = 1; seed <= FrameCount; seed++) + { + var value = InteropValue.Build(valueType, register, seed); + var message = (HarpMessage)fromPayload.Invoke(null, [MessageType.Write, value])!; + binaryStream.Write(message.MessageBytes, 0, message.MessageBytes.Length); + expected.Add(InteropValue.Canonicalize(value, register)); + } + } + + if (!first) manifest.Append(",\n"); + first = false; + manifest.Append(" {\n"); + manifest.Append($" \"name\": \"{registerMetadata.Key}\",\n"); + manifest.Append($" \"address\": {register.Address},\n"); + manifest.Append($" \"frames\": {FrameCount},\n"); + manifest.Append(" \"expected\": [").Append(string.Join(", ", expected)).Append("]\n"); + manifest.Append(" }"); + } + manifest.Append("\n]\n"); + File.WriteAllText(Path.Combine(outputDirectory, "manifest.json"), manifest.ToString()); + + Assert.IsTrue(Directory.GetFiles(dataDirectory, "*.bin").Length > 0); + } + + static MethodInfo FindFromPayloadMethod(Type registerType) + { + foreach (var method in registerType.GetMethods(BindingFlags.Public | BindingFlags.Static)) + { + if (method.Name != "FromPayload") continue; + var parameters = method.GetParameters(); + if (parameters.Length == 2 && parameters[0].ParameterType == typeof(MessageType)) + return method; + } + throw new InvalidOperationException($"No FromPayload(MessageType, value) method on {registerType.Name}."); + } +} + +static class InteropValue +{ + public static object Build(Type valueType, RegisterInfo register, int seed) + { + if (register.PayloadSpec != null) + { + var instance = Activator.CreateInstance(valueType)!; + foreach (var member in register.PayloadSpec) + { + var field = valueType.GetField(member.Key) + ?? throw new InvalidOperationException($"Field not found on payload struct: {member.Key}"); + field.SetValue(instance, BuildField(field.FieldType, member.Value, seed)); + } + return instance; + } + + return BuildScalar(valueType, Math.Max(1, register.Length), GetElementRange(register.Type), seed); + } + + static object BuildField(Type fieldType, PayloadMemberInfo member, int seed) + { + if (fieldType.IsArray) + return BuildArray(fieldType.GetElementType()!, member.Length.GetValueOrDefault(1), seed); + if (fieldType.IsEnum) + return SmallestFittingEnumValue(fieldType, GetFieldRange(member)); + return BuildScalar(fieldType, member.Length.GetValueOrDefault(1), GetFieldRange(member), seed); + } + + static long GetElementRange(PayloadType payloadType) + { + var size = (int)payloadType & 0xF; + return size >= 8 ? long.MaxValue : (1L << (size * 8)) - 1; + } + + static long GetFieldRange(PayloadMemberInfo member) + { + if (!member.Mask.HasValue) return long.MaxValue; + var mask = member.Mask.GetValueOrDefault(); + var shift = 0; + while (((mask >> shift) & 1) == 0 && shift < 32) shift++; + return mask >> shift; + } + + static object BuildScalar(Type type, int length, long range, int seed) + { + if (type.IsArray) + return BuildArray(type.GetElementType()!, length, seed); + if (type == typeof(string)) + return new string('A', Math.Min(length > 1 ? length - 1 : 3, 8)); + if (type == typeof(bool)) + return seed % 2 == 1; + if (type.IsEnum) + return SmallestFittingEnumValue(type, range); + if (type.FullName == "Bonsai.Harp.HarpVersion") + return BuildHarpVersion(type, seed); + if (type == typeof(float)) + return seed + 0.5f; + if (type == typeof(double)) + return seed + 0.5d; + var clamped = range >= long.MaxValue ? seed : (int)Math.Min(seed, range); + return Convert.ChangeType(clamped, type, CultureInfo.InvariantCulture); + } + + static object BuildArray(Type elementType, int length, int seed) + { + var array = Array.CreateInstance(elementType, Math.Max(1, length)); + for (int i = 0; i < array.Length; i++) + array.SetValue(BuildScalar(elementType, 1, long.MaxValue, seed + i), i); + return array; + } + + static object SmallestFittingEnumValue(Type type, long fieldRange) + { + object? chosen = null; + long chosenValue = long.MaxValue; + foreach (var value in Enum.GetValues(type)) + { + var numeric = Convert.ToInt64(value, CultureInfo.InvariantCulture); + if (numeric <= 0 || numeric > fieldRange) continue; + if (numeric < chosenValue) + { + chosenValue = numeric; + chosen = value; + } + } + return chosen ?? Enum.GetValues(type).GetValue(0)!; + } + + static object BuildHarpVersion(Type type, int seed) + { + var constructor = type.GetConstructors() + .OrderByDescending(c => c.GetParameters().Length) + .First(c => c.GetParameters().Length >= 2); + var parameters = constructor.GetParameters(); + var arguments = new object?[parameters.Length]; + for (int i = 0; i < parameters.Length; i++) + { + var parameterType = Nullable.GetUnderlyingType(parameters[i].ParameterType) ?? parameters[i].ParameterType; + if (parameterType.IsValueType && i < 2) + arguments[i] = Convert.ChangeType(seed + i, parameterType, CultureInfo.InvariantCulture); + else + arguments[i] = parameters[i].HasDefaultValue ? parameters[i].DefaultValue : null; + } + return constructor.Invoke(arguments); + } + + public static string Canonicalize(object value, RegisterInfo register) + { + if (register.PayloadSpec != null) + { + var type = value.GetType(); + var members = register.PayloadSpec.Select(member => + { + var field = type.GetField(member.Key)!; + return $"\"{ToSnakeCase(member.Key)}\": {CanonicalizeLeaf(field.GetValue(value)!)}"; + }); + return $"{{{string.Join(", ", members)}}}"; + } + + return CanonicalizeLeaf(value); + } + + static string CanonicalizeLeaf(object value) + { + var type = value.GetType(); + if (type == typeof(bool)) return (bool)value ? "1" : "0"; + if (type == typeof(string)) return $"\"{value}\""; + if (type.IsEnum) return Convert.ToInt64(value, CultureInfo.InvariantCulture).ToString(CultureInfo.InvariantCulture); + if (type.FullName == "Bonsai.Harp.HarpVersion") return HarpVersionToJson(value); + if (type == typeof(float) || type == typeof(double)) + return Convert.ToDouble(value, CultureInfo.InvariantCulture).ToString("0.000", CultureInfo.InvariantCulture); + if (type.IsArray) + { + var array = (Array)value; + var elements = new string[array.Length]; + for (int i = 0; i < array.Length; i++) + elements[i] = CanonicalizeLeaf(array.GetValue(i)!); + return $"[{string.Join(", ", elements)}]"; + } + return Convert.ToInt64(value, CultureInfo.InvariantCulture).ToString(CultureInfo.InvariantCulture); + } + + static string HarpVersionToJson(object value) + { + var type = value.GetType(); + var major = type.GetProperty("Major")?.GetValue(value); + var minor = type.GetProperty("Minor")?.GetValue(value); + return $"[{ConvertComponent(major)}, {ConvertComponent(minor)}]"; + } + + static string ConvertComponent(object? component) + { + if (component is null) return "0"; + return Convert.ToInt64(component, CultureInfo.InvariantCulture).ToString(CultureInfo.InvariantCulture); + } + + static string ToSnakeCase(string name) + { + return FirmwareNamingConvention.Instance.Apply(name).ToLowerInvariant(); + } +}