Skip to content

feat(dart): add bfloat16 and bfloat16_array support #3564

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
miantalha45 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(dart): add bfloat16 and bfloat16_array support #3564

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dart/packages/fory/lib/fory.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export 'src/serializer/enum_serializer.dart';
export 'src/serializer/serializer.dart';
export 'src/serializer/union_serializer.dart';
export 'src/types/fixed_ints.dart';
export 'src/types/bfloat16.dart';
export 'src/types/float16.dart';
export 'src/types/float32.dart';
export 'src/types/local_date.dart';
Expand Down
7 changes: 7 additions & 0 deletions dart/packages/fory/lib/src/buffer.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import 'dart:typed_data';

import 'package:meta/meta.dart';

import 'package:fory/src/types/bfloat16.dart';
import 'package:fory/src/types/float16.dart';

final BigInt _mask64Big = (BigInt.one << 64) - BigInt.one;
Expand Down Expand Up @@ -236,6 +237,12 @@ final class Buffer {
/// Reads a half-precision floating-point value.
Float16 readFloat16() => Float16.fromBits(readUint16());

/// Writes a brain floating-point (bfloat16) value.
void writeBfloat16(BFloat16 value) => writeUint16(value.toBits());

/// Reads a brain floating-point (bfloat16) value.
BFloat16 readBfloat16() => BFloat16.fromBits(readUint16());

/// Writes [value] verbatim.
void writeBytes(List<int> value) {
ensureWritable(value.length);
Expand Down
24 changes: 20 additions & 4 deletions dart/packages/fory/lib/src/codegen/fory_generator.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,7 @@ final class ForyGenerator extends Generator {

final fields = element.fields
.where(
(field) =>
!field.isStatic &&
!field.isSynthetic,
(field) => !field.isStatic && !field.isSynthetic,
)
.where((field) => !_isSkipped(field))
.map(_analyzeField)
Expand Down Expand Up @@ -669,7 +667,8 @@ final class ForyGenerator extends Generator {
" throw ArgumentError('Exactly one registration mode is required: id, or namespace + typeName.');",
)
..writeln(' }')
..writeln(' if (hasNamed && (namespace == null || typeName == null)) {')
..writeln(
' if (hasNamed && (namespace == null || typeName == null)) {')
..writeln(
" throw ArgumentError('Both namespace and typeName are required for named registration.');",
)
Expand Down Expand Up @@ -1002,6 +1001,7 @@ GeneratedFieldType(
case TypeIds.int16:
case TypeIds.uint16:
case TypeIds.float16:
case TypeIds.bfloat16:
return 2;
case TypeIds.int32:
case TypeIds.uint32:
Expand Down Expand Up @@ -1065,6 +1065,8 @@ GeneratedFieldType(
return 'buffer.writeTaggedUint64(${_directGeneratedScalarExpression(field, valueExpression)})';
case TypeIds.float16:
return 'buffer.writeFloat16($valueExpression)';
case TypeIds.bfloat16:
return 'buffer.writeBfloat16($valueExpression)';
case TypeIds.float32:
return 'buffer.writeFloat32(${_directGeneratedScalarExpression(field, valueExpression)})';
case TypeIds.float64:
Expand All @@ -1088,6 +1090,7 @@ GeneratedFieldType(
case TypeIds.uint32Array:
case TypeIds.uint64Array:
case TypeIds.float16Array:
case TypeIds.bfloat16Array:
case TypeIds.float32Array:
case TypeIds.float64Array:
return 'writeGeneratedFixedArrayValue(context, $valueExpression)';
Expand Down Expand Up @@ -1138,6 +1141,8 @@ GeneratedFieldType(
return '$cursorExpression.writeTaggedUint64(${_directGeneratedScalarExpression(field, valueExpression)})';
case TypeIds.float16:
return '$cursorExpression.writeFloat16($valueExpression)';
case TypeIds.bfloat16:
return '$cursorExpression.writeBfloat16($valueExpression)';
case TypeIds.float32:
return '$cursorExpression.writeFloat32(${_directGeneratedScalarExpression(field, valueExpression)})';
case TypeIds.float64:
Expand Down Expand Up @@ -1201,6 +1206,8 @@ GeneratedFieldType(
return 'buffer.readTaggedUint64()';
case TypeIds.float16:
return 'buffer.readFloat16()';
case TypeIds.bfloat16:
return 'buffer.readBfloat16()';
case TypeIds.float32:
return field.type.isDartCoreDouble
? 'buffer.readFloat32()'
Expand Down Expand Up @@ -1229,6 +1236,7 @@ GeneratedFieldType(
return 'readGeneratedBinaryValue(context)';
case TypeIds.uint16Array:
case TypeIds.float16Array:
case TypeIds.bfloat16Array:
return 'readGeneratedTypedArrayValue<Uint16List>(context, 2, (bytes) => bytes.buffer.asUint16List(bytes.offsetInBytes, bytes.lengthInBytes ~/ 2))';
case TypeIds.uint32Array:
return 'readGeneratedTypedArrayValue<Uint32List>(context, 4, (bytes) => bytes.buffer.asUint32List(bytes.offsetInBytes, bytes.lengthInBytes ~/ 4))';
Expand Down Expand Up @@ -1296,6 +1304,8 @@ GeneratedFieldType(
return '$cursorExpression.readTaggedUint64()';
case TypeIds.float16:
return '$cursorExpression.readFloat16()';
case TypeIds.bfloat16:
return '$cursorExpression.readBfloat16()';
case TypeIds.float32:
return field.type.isDartCoreDouble
? '$cursorExpression.readFloat32()'
Expand Down Expand Up @@ -1349,6 +1359,7 @@ GeneratedFieldType(
}
switch (field.fieldType.typeId) {
case TypeIds.float16:
case TypeIds.bfloat16:
return valueExpression;
default:
return '$valueExpression.value';
Expand Down Expand Up @@ -1577,6 +1588,7 @@ GeneratedFieldType(
case TypeIds.int16:
case TypeIds.uint16:
case TypeIds.float16:
case TypeIds.bfloat16:
return 2;
case TypeIds.int32:
case TypeIds.varInt32:
Expand Down Expand Up @@ -1629,6 +1641,7 @@ GeneratedFieldType(
case TypeIds.varUint64:
case TypeIds.taggedUint64:
case TypeIds.float16:
case TypeIds.bfloat16:
case TypeIds.float32:
case TypeIds.float64:
return true;
Expand All @@ -1653,6 +1666,7 @@ GeneratedFieldType(
case TypeIds.uint32Array:
case TypeIds.uint64Array:
case TypeIds.float16Array:
case TypeIds.bfloat16Array:
case TypeIds.float32Array:
case TypeIds.float64Array:
return true;
Expand Down Expand Up @@ -1792,6 +1806,8 @@ GeneratedFieldType(
return TypeIds.uint32;
case 'Float16':
return TypeIds.float16;
case 'BFloat16':
return TypeIds.bfloat16;
case 'Float32':
return TypeIds.float32;
case 'Timestamp':
Expand Down
6 changes: 6 additions & 0 deletions dart/packages/fory/lib/src/codegen/generated_support.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,10 @@ final class GeneratedWriteCursor {
writeUint16(value.toBits());
}

void writeBfloat16(BFloat16 value) {
writeUint16(value.toBits());
}

void writeFloat32(double value) {
_view.setFloat32(_offset, value, Endian.little);
_offset += 4;
Expand Down Expand Up @@ -271,6 +275,8 @@ final class GeneratedReadCursor {

Float16 readFloat16() => Float16.fromBits(readUint16());

BFloat16 readBfloat16() => BFloat16.fromBits(readUint16());

double readFloat32() {
final value = _view.getFloat32(_offset, Endian.little);
_offset += 4;
Expand Down
9 changes: 7 additions & 2 deletions dart/packages/fory/lib/src/context/meta_string_reader.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,13 @@ import 'package:fory/src/buffer.dart';
import 'package:fory/src/meta/meta_string.dart';
import 'package:fory/src/resolver/type_resolver.dart';

typedef _MetaStringWords =
({int length, int word0, int word1, int word2, int word3});
typedef _MetaStringWords = ({
int length,
int word0,
int word1,
int word2,
int word3
});

/// Read-side state for meta-string references in one deserialization stream.
final class MetaStringReader {
Expand Down
4 changes: 4 additions & 0 deletions dart/packages/fory/lib/src/context/read_context.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import 'package:fory/src/serializer/primitive_serializers.dart';
import 'package:fory/src/serializer/scalar_serializers.dart';
import 'package:fory/src/serializer/serializer.dart';
import 'package:fory/src/serializer/typed_array_serializers.dart';
import 'package:fory/src/types/bfloat16.dart';
import 'package:fory/src/types/float16.dart';

/// Read-side serializer context.
Expand Down Expand Up @@ -157,6 +158,9 @@ final class ReadContext {
/// Reads a half-precision floating-point value.
Float16 readFloat16() => _buffer.readFloat16();

/// Reads a brain floating-point (bfloat16) value.
BFloat16 readBfloat16() => _buffer.readBfloat16();

/// Reads a single-precision floating-point value.
double readFloat32() => _buffer.readFloat32();

Expand Down
5 changes: 5 additions & 0 deletions dart/packages/fory/lib/src/context/write_context.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import 'package:fory/src/serializer/map_serializers.dart';
import 'package:fory/src/serializer/primitive_serializers.dart';
import 'package:fory/src/serializer/scalar_serializers.dart';
import 'package:fory/src/serializer/typed_array_serializers.dart';
import 'package:fory/src/types/bfloat16.dart';
import 'package:fory/src/types/float16.dart';
import 'package:fory/src/types/local_date.dart';
import 'package:fory/src/types/timestamp.dart';
Expand Down Expand Up @@ -140,6 +141,9 @@ final class WriteContext {
/// Writes a half-precision floating-point value.
void writeFloat16(Float16 value) => _buffer.writeFloat16(value);

/// Writes a brain floating-point (bfloat16) value.
void writeBfloat16(BFloat16 value) => _buffer.writeBfloat16(value);

/// Writes a single-precision floating-point value.
void writeFloat32(double value) => _buffer.writeFloat32(value);

Expand Down Expand Up @@ -272,6 +276,7 @@ final class WriteContext {
case TypeIds.varUint64:
case TypeIds.taggedUint64:
case TypeIds.float16:
case TypeIds.bfloat16:
case TypeIds.float32:
case TypeIds.float64:
PrimitiveSerializer.writePayload(this, resolved.typeId, value);
Expand Down
2 changes: 2 additions & 0 deletions dart/packages/fory/lib/src/meta/type_meta.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ final class WireTypeMetaDecoder {
wireTypeId == TypeIds.varUint64 ||
wireTypeId == TypeIds.taggedUint64 ||
wireTypeId == TypeIds.float16 ||
wireTypeId == TypeIds.bfloat16 ||
wireTypeId == TypeIds.float32 ||
wireTypeId == TypeIds.float64 ||
wireTypeId == TypeIds.string ||
Expand All @@ -258,6 +259,7 @@ final class WireTypeMetaDecoder {
wireTypeId == TypeIds.uint32Array ||
wireTypeId == TypeIds.uint64Array ||
wireTypeId == TypeIds.float16Array ||
wireTypeId == TypeIds.bfloat16Array ||
wireTypeId == TypeIds.float32Array ||
wireTypeId == TypeIds.float64Array;
}
20 changes: 20 additions & 0 deletions dart/packages/fory/lib/src/resolver/type_resolver.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import 'package:fory/src/serializer/struct_serializer.dart';
import 'package:fory/src/serializer/typed_array_serializers.dart';
import 'package:fory/src/serializer/union_serializer.dart';
import 'package:fory/src/types/fixed_ints.dart';
import 'package:fory/src/types/bfloat16.dart';
import 'package:fory/src/types/float16.dart';
import 'package:fory/src/types/float32.dart';
import 'package:fory/src/types/local_date.dart';
Expand All @@ -46,6 +47,7 @@ abstract final class TypeIds {
static const int varUint64 = 14;
static const int taggedUint64 = 15;
static const int float16 = 17;
static const int bfloat16 = 18;
static const int float32 = 19;
static const int float64 = 20;
static const int string = 21;
Expand Down Expand Up @@ -77,6 +79,7 @@ abstract final class TypeIds {
static const int uint32Array = 50;
static const int uint64Array = 51;
static const int float16Array = 53;
static const int bfloat16Array = 54;
static const int float32Array = 55;
static const int float64Array = 56;

Expand All @@ -97,6 +100,7 @@ abstract final class TypeIds {
typeId == varUint64 ||
typeId == taggedUint64 ||
typeId == float16 ||
typeId == bfloat16 ||
typeId == float32 ||
typeId == float64;

Expand Down Expand Up @@ -132,6 +136,7 @@ abstract final class TypeIds {
typeId == uint32Array ||
typeId == uint64Array ||
typeId == float16Array ||
typeId == bfloat16Array ||
typeId == float32Array ||
typeId == float64Array;

Expand All @@ -153,6 +158,7 @@ abstract final class TypeIds {
case uint32Array:
case uint64Array:
case float16Array:
case bfloat16Array:
case float32Array:
case float64Array:
return false;
Expand Down Expand Up @@ -443,6 +449,9 @@ final class TypeResolver {
if (value is Float16) {
return _builtin(Float16, TypeIds.float16);
}
if (value is BFloat16) {
return _builtin(BFloat16, TypeIds.bfloat16);
}
if (value is Float32) {
return _builtin(Float32, TypeIds.float32);
}
Expand Down Expand Up @@ -525,6 +534,7 @@ final class TypeResolver {
case TypeIds.varUint64:
case TypeIds.taggedUint64:
case TypeIds.float16:
case TypeIds.bfloat16:
case TypeIds.float32:
case TypeIds.float64:
case TypeIds.string:
Expand All @@ -544,6 +554,7 @@ final class TypeResolver {
case TypeIds.uint32Array:
case TypeIds.uint64Array:
case TypeIds.float16Array:
case TypeIds.bfloat16Array:
case TypeIds.float32Array:
case TypeIds.float64Array:
return _builtin(fieldType.type, fieldType.typeId);
Expand Down Expand Up @@ -1117,6 +1128,8 @@ final class TypeResolver {
return _builtin(int, TypeIds.taggedUint64);
case TypeIds.float16:
return _builtin(Float16, TypeIds.float16);
case TypeIds.bfloat16:
return _builtin(BFloat16, TypeIds.bfloat16);
case TypeIds.float32:
return _builtin(Float32, TypeIds.float32);
case TypeIds.float64:
Expand Down Expand Up @@ -1155,6 +1168,8 @@ final class TypeResolver {
return _builtin(Uint64List, TypeIds.uint64Array);
case TypeIds.float16Array:
return _builtin(Uint16List, TypeIds.float16Array);
case TypeIds.bfloat16Array:
return _builtin(Uint16List, TypeIds.bfloat16Array);
case TypeIds.float32Array:
return _builtin(Float32List, TypeIds.float32Array);
case TypeIds.float64Array:
Expand Down Expand Up @@ -1222,6 +1237,8 @@ final class TypeResolver {
return taggedUint64Serializer as Serializer<Object?>;
case TypeIds.float16:
return float16Serializer as Serializer<Object?>;
case TypeIds.bfloat16:
return bfloat16Serializer as Serializer<Object?>;
case TypeIds.float32:
return float32Serializer as Serializer<Object?>;
case TypeIds.float64:
Expand Down Expand Up @@ -1307,6 +1324,9 @@ final class TypeResolver {
if (type == Float16) {
return TypeIds.float16;
}
if (type == BFloat16) {
return TypeIds.bfloat16;
}
if (type == Float32) {
return TypeIds.float32;
}
Expand Down
14 changes: 7 additions & 7 deletions dart/packages/fory/lib/src/serializer/map_serializers.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,13 +277,13 @@ Map<K, V> readTypedMapPayload<K, V>(
final valueTypeInfo = valueDeclared ? null : context.readTypeMetaValue();
final tracksDepth =
((keyDeclared ? declaredKeyTypeInfo : keyTypeInfo) != null &&
tracksNestedPayloadDepth(
keyDeclared ? declaredKeyTypeInfo! : keyTypeInfo!,
)) ||
((valueDeclared ? declaredValueTypeInfo : valueTypeInfo) != null &&
tracksNestedPayloadDepth(
valueDeclared ? declaredValueTypeInfo! : valueTypeInfo!,
));
tracksNestedPayloadDepth(
keyDeclared ? declaredKeyTypeInfo! : keyTypeInfo!,
)) ||
((valueDeclared ? declaredValueTypeInfo : valueTypeInfo) != null &&
tracksNestedPayloadDepth(
valueDeclared ? declaredValueTypeInfo! : valueTypeInfo!,
));
if (tracksDepth) {
context.increaseDepth();
}
Expand Down
Loading
Loading