Skip to content
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 87 additions & 0 deletions .travis/benches.toml
Original file line number Diff line number Diff line change
Expand Up @@ -300,3 +300,90 @@ kind = "llm"
name = "qwen3-1_7B-q40ef16-541"
model = "Qwen3-1.7B-q40ef16.541.nnef.tgz"
backends = ["metal", "cuda"]

# --- Pocket TTS (Kyutai, CC-BY-4.0; 100M-param CPU TTS, 24 kHz / 12.5 Hz frames).
# ONNX export with explicit KV-cache states rewritten shape-static so the streaming
# step position is a runtime input. decode = one 80 ms frame of the flow LM;
# prefill25 = one 25-token text chunk; flow = one Euler step of the latent flow net;
# mimi decoder = a 12-frame (960 ms) chunk. Random i64 inputs cast to 0: a valid
# stream position, and the graphs are full-window so compute is position-independent.

[[bench]]
kind = "net"
name = "pocket-tts-flow-lm"
variant = "decode"
model = "pocket-tts-en/flow_lm_main_static.onnx"
archive = "pocket-tts-en.tar.gz"
args = """--onnx-ignore-output-shapes --onnx-ignore-output-types --onnx-ignore-value-info
-i 1,1,32,f32 -i 1,0,1024,f32
-i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64 -i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64 -i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64 -i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64 -i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64 -i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64"""

[[bench]]
kind = "net"
name = "pocket-tts-flow-lm"
variant = "prefill25"
model = "pocket-tts-en/flow_lm_main_static.onnx"
archive = "pocket-tts-en.tar.gz"
args = """--onnx-ignore-output-shapes --onnx-ignore-output-types --onnx-ignore-value-info
-i 1,0,32,f32 -i 1,25,1024,f32
-i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64 -i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64 -i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64 -i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64 -i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64 -i 2,1,1000,16,64,f32 -i 0,f32 -i 1,i64"""

[[bench]]
kind = "net"
name = "pocket-tts-flow-net"
variant = "euler1"
model = "pocket-tts-en/flow_lm_flow.onnx"
archive = "pocket-tts-en.tar.gz"
args = "--onnx-ignore-output-shapes -i 1,1024,f32 -i 1,1,f32 -i 1,1,f32 -i 1,32,f32"

[[bench]]
kind = "net"
name = "pocket-tts-mimi-decoder"
variant = "chunk12"
model = "pocket-tts-en/mimi_decoder.onnx"
archive = "pocket-tts-en.tar.gz"
args = """--onnx-ignore-output-shapes --onnx-ignore-output-types --onnx-ignore-value-info
-i 1,12,32,f32 -i 1,bool -i 1,512,6,f32 -i 1,bool -i 1,64,2,f32 -i 1,256,6,f32 -i 1,bool -i 1,256,2,f32 -i 1,bool -i 1,128,0,f32 -i 1,128,5,f32 -i 1,bool -i 1,128,2,f32 -i 1,bool -i 1,64,0,f32 -i 1,64,4,f32 -i 1,bool -i 1,64,2,f32 -i 1,bool -i 1,32,0,f32 -i 2,1,8,1000,64,f32 -i 1,i64 -i 1,i64 -i 2,1,8,1000,64,f32 -i 1,i64 -i 1,i64 -i 1,bool -i 1,512,16,f32 -i 1,bool -i 1,1,6,f32 -i 1,bool -i 1,64,2,f32 -i 1,bool -i 1,32,0,f32 -i 1,bool -i 1,512,2,f32 -i 1,bool -i 1,64,4,f32 -i 1,bool -i 1,128,2,f32 -i 1,bool -i 1,64,0,f32 -i 1,bool -i 1,128,5,f32 -i 1,bool -i 1,256,2,f32 -i 1,bool -i 1,128,0,f32 -i 1,bool -i 1,256,6,f32 -i 2,1,8,1000,64,f32 -i 1,i64 -i 1,i64 -i 2,1,8,1000,64,f32 -i 1,i64 -i 1,i64 -i 1,512,16,f32"""

# --- DeepFilterNet3 (Rikorose/DeepFilterNet, MIT/Apache-2.0) — speech enhancement.
# The three streaming submodels, symbolic time pinned at export to 100 frames
# (1 s of audio at the 10 ms hop): encoder, ERB-mask decoder, deep-filter decoder.

[[bench]]
kind = "net"
name = "dfn3-enc"
variant = "1s"
model = "dfn3/enc.onnx"
archive = "dfn3.tar.gz"

[[bench]]
kind = "net"
name = "dfn3-erb-dec"
variant = "1s"
model = "dfn3/erb_dec.onnx"
archive = "dfn3.tar.gz"

[[bench]]
kind = "net"
name = "dfn3-df-dec"
variant = "1s"
model = "dfn3/df_dec.onnx"
archive = "dfn3.tar.gz"

# --- DTLN (breizhn/DTLN, MIT) — dual-stage LSTM speech denoiser, one 8 ms frame
# per call with LSTM state threaded through graph I/O (external-state pattern).
# model_2's free batch dim on outputs is pinned to 1 (tf2onnx export artifact).

[[bench]]
kind = "net"
name = "dtln-stage1"
variant = "frame"
model = "dtln/model_1.onnx"
archive = "dtln.tar.gz"

[[bench]]
kind = "net"
name = "dtln-stage2"
variant = "frame"
model = "dtln/model_2_fixed.onnx"
archive = "dtln.tar.gz"