Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ on:
- main
- master
pull_request:
branches:
- main
- master

jobs:
test:
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# MLX Ruby Examples

[![CI](https://github.com/skryl/mlx-ruby-examples/actions/workflows/ci.yml/badge.svg)](https://github.com/skryl/mlx-ruby-examples/actions/workflows/ci.yml)

Ruby/MLX example ports that depend on Apple Metal when building the `mlx` gem.

## Prerequisites (macOS)
Expand Down
14 changes: 7 additions & 7 deletions lora/lora.rb
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ def train(model, train_set, valid_set, optimizer, tokenizer, options)
seed: options[:seed]
)

options[:iters].times do |it|
options[:iters].times do |iter|
inputs, targets, lengths = batches.next
(lvalue, toks), grad = loss_value_and_grad.call(inputs, targets, lengths)
optimizer.update(model, grad)
Expand All @@ -164,12 +164,12 @@ def train(model, train_set, valid_set, optimizer, tokenizer, options)
losses << lvalue.item.to_f
n_tokens += toks.item.to_f

if ((it + 1) % options[:steps_per_report]).zero?
if ((iter + 1) % options[:steps_per_report]).zero?
stop = Process.clock_gettime(Process::CLOCK_MONOTONIC)
train_loss = losses.sum / [losses.length, 1].max.to_f
puts format(
"Iter %d: Train loss %.3f, It/sec %.3f, Tokens/sec %.3f",
it + 1,
iter + 1,
train_loss,
options[:steps_per_report] / [stop - start, 1e-9].max,
n_tokens / [stop - start, 1e-9].max
Expand All @@ -179,7 +179,7 @@ def train(model, train_set, valid_set, optimizer, tokenizer, options)
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
end

if it.zero? || ((it + 1) % options[:steps_per_eval]).zero?
if iter.zero? || ((iter + 1) % options[:steps_per_eval]).zero?
eval_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
val_loss = evaluate(
model,
Expand All @@ -191,16 +191,16 @@ def train(model, train_set, valid_set, optimizer, tokenizer, options)
)
puts format(
"Iter %d: Val loss %.3f, Val took %.3fs",
it + 1,
iter + 1,
val_loss,
Process.clock_gettime(Process::CLOCK_MONOTONIC) - eval_start
)
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
end

if ((it + 1) % options[:save_every]).zero?
if ((iter + 1) % options[:save_every]).zero?
save_trainable_adapters(options[:adapter_file], model)
puts "Iter #{it + 1}: Saved adapter weights to #{options[:adapter_file]}."
puts "Iter #{iter + 1}: Saved adapter weights to #{options[:adapter_file]}."
end
end
end
Expand Down
14 changes: 7 additions & 7 deletions no_dsl/lora/lora.rb
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ def train(model, train_set, valid_set, optimizer, tokenizer, options)
seed: options[:seed]
)

options[:iters].times do |it|
options[:iters].times do |iter|
inputs, targets, lengths = batches.next
(lvalue, toks), grad = loss_value_and_grad.call(inputs, targets, lengths)
optimizer.update(model, grad)
Expand All @@ -164,12 +164,12 @@ def train(model, train_set, valid_set, optimizer, tokenizer, options)
losses << lvalue.item.to_f
n_tokens += toks.item.to_f

if ((it + 1) % options[:steps_per_report]).zero?
if ((iter + 1) % options[:steps_per_report]).zero?
stop = Process.clock_gettime(Process::CLOCK_MONOTONIC)
train_loss = losses.sum / [losses.length, 1].max.to_f
puts format(
"Iter %d: Train loss %.3f, It/sec %.3f, Tokens/sec %.3f",
it + 1,
iter + 1,
train_loss,
options[:steps_per_report] / [stop - start, 1e-9].max,
n_tokens / [stop - start, 1e-9].max
Expand All @@ -179,7 +179,7 @@ def train(model, train_set, valid_set, optimizer, tokenizer, options)
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
end

if it.zero? || ((it + 1) % options[:steps_per_eval]).zero?
if iter.zero? || ((iter + 1) % options[:steps_per_eval]).zero?
eval_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
val_loss = evaluate(
model,
Expand All @@ -191,16 +191,16 @@ def train(model, train_set, valid_set, optimizer, tokenizer, options)
)
puts format(
"Iter %d: Val loss %.3f, Val took %.3fs",
it + 1,
iter + 1,
val_loss,
Process.clock_gettime(Process::CLOCK_MONOTONIC) - eval_start
)
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
end

if ((it + 1) % options[:save_every]).zero?
if ((iter + 1) % options[:save_every]).zero?
save_trainable_adapters(options[:adapter_file], model)
puts "Iter #{it + 1}: Saved adapter weights to #{options[:adapter_file]}."
puts "Iter #{iter + 1}: Saved adapter weights to #{options[:adapter_file]}."
end
end
end
Expand Down
6 changes: 3 additions & 3 deletions no_dsl/normalizing_flow/main.rb
Original file line number Diff line number Diff line change
Expand Up @@ -74,19 +74,19 @@ def run(options)
rng = Random.new(options[:seed] + 1)
all_indices = (0...x.shape[0]).to_a
tic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
options[:n_steps].times do |it|
options[:n_steps].times do |iter|
ids = all_indices.sample(options[:n_batch], random: rng)
batch = MLX::Core.take(x, MLX::Core.array(ids, MLX::Core.int32), 0)
loss, grads = loss_and_grad_fn.call(batch)
optimizer.update(model, grads)
MLX::Core.eval(loss, model.parameters, optimizer.state)

next unless ((it + 1) % options[:report_every]).zero?
next unless ((iter + 1) % options[:report_every]).zero?

toc = Process.clock_gettime(Process::CLOCK_MONOTONIC)
puts format(
"Step %d: Loss %.4f | It/sec %.2f",
it + 1,
iter + 1,
loss.item.to_f,
options[:report_every] / [toc - tic, 1e-9].max
)
Expand Down
12 changes: 6 additions & 6 deletions no_dsl/transformer_lm/main.rb
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,11 @@ def run(options)

losses = []
tic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
options[:num_iters].times do |it|
options[:num_iters].times do |iter|
warmup = if options[:lr_warmup] <= 0
1.0
else
[1.0, it.to_f / options[:lr_warmup].to_f].min
[1.0, iter.to_f / options[:lr_warmup].to_f].min
end
optimizer.learning_rate = warmup * options[:learning_rate]

Expand All @@ -160,20 +160,20 @@ def run(options)
MLX::Core.eval(loss, model.parameters, optimizer.state)
losses << loss.item.to_f

if ((it + 1) % options[:steps_per_report]).zero?
if ((iter + 1) % options[:steps_per_report]).zero?
toc = Process.clock_gettime(Process::CLOCK_MONOTONIC)
train_loss = losses.sum / [losses.length, 1].max.to_f
puts format(
"Iter %d: Train loss %.3f, It/sec %.3f",
it + 1,
iter + 1,
train_loss,
options[:steps_per_report] / [toc - tic, 1e-9].max
)
losses = []
tic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
end

next unless ((it + 1) % options[:steps_per_eval]).zero?
next unless ((iter + 1) % options[:steps_per_eval]).zero?

eval_tic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
val_loss = eval_fn(
Expand All @@ -186,7 +186,7 @@ def run(options)
val_ppl = Math.exp(val_loss)
puts format(
"Iter %d: Val loss %.3f, Val ppl %.3f, Val took %.3fs",
it + 1,
iter + 1,
val_loss,
val_ppl,
eval_toc - eval_tic
Expand Down
14 changes: 7 additions & 7 deletions transformer_lm/main.rb
Original file line number Diff line number Diff line change
Expand Up @@ -153,35 +153,35 @@ def run(options)
end

trainer.before_epoch do |ctx|
it = ctx.fetch(:epoch).to_i
iter = ctx.fetch(:epoch).to_i
warmup = if options[:lr_warmup] <= 0
1.0
else
[1.0, it.to_f / options[:lr_warmup].to_f].min
[1.0, iter.to_f / options[:lr_warmup].to_f].min
end
optimizer.learning_rate = warmup * options[:learning_rate]
end

losses = []
tic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
trainer.after_epoch do |ctx|
it = ctx.fetch(:epoch).to_i + 1
iter = ctx.fetch(:epoch).to_i + 1
losses << ctx.fetch(:epoch_loss).to_f

if (it % options[:steps_per_report]).zero?
if (iter % options[:steps_per_report]).zero?
toc = Process.clock_gettime(Process::CLOCK_MONOTONIC)
train_loss = losses.sum / [losses.length, 1].max.to_f
puts format(
"Iter %d: Train loss %.3f, It/sec %.3f",
it,
iter,
train_loss,
options[:steps_per_report] / [toc - tic, 1e-9].max
)
losses = []
tic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
end

next unless (it % options[:steps_per_eval]).zero?
next unless (iter % options[:steps_per_eval]).zero?

eval_tic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
val_loss = eval_fn(
Expand All @@ -194,7 +194,7 @@ def run(options)
val_ppl = Math.exp(val_loss)
puts format(
"Iter %d: Val loss %.3f, Val ppl %.3f, Val took %.3fs",
it,
iter,
val_loss,
val_ppl,
eval_toc - eval_tic
Expand Down