From 72e75cd2dea0fa3fe547a511924d9191fc87d926 Mon Sep 17 00:00:00 2001 From: Matthias Baur Date: Tue, 19 May 2026 15:00:19 +0200 Subject: [PATCH] Remove instances in ERROR state rather than marking them timed out The current implementation only marks ERRORed instances as timed out. This is removes them from fleeting, but keeps them present in OpenStack. With the new implementation the instances gets removed from OpenStack as well. --- provider.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/provider.go b/provider.go index 532b23b..3425861 100644 --- a/provider.go +++ b/provider.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path" + "strings" "sync/atomic" "time" @@ -105,14 +106,21 @@ func (g *InstanceGroup) Init(ctx context.Context, log hclog.Logger, settings pro } func (g *InstanceGroup) Update(ctx context.Context, update func(instance string, state provider.State)) error { - instances, err := g.getInstances(ctx) if err != nil { return err } var reterr error + var cleanup []string + for _, srv := range instances { + // VMs in ERROR state are unrecoverable, clean them up + if srv.Status == "ERROR" { + cleanup = append(cleanup, srv.ID) + continue + } + state := provider.StateCreating lg := g.log.With("server_id", srv.ID, "created", srv.Created, "status", srv.Status) @@ -123,11 +131,6 @@ func (g *InstanceGroup) Update(ctx context.Context, update func(instance string, case "DELETED", "SHUTOFF", "UNKNOWN": state = provider.StateDeleting - case "ERROR": - // unsure if that's proper way... - lg.Warn("Instance is in ERROR state. Marking as a timeout.") - state = provider.StateTimeout - case "ACTIVE": if srv.Created.Add(g.BootTime).Before(time.Now()) { // treat all nodes running long enough as Running @@ -154,6 +157,11 @@ func (g *InstanceGroup) Update(ctx context.Context, update func(instance string, update(srv.ID, state) } + if len(cleanup) > 0 { + _, err := g.Decrease(ctx, cleanup) + g.log.Warn("Removed instances in ERROR state", "instances", strings.Join(cleanup, ","), "err", err) + } + return reterr }