feat(restore): restore VM from :hibernate on cross-node create

provider/cocoon/create.go

@@ -46,13 +46,25 @@ func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
 		return nil
 	}
 
+	// A restore reuses wake()'s post-restore path (CH+Windows waits on the fresh
+	// NIC's lease; others run runPostCloneSetup) and skips the base-image post-clone.
+	restoring := meta.ReadRestoreFromHibernate(pod)
 	bootStart := time.Now()
 	v, sourceImage, err := p.bringUpVM(ctx, pod, spec)
 	if err != nil {
+		// A restore is a wake: count its failure like wake() does, not just create.
+		if restoring {
+			metrics.WakeTotal.WithLabelValues("failed").Inc()
+		}
 		p.failOp(ctx, pod, "CreateBringUpFailed", "create", err)
 		return err
 	}
-	metrics.VMBootDuration.WithLabelValues(spec.Mode, spec.Backend).Observe(time.Since(bootStart).Seconds())
+	// A restore is a clone-from-hibernate; label its boot "clone" like wake does.
+	bootMode := spec.Mode
+	if restoring {
+		bootMode = "clone"
+	}
+	metrics.VMBootDuration.WithLabelValues(bootMode, spec.Backend).Observe(time.Since(bootStart).Seconds())
 
 	if v.IP == "" && v.MAC != "" && p.LeaseParser != nil {
 		if lease, err := p.LeaseParser.LookupByMAC(v.MAC); err == nil {
@@ -66,7 +78,10 @@ func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
 	// trackPod first: goroutines below call markLifecycleState, which reads p.pods[key].
 	p.trackPod(pod, v)
 	willRunSAC := p.willRunSAC(spec, v)
-	if spec.OS == string(cocoonv1.OSWindows) {
+	if restoring {
+		p.dispatchHibernateRestore(pod, spec, v, "create")
+	}
+	if spec.OS == string(cocoonv1.OSWindows) && !restoring {
 		p.goBackground(func() {
 			ran, err := p.applyWindowsStaticIP(p.lifecycleCtx, pod, v)
 			if err != nil {
@@ -88,7 +103,7 @@ func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
 			}
 		})
 	}
-	if cloned {
+	if cloned && !restoring {
 		p.goBackground(func() {
 			p.runPostCloneSetup(p.lifecycleCtx, pod, spec, v, sourceImage, "create")
 		})
@@ -105,8 +120,9 @@ func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
 	p.mu.Unlock()
 	p.refreshStatus(ctx, pod)
 	p.notify(pod)
-	// Cloned defers Ready to runPostCloneSetup; Windows+static defers to applyWindowsStaticIP.
-	if !cloned && !willRunSAC && !p.lifecycleAlreadyFailed(pod) {
+	// Cloned defers Ready to runPostCloneSetup; Windows+static defers to applyWindowsStaticIP;
+	// restore defers to dispatchHibernateRestore.
+	if !cloned && !willRunSAC && !restoring && !p.lifecycleAlreadyFailed(pod) {
 		p.markLifecycleState(ctx, pod, meta.LifecycleStateReady, "")
 	}
 	metrics.PodLifecycleTotal.WithLabelValues("create", "ok").Inc()
@@ -132,6 +148,17 @@ func (p *Provider) bringUpVM(ctx context.Context, pod *corev1.Pod, spec meta.VMS
 		return nil, "", err
 	}
 	switch {
+	case meta.ReadRestoreFromHibernate(pod):
+		sourceName, err := p.resolveWakeSource(ctx, spec.VMName)
+		if err != nil {
+			return nil, "", err
+		}
+		v, err := p.cloneFromHibernate(ctx, spec, sourceName)
+		if err != nil {
+			return nil, "", err
+		}
+		return v, "", nil
+
 	case fromDir != "":
 		if mode == string(cocoonv1.AgentModeRun) {
 			return nil, "", fmt.Errorf("annotation %s is incompatible with mode=run", meta.AnnotationCloneFromDir)

provider/cocoon/delete_test.go

@@ -0,0 +1,34 @@
+package cocoon
+
+import (
+	"testing"
+
+	"github.com/cocoonstack/cocoon-common/meta"
+)
+
+// TestDeletePodForgottenVMTouchesNoSnapshot locks the invariant the cross-node
+// migration relies on: when the operator deletes the old pod, hibernate() has
+// already removed + forgotten the VM, so DeletePod takes its v==nil early
+// return and removes no snapshot. The epoch :hibernate checkpoint the target
+// node restores from must survive the old pod's deletion.
+func TestDeletePodForgottenVMTouchesNoSnapshot(t *testing.T) {
+	rt := &fakeRuntime{}
+	p := newTestProvider(t)
+	p.Runtime = rt
+
+	// Pod is never tracked → vmForPod returns nil (VM already forgotten).
+	pod := newPodWithSpec(meta.VMSpec{VMName: "vk-ns-demo-0"})
+
+	if err := p.DeletePod(t.Context(), pod); err != nil {
+		t.Fatalf("DeletePod: %v", err)
+	}
+	if len(rt.snapshotRemoveCalls) != 0 {
+		t.Errorf("forgotten-VM delete must remove no snapshot, got %v", rt.snapshotRemoveCalls)
+	}
+	if rt.removedID != "" {
+		t.Errorf("forgotten-VM delete must not call Runtime.Remove, got %q", rt.removedID)
+	}
+	if rt.snapshotSaveCount != 0 {
+		t.Errorf("forgotten-VM delete must not save a snapshot, got %d", rt.snapshotSaveCount)
+	}
+}

provider/cocoon/restore_test.go

@@ -0,0 +1,59 @@
+package cocoon
+
+import (
+	"testing"
+
+	cocoonv1 "github.com/cocoonstack/cocoon-common/apis/v1"
+	"github.com/cocoonstack/cocoon-common/meta"
+	"github.com/cocoonstack/vk-cocoon/probes"
+	"github.com/cocoonstack/vk-cocoon/vm"
+)
+
+func TestBringUpVMRestoreFromHibernate(t *testing.T) {
+	cases := []struct {
+		name     string
+		os       string
+		wantNICs bool // CH+Windows hibernate snapshots are NIC-less → clone hot-adds one
+	}{
+		{"windows+ch hot-adds a NIC", string(cocoonv1.OSWindows), true},
+		{"linux inherits the snapshot NIC", string(cocoonv1.OSLinux), false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			const vmName = "vk-ns-demo-0"
+			rt := &fakeRuntime{snapshots: map[string]*vm.Snapshot{vmName: {Name: vmName}}}
+			p := newTestProvider(t)
+			p.Runtime = rt
+			p.Probes = probes.NewManager(t.Context())
+
+			pod := newPodWithSpec(meta.VMSpec{
+				VMName:  vmName,
+				Backend: string(cocoonv1.BackendCloudHypervisor),
+				OS:      tc.os,
+			})
+			pod.Annotations[meta.AnnotationRestoreFromHibernate] = "true"
+			spec := meta.ParseVMSpec(pod)
+
+			v, _, err := p.bringUpVM(t.Context(), pod, spec)
+			if err != nil {
+				t.Fatalf("bringUpVM: %v", err)
+			}
+			if v == nil {
+				t.Fatal("bringUpVM returned nil vm")
+			}
+			if rt.cloned == nil {
+				t.Fatal("restore must clone from the hibernate snapshot, Clone was never called")
+			}
+			if rt.cloned.From != vmName {
+				t.Errorf("restore must clone from the local hibernate snapshot; From=%q want %q", rt.cloned.From, vmName)
+			}
+			gotNICs := rt.cloned.NICs != nil
+			if gotNICs != tc.wantNICs {
+				t.Errorf("NICs override present = %v, want %v", gotNICs, tc.wantNICs)
+			}
+			if tc.wantNICs && (rt.cloned.NICs == nil || *rt.cloned.NICs != 1) {
+				t.Errorf("CH+Windows restore must clone with --nics 1; got %v", rt.cloned.NICs)
+			}
+		})
+	}
+}

provider/cocoon/update.go

@@ -165,7 +165,28 @@ func (p *Provider) wake(ctx context.Context, pod *corev1.Pod) error {
 		p.failOp(ctx, pod, "WakePullFailed", "update", err)
 		return err
 	}
-	dropNIC := shouldDropNICBeforeHibernate(spec)
+	cloneStart := time.Now()
+	v, err := p.cloneFromHibernate(ctx, spec, sourceName)
+	if err != nil {
+		metrics.WakeTotal.WithLabelValues("failed").Inc()
+		p.failOp(ctx, pod, "WakeCloneFailed", "update", err)
+		return err
+	}
+	metrics.VMBootDuration.WithLabelValues("clone", spec.Backend).Observe(time.Since(cloneStart).Seconds())
+	p.applyRuntime(ctx, pod, v)
+	p.trackPod(pod, v)
+	p.startProbeIfEnabled(pod)
+	p.dispatchHibernateRestore(pod, spec, v, "update")
+	p.emitNormalf(pod, "Woken", "cloned from %s", sourceName)
+	return nil
+}
+
+// cloneFromHibernate clones the VM from an already-resolved hibernate snapshot
+// source. CH+Windows hibernate snapshots are captured NIC-less, so the clone
+// hot-adds a fresh NIC that Windows enumerates as new hardware. The local import
+// copy (cross-node pull) is dropped whether the clone succeeds or fails.
+func (p *Provider) cloneFromHibernate(ctx context.Context, spec meta.VMSpec, sourceName string) (*vm.VM, error) {
+	defer p.cleanupWakeImport(spec.VMName, sourceName)
 	opts := vm.CloneOptions{
 		From:       sourceName,
 		To:         spec.VMName,
@@ -174,34 +195,31 @@ func (p *Provider) wake(ctx context.Context, pod *corev1.Pod) error {
 		NoDirectIO: spec.NoDirectIO,
 		OnDemand:   useOnDemandClone(spec.OS),
 	}
-	if dropNIC {
+	if shouldDropNICBeforeHibernate(spec) {
 		opts.NICs = ptr.To(1)
 	}
-	cloneStart := time.Now()
 	v, err := p.Runtime.Clone(ctx, opts)
 	if err != nil {
-		metrics.WakeTotal.WithLabelValues("failed").Inc()
-		err = fmt.Errorf("clone vm %s from %s: %w", spec.VMName, sourceName, err)
-		p.failOp(ctx, pod, "WakeCloneFailed", "update", err)
-		return err
+		return nil, fmt.Errorf("clone vm %s from %s: %w", spec.VMName, sourceName, err)
 	}
-	metrics.VMBootDuration.WithLabelValues("clone", spec.Backend).Observe(time.Since(cloneStart).Seconds())
-	p.applyRuntime(ctx, pod, v)
-	p.cleanupWakeImport(spec.VMName, sourceName)
-	p.trackPod(pod, v)
-	p.startProbeIfEnabled(pod)
-	if dropNIC {
+	return v, nil
+}
+
+// dispatchHibernateRestore schedules the post-restore step. A CH+Windows restore
+// hot-added a fresh NIC, so Ready waits on that NIC's DHCP lease (no PnP rebind);
+// other backends re-derive networking via runPostCloneSetup.
+func (p *Provider) dispatchHibernateRestore(pod *corev1.Pod, spec meta.VMSpec, v *vm.VM, op string) {
+	if shouldDropNICBeforeHibernate(spec) {
 		p.goBackground(func() {
 			p.finalizeDropNICWake(p.lifecycleCtx, pod, v)
 		})
-	} else {
-		metrics.WakeTotal.WithLabelValues("ok").Inc()
-		p.goBackground(func() {
-			p.runPostCloneSetup(p.lifecycleCtx, pod, spec, v, "", "update")
-		})
+		return
 	}
-	p.emitNormalf(pod, "Woken", "cloned from %s", sourceName)
-	return nil
+	// A restore counts as a wake regardless of trigger (cross-node create or update).
+	metrics.WakeTotal.WithLabelValues("ok").Inc()
+	p.goBackground(func() {
+		p.runPostCloneSetup(p.lifecycleCtx, pod, spec, v, "", op)
+	})
 }
 
 // finalizeDropNICWake holds Ready until the fresh NIC's lease lands.