fix(interactive): tolerate already-closed shutdown

lib/interactive.py

@@ -1,4 +1,5 @@
 import cmd
+import errno
 import socket
 import sys
 import threading
@@ -752,17 +753,54 @@ def close_nodes(self):
         print("\nClosing all nodes...")
         pub.unsubAll()
         for n in self.nodes:
-            n.iface.localNode.exitSimulator()
+            self.exit_simulator(n)
             try:
                 close_interface(n.iface)
+            except OSError as ex:
+                if ex.errno != errno.EBADF:
+                    raise
+                print(f"Warning: node {n.nodeid} interface was already closed during shutdown ({ex}).")
             except InterfaceCloseTimeout as ex:
                 print(f"Warning: node {n.nodeid} interface close timed out during shutdown ({ex}).")
+            finally:
+                n.iface = None
         if self.docker:
-            self.container.stop()
+            self.stop_container()
         if self.forwardToClient:
-            self._wantExit = True
-            self.forwardSocket.close()
-            self.clientSocket.close()
+            self.wantExit = True
+            if self.forwardSocket is not None:
+                self.forwardSocket.close()
+            if self.clientSocket is not None:
+                self.clientSocket.close()
+
+    @staticmethod
+    def exit_simulator(node):
+        iface = node.iface
+        if iface is None or getattr(iface, "localNode", None) is None:
+            return
+        try:
+            iface.localNode.exitSimulator()
+        except OSError as ex:
+            if ex.errno != errno.EBADF:
+                raise
+            print(f"Warning: node {node.nodeid} simulator was already closed during shutdown ({ex}).")
+
+    def stop_container(self):
+        try:
+            self.container.stop()
+        except Exception as ex:
+            if self.is_docker_not_found(ex):
+                print(f"Warning: Docker container was already removed during shutdown ({ex}).")
+                return
+            raise
+
+    @staticmethod
+    def is_docker_not_found(ex):
+        response = getattr(ex, "response", None)
+        status_code = getattr(ex, "status_code", None) or getattr(response, "status_code", None)
+        return status_code == 404 or (
+            ex.__class__.__module__ == "docker.errors" and ex.__class__.__name__ == "NotFound"
+        )
 
 
 class CommandProcessor(cmd.Cmd):

tests/test_interactive.py

@@ -1,9 +1,43 @@
+import errno
+import importlib.util
+import os
+import sys
 import threading
+import types
 import unittest
 
 from lib.interface_close import InterfaceCloseTimeout, close_interface
 
 
+class Graph:
+    pass
+
+
+def load_interactive_sim():
+    """Load interactive.py with a GUI stub without poisoning lib.interactive."""
+    module_name = "_interactive_shutdown_test"
+    gui_stub = types.ModuleType("lib.gui")
+    gui_stub.Graph = Graph
+    gui_stub.gen_scenario = lambda conf: []
+    previous_gui = sys.modules.get("lib.gui")
+    sys.modules["lib.gui"] = gui_stub
+    module_path = os.path.join(os.path.dirname(__file__), "..", "lib", "interactive.py")
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    module = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(module)
+        return module.InteractiveSim
+    finally:
+        sys.modules.pop(module_name, None)
+        if previous_gui is None:
+            del sys.modules["lib.gui"]
+        else:
+            sys.modules["lib.gui"] = previous_gui
+
+
+InteractiveSim = load_interactive_sim()
+
+
 class TestInteractiveInterfaceClose(unittest.TestCase):
     def test_close_interface_calls_close(self):
         class Interface:
@@ -46,5 +80,97 @@ def close(self):
             iface.release.set()
 
 
+class TestInteractiveShutdown(unittest.TestCase):
+    def make_sim(self, nodes, container=None):
+        sim = object.__new__(InteractiveSim)
+        sim.nodes = nodes
+        sim.docker = container is not None
+        sim.container = container
+        sim.forwardToClient = False
+        sim.forwardSocket = None
+        sim.clientSocket = None
+        sim.wantExit = False
+        return sim
+
+    def test_close_nodes_ignores_already_closed_node_socket(self):
+        class LocalNode:
+            def exitSimulator(self):
+                raise OSError(errno.EBADF, "Bad file descriptor")
+
+        class Interface:
+            localNode = LocalNode()
+            close_attempted = False
+
+            def close(self):
+                self.close_attempted = True
+                raise OSError(errno.EBADF, "Bad file descriptor")
+
+        iface = Interface()
+
+        node = type("Node", (), {})()
+        node.nodeid = 0
+        node.iface = iface
+        sim = self.make_sim([node])
+
+        sim.close_nodes()
+
+        self.assertIsNone(node.iface)
+        self.assertTrue(iface.close_attempted)
+
+    def test_close_nodes_preserves_unexpected_exit_errors(self):
+        class LocalNode:
+            def exitSimulator(self):
+                raise OSError(errno.EIO, "device IO failed")
+
+        class Interface:
+            localNode = LocalNode()
+
+        class Node:
+            nodeid = 0
+            iface = Interface()
+
+        sim = self.make_sim([Node()])
+
+        with self.assertRaisesRegex(OSError, "device IO failed"):
+            sim.close_nodes()
+
+    def test_close_nodes_ignores_auto_removed_docker_container(self):
+        class Interface:
+            class localNode:
+                @staticmethod
+                def exitSimulator():
+                    pass
+
+            def close(self):
+                pass
+
+        class Node:
+            nodeid = 0
+            iface = Interface()
+
+        class DockerNotFound(Exception):
+            status_code = 404
+
+        class Container:
+            def stop(self):
+                raise DockerNotFound("No such container")
+
+        sim = self.make_sim([Node()], container=Container())
+
+        sim.close_nodes()
+
+        self.assertIsNone(sim.nodes[0].iface)
+
+    def test_stop_container_keeps_unexpected_docker_errors_visible(self):
+        class Container:
+            def stop(self):
+                raise RuntimeError("docker daemon unavailable")
+
+        sim = self.make_sim([], container=Container())
+
+        with self.assertRaisesRegex(RuntimeError, "docker daemon unavailable"):
+            sim.stop_container()
+
+
 if __name__ == "__main__":
     unittest.main()