Reconsider termination reason naming

- Drop `starting_timeout`, use
  `provisioning_timeout` instead. These two
  termination reasons had similar semantics. The
  difference was that `starting_timeout` was used
  for cloud instances and `provisioning_timeout`
  for SSH.
- Rename `termination_deadline` -> `unreachable`.
  The old name was based on implementation details
  rather than semantics, so it was not very
  informative to the user.
- For `no_offers`, add a termination reason
  message to highlight the difference between
  failing to find offers and failing to provision
  offers.

Author: jvstme

src/dstack/_internal/core/models/instances.py

@@ -262,8 +262,7 @@ class InstanceTerminationReason(str, Enum):
     PROVISIONING_TIMEOUT = "provisioning_timeout"
     ERROR = "error"
     JOB_FINISHED = "job_finished"
-    TERMINATION_TIMEOUT = "termination_timeout"
-    STARTING_TIMEOUT = "starting_timeout"
+    UNREACHABLE = "unreachable"
     NO_OFFERS = "no_offers"
     MASTER_FAILED = "master_failed"
     MAX_INSTANCES_LIMIT = "max_instances_limit"
@@ -281,6 +280,7 @@ def from_legacy_str(cls, v: str) -> "InstanceTerminationReason":
         if v == "Idle timeout":
             return cls.IDLE_TIMEOUT
         if v in (
+            "Instance has not become running in time",
             "Provisioning timeout expired",
             "Proivisioning timeout expired",  # typo is intentional
             "The proivisioning timeout expired",  # typo is intentional
@@ -312,9 +312,7 @@ def from_legacy_str(cls, v: str) -> "InstanceTerminationReason":
         if v == "Instance job finished":
             return cls.JOB_FINISHED
         if v == "Termination deadline":
-            return cls.TERMINATION_TIMEOUT
-        if v == "Instance has not become running in time":
-            return cls.STARTING_TIMEOUT
+            return cls.UNREACHABLE
         if v == "Fleet has too many instances":
             return cls.MAX_INSTANCES_LIMIT
         if v == "Low account balance":

src/dstack/_internal/server/background/tasks/process_instances.py

@@ -679,7 +679,11 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
             )
         return
 
-    _mark_terminated(instance, InstanceTerminationReason.NO_OFFERS)
+    _mark_terminated(
+        instance,
+        InstanceTerminationReason.NO_OFFERS,
+        "All offers failed" if offers else "No offers found",
+    )
     if instance.fleet and is_fleet_master_instance(instance) and is_cloud_cluster(instance.fleet):
         # Do not attempt to deploy other instances, as they won't determine the correct cluster
         # backend, region, and placement group without a successfully deployed master instance
@@ -690,10 +694,13 @@ async def _create_instance(session: AsyncSession, instance: InstanceModel) -> No
 
 
 def _mark_terminated(
-    instance: InstanceModel, termination_reason: InstanceTerminationReason
+    instance: InstanceModel,
+    termination_reason: InstanceTerminationReason,
+    termination_reason_message: Optional[str] = None,
 ) -> None:
     instance.status = InstanceStatus.TERMINATED
     instance.termination_reason = termination_reason
+    instance.termination_reason_message = termination_reason_message
     logger.info(
         "Terminated instance %s: %s",
         instance.name,
@@ -842,7 +849,7 @@ async def _check_instance(session: AsyncSession, instance: InstanceModel) -> Non
         deadline = instance.termination_deadline
         if get_current_datetime() > deadline:
             instance.status = InstanceStatus.TERMINATING
-            instance.termination_reason = InstanceTerminationReason.TERMINATION_TIMEOUT
+            instance.termination_reason = InstanceTerminationReason.UNREACHABLE
             logger.warning(
                 "Instance %s shim waiting timeout. Marked as TERMINATING",
                 instance.name,
@@ -871,7 +878,8 @@ async def _wait_for_instance_provisioning_data(
             "Instance %s failed because instance has not become running in time", instance.name
         )
         instance.status = InstanceStatus.TERMINATING
-        instance.termination_reason = InstanceTerminationReason.STARTING_TIMEOUT
+        instance.termination_reason = InstanceTerminationReason.PROVISIONING_TIMEOUT
+        instance.termination_reason_message = "Backend did not complete provisioning in time"
         return
 
     backend = await backends_services.get_project_backend_by_type(

src/tests/_internal/server/background/tasks/test_process_instances.py

@@ -263,7 +263,7 @@ async def test_check_shim_terminate_instance_by_deadline(self, test_db, session:
         assert instance is not None
         assert instance.status == InstanceStatus.TERMINATING
         assert instance.termination_deadline == termination_deadline_time
-        assert instance.termination_reason == InstanceTerminationReason.TERMINATION_TIMEOUT
+        assert instance.termination_reason == InstanceTerminationReason.UNREACHABLE
 
     @pytest.mark.asyncio
     @pytest.mark.parametrize(