Revert job-related changes

To avoid extra testing and changes, limit the
scope of the PR to instance termination reasons,
as per the PR name.

The introduction of
`JobTerminationReason.NO_BALANCE` can be done in a
separate PR. Its implementation may need to be
updated to also use
`JobTerminationReason.NO_BALANCE` on jobs with
statuses other than `running`.

Author: jvstme

src/dstack/_internal/core/models/runs.py

@@ -139,7 +139,6 @@ class JobTerminationReason(str, Enum):
     TERMINATED_BY_SERVER = "terminated_by_server"
     INACTIVITY_DURATION_EXCEEDED = "inactivity_duration_exceeded"
     TERMINATED_DUE_TO_UTILIZATION_POLICY = "terminated_due_to_utilization_policy"
-    NO_BALANCE = "no_balance"
     # Set by the runner
     CONTAINER_EXITED_WITH_ERROR = "container_exited_with_error"
     PORTS_BINDING_FAILED = "ports_binding_failed"
@@ -163,7 +162,6 @@ def to_status(self) -> JobStatus:
             self.TERMINATED_BY_SERVER: JobStatus.TERMINATED,
             self.INACTIVITY_DURATION_EXCEEDED: JobStatus.TERMINATED,
             self.TERMINATED_DUE_TO_UTILIZATION_POLICY: JobStatus.TERMINATED,
-            self.NO_BALANCE: JobStatus.TERMINATED,
             self.CONTAINER_EXITED_WITH_ERROR: JobStatus.FAILED,
             self.PORTS_BINDING_FAILED: JobStatus.FAILED,
             self.CREATING_CONTAINER_ERROR: JobStatus.FAILED,

src/dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -18,7 +18,6 @@
 from dstack._internal.core.models.files import FileArchiveMapping
 from dstack._internal.core.models.instances import (
     InstanceStatus,
-    InstanceTerminationReason,
     RemoteConnectionInfo,
     SSHConnectionParams,
 )
@@ -59,7 +58,6 @@
 from dstack._internal.server.services.jobs import (
     find_job,
     get_job_attached_volumes,
-    get_job_provisioning_data,
     get_job_runtime_data,
     is_master_job,
     job_model_to_job_submission,
@@ -390,22 +388,9 @@ async def _process_running_job(session: AsyncSession, job_model: JobModel):
                 if job_model.disconnected_at is None:
                     job_model.disconnected_at = common_utils.get_current_datetime()
                 if _should_terminate_job_due_to_disconnect(job_model):
-                    if (
-                        job_model.instance is not None
-                        and job_model.instance.termination_reason
-                        == InstanceTerminationReason.NO_BALANCE
-                    ):
-                        # if instance was terminated due to no balance, set job termination reason accodingly
-                        job_model.termination_reason = JobTerminationReason.NO_BALANCE
-                    else:
-                        job_provisioning_data = get_job_provisioning_data(job_model)
-                        # use JobTerminationReason.INSTANCE_UNREACHABLE for on-demand instances only
-                        job_model.termination_reason = (
-                            JobTerminationReason.INSTANCE_UNREACHABLE
-                            if job_provisioning_data
-                            and not job_provisioning_data.instance_type.resources.spot
-                            else JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY
-                        )
+                    # TODO: Replace with JobTerminationReason.INSTANCE_UNREACHABLE for on-demand.
+                    job_model.termination_reason = JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY
+                    job_model.termination_reason_message = "Instance is unreachable"
                     switch_job_status(session, job_model, JobStatus.TERMINATING)
                 else:
                     logger.warning(

src/dstack/_internal/server/services/jobs/__init__.py

@@ -814,8 +814,6 @@ def _get_job_status_message(job_model: JobModel) -> str:
             return "stopped"
         elif job_model.termination_reason == JobTerminationReason.ABORTED_BY_USER:
             return "aborted"
-        elif job_model.termination_reason == JobTerminationReason.NO_BALANCE:
-            return "no balance"
     return job_model.status.value
 
 

src/tests/_internal/core/models/test_runs.py

@@ -43,7 +43,6 @@ def test_get_error_returns_expected_messages():
         JobTerminationReason.ABORTED_BY_USER,
         JobTerminationReason.TERMINATED_BY_SERVER,
         JobTerminationReason.CONTAINER_EXITED_WITH_ERROR,
-        JobTerminationReason.NO_BALANCE,
     ]
 
     for reason in JobTerminationReason:

src/tests/_internal/server/background/tasks/test_process_running_jobs.py

@@ -528,7 +528,7 @@ async def test_pulling_shim_failed(self, test_db, session: AsyncSession):
             assert SSHTunnelMock.call_count == 3
         await session.refresh(job)
         assert job.status == JobStatus.TERMINATING
-        assert job.termination_reason == JobTerminationReason.INSTANCE_UNREACHABLE
+        assert job.termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY
         assert job.remove_at is None
 
     @pytest.mark.asyncio