Add command to error out migrations which are 'orphaned'

nova/cmd/manage.py

@@ -766,6 +766,59 @@ def soft_delete_excessive_instance_faults(
         # Return 1 if we soft-deleted something
         return int(bool(cell_to_rows_deleted))
 
+    @args('--verbose', action='store_true', dest='verbose', default=False,
+          help='Print how many rows were changed per cell.')
+    @args('--all-cells', action='store_true', dest='all_cells',
+          default=False, help='Run command across all cells.')
+    def error_out_orphaned_migrations(self, all_cells=False, verbose=False):
+        """Error-out orphaned migrations.
+
+        Returns 0 if nothing was changed, 1 if at least one row was
+        changed, 3 if no connection could be established to the API DB.
+        """
+
+        ctxt = context.get_admin_context()
+        try:
+            cell_mappings = objects.CellMappingList.get_all(ctxt)
+        except db_exc.CantStartEngineError:
+            print(_('Failed to connect to API DB so aborting this '
+                    'change attempt. Please check your config file to '
+                    'make sure that [api_database]/connection is set and run '
+                    'this command again.'))
+            return 3
+
+        cell_to_rows_changed = {}
+        if not all_cells:
+            cell_mappings = [None]
+        for cell_mapping in cell_mappings:
+            with context.target_cell(ctxt, cell_mapping) as cctxt:
+                # If all_cells=False, cell_mapping is None
+                cell_name = cell_mapping.name if cell_mapping else ''
+                try:
+                    while True:
+                        rows_deleted = \
+                            db.error_out_orphaned_migrations(cctxt)
+                        if rows_deleted:
+                            cell_to_rows_changed.setdefault(cell_name, 0)
+                            cell_to_rows_changed[cell_name] += rows_deleted
+                        if verbose:
+                            print('.', end='')
+                except KeyboardInterrupt:
+                    break
+
+        if verbose:
+            if cell_to_rows_changed:
+                print(format_dict(
+                    cell_to_rows_changed,
+                    dict_property=_('Cell'),
+                    dict_value=_('Number of Changed Rows'),
+                ))
+            else:
+                print(_('No row was changed.'))
+
+        # Return 1 if we changed something
+        return int(bool(cell_to_rows_changed))
+
 
 class ApiDbCommands(object):
     """Class for managing the api database."""

nova/db/main/api.py

@@ -4708,6 +4708,31 @@ def soft_delete_excessive_instance_faults(context, max_rows, max_faults):
     return deleted
 
 
+@pick_context_manager_writer
+def error_out_orphaned_migrations(context):
+    """Error-out orphaned migrations.
+
+    We occasionally end up with instances in a running/stopped state without
+    any task, but the migrations are still "running".
+    As a workaround, we simply error them out.
+
+    :returns: The number of soft-deleted rows.
+    """
+
+    when = timeutils.utcnow() - datetime.timedelta(seconds=10)
+    mig = models.Migration
+    migrations = mig.__table__
+    subquery = sa.select(mig.id).join(mig.instance).where(
+        sa.and_(mig.status == 'running',
+                models.Instance.task_state == sa.null(),
+                models.Instance.updated_at < when)
+    )
+    query = sa.update(migrations).where(
+        migrations.c.id.in_(subquery)).values(status='error')
+    rowcount = context.session.execute(query).rowcount
+    return rowcount
+
+
 ####################