Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions charmcraft.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ config:
Minutes between each reconciliation of the current runners state and their targeted state.
On reconciliation, the charm polls the state of runners and see if actions are needed. The
value should be kept low, unless Github API rate limiting is encountered.
planner-pressure-mode:
type: string
default: stream
description: >-
Planner pressure fetch mode. Set to `stream` to consume continuous planner updates,
or `request` to poll the planner with periodic one-shot requests.
allow-external-contributor:
type: boolean
default: True
Expand Down
5 changes: 5 additions & 0 deletions docs/changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

This changelog documents user-relevant changes to the GitHub runner charm.

## 2026-07-03

- Added charm configuration option `planner-pressure-mode` with `stream` (default) and `request` values to control how planner pressure is fetched.
- Updated pressure reconciler behavior to support both continuous planner streaming and periodic single-request polling based on the configured mode.

## 2026-06-26

- Fixed runners whose cloud VM entered an error state being kept until the creation timeout (~23 minutes) before cleanup. Such VMs are now cleaned up immediately, freeing the slot for a replacement runner.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import logging
from dataclasses import dataclass
from typing import Optional, TextIO
from typing import Literal, Optional, TextIO

import yaml
from pydantic import AnyHttpUrl, BaseModel, Field, IPvAnyAddress, root_validator
Expand Down Expand Up @@ -52,6 +52,7 @@ class ApplicationConfiguration(BaseModel):
planner_url: Base URL of the planner service.
planner_token: Bearer token to authenticate against the planner service.
reconcile_interval: Minutes to wait between reconciliation.
planner_pressure_mode: Pressure fetch mode (`stream` or `request`).
"""

allow_external_contributor: bool = False
Expand All @@ -64,6 +65,7 @@ class ApplicationConfiguration(BaseModel):
planner_url: Optional[AnyHttpUrl] = None
planner_token: Optional[str] = None
reconcile_interval: int = Field(ge=1)
planner_pressure_mode: Literal["stream", "request"] = "stream"

@staticmethod
def from_yaml_file(file: TextIO) -> "ApplicationConfiguration":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import time
from dataclasses import dataclass
from threading import Event, Lock
from typing import Optional
from typing import Literal, Optional

from github_runner_manager.configuration import ApplicationConfiguration
from github_runner_manager.configuration.base import RunnerCombination, UserInfo
Expand Down Expand Up @@ -59,13 +59,15 @@ class PressureReconcilerConfig:
Attributes:
flavor_name: Name of the planner flavor to reconcile.
reconcile_interval: Minutes between timer-based delete reconciliations.
planner_pressure_mode: Planner pressure fetch mode (`stream` or `request`).
min_pressure: Minimum desired runner count (floor) for the flavor.
Also used as fallback when the planner is unavailable.
max_pressure: Maximum desired runner count (ceiling). 0 means no cap.
"""

flavor_name: str
reconcile_interval: int = 5
planner_pressure_mode: Literal["stream", "request"] = "stream"
min_pressure: int = 0
max_pressure: int = 0

Expand Down Expand Up @@ -95,16 +97,15 @@ class PressureReconciler: # pylint: disable=too-few-public-methods,too-many-ins

The reconcile loop uses the last pressure seen by the create loop rather than
fetching a fresh value, so it may act on a stale reading if pressure changed
between stream events. This is an accepted trade-off: the window is bounded
by the stream update frequency.
between planner updates. This is an accepted trade-off.

Attributes:
_manager: Runner manager used to list, create, and clean up runners.
_planner: Client used to stream pressure updates.
_planner: Client used to fetch pressure updates.
_config: Reconciler configuration.
_lock: Shared lock to serialize operations with other reconcile loops.
_stop: Event used to signal streaming loops to stop gracefully.
_last_pressure: Last pressure value seen in the create stream.
_stop: Event used to signal reconciliation loops to stop gracefully.
_last_pressure: Last pressure value seen in the create loop.
_runner_count: In-memory runner count used by the create loop.
_create_paused: True when creation returned zero IDs, cleared by reconcile loop.
"""
Expand All @@ -121,7 +122,7 @@ def __init__(
Args:
manager: Runner manager interface for creating, cleaning up,
and listing runners.
planner_client: Client used to stream pressure updates.
planner_client: Client used to fetch pressure updates.
None when no planner relation is configured.
config: Reconciler configuration.
lock: Shared lock to serialize operations with other reconcile loops.
Expand Down Expand Up @@ -152,29 +153,51 @@ def start_create_loop(self) -> None:
return
while not self._stop.is_set():
try:
for update in self._planner.stream_pressure(self._config.flavor_name):
if self._config.planner_pressure_mode == "request":
update = self._planner.get_pressure(self._config.flavor_name)
if self._stop.is_set():
return
self._handle_create_runners(update.pressure)
self._stop.wait(5)
else:
Comment on lines 159 to +162
for update in self._planner.stream_pressure(self._config.flavor_name):
if self._stop.is_set():
return
self._handle_create_runners(update.pressure)
except PlannerConnectionError as exc:
fallback = max(self._last_pressure or 0, self._config.min_pressure)
logger.warning(
"Pressure stream interrupted for flavor %s (%s), falling back to %s runners.",
self._config.flavor_name,
exc,
fallback,
)
if self._config.planner_pressure_mode == "request":
logger.warning(
"Pressure request failed for flavor %s (%s), falling back to %s runners.",
self._config.flavor_name,
exc,
fallback,
)
else:
logger.warning(
"Pressure stream interrupted for flavor %s (%s), falling back to %s runners.",
self._config.flavor_name,
exc,
fallback,
)
if self._stop.is_set():
return
self._handle_create_runners(fallback)
self._stop.wait(5)
except PlannerApiError:
fallback = max(self._last_pressure or 0, self._config.min_pressure)
logger.exception(
"Error in pressure stream loop for flavor %s, falling back to %s runners.",
self._config.flavor_name,
fallback,
)
if self._config.planner_pressure_mode == "request":
logger.exception(
"Error in pressure request loop for flavor %s, falling back to %s runners.",
self._config.flavor_name,
fallback,
)
else:
logger.exception(
"Error in pressure stream loop for flavor %s, falling back to %s runners.",
self._config.flavor_name,
fallback,
)
if self._stop.is_set():
return
self._handle_create_runners(fallback)
Expand Down Expand Up @@ -439,6 +462,7 @@ def build_pressure_reconciler(
config=PressureReconcilerConfig(
flavor_name=config.name,
reconcile_interval=config.reconcile_interval,
planner_pressure_mode=config.planner_pressure_mode,
min_pressure=first.base_virtual_machines,
max_pressure=first.max_total_virtual_machines,
),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,42 @@ def __init__(self, config: PlannerConfiguration) -> None:
self._session = self._create_session()
self._config = config

def get_pressure(self, name: str) -> PressureInfo:
"""Get pressure for the given flavor with a single request.

Args:
name: Flavor name.

Returns:
Parsed pressure info.

Raises:
PlannerConnectionError: On transient connection failures or timeouts.
PlannerApiError: On HTTP errors or invalid payloads.
"""
base = str(self._config.base_url).rstrip("/") + "/"
url = urljoin(base, f"api/v1/flavors/{name}/pressure")
Comment on lines +66 to +80
try:
response = self._session.get(
url,
headers={"Authorization": f"Bearer {self._config.token}"},
timeout=self._config.timeout,
)
response.raise_for_status()

data = response.json()
if not isinstance(data, dict) or name not in data:
raise PlannerApiError(f"Unexpected pressure response payload: {data}")

Comment on lines +89 to +92
try:
return PressureInfo(pressure=int(data[name]))
except (TypeError, ValueError) as exc:
raise PlannerApiError(f"Invalid pressure value in payload: {data}") from exc
except (requests.ConnectionError, requests.Timeout) as exc:
raise PlannerConnectionError(str(exc)) from exc
except requests.RequestException as exc:
raise PlannerApiError(str(exc)) from exc

def stream_pressure(self, name: str) -> Iterable[PressureInfo]:
"""Stream pressure updates for the given flavor.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,23 +82,36 @@ class _FakePlanner:

def __init__(
self,
stream_updates: list[int] | None = None,
stream_exception: Exception | None = None,
pressure_updates: list[int] | None = None,
pressure_exception: Exception | None = None,
):
"""Initialize with configurable stream behavior."""
self._stream_updates = stream_updates or []
self._stream_exception = stream_exception
"""Initialize with configurable pressure request behavior."""
self._pressure_updates = pressure_updates or []
self._pressure_exception = pressure_exception
self._index = 0

def stream_pressure(self, name: str): # noqa: ARG002
"""Yield pressure updates or raise the configured exception.
"""Yield pressure updates for stream mode tests."""
if self._pressure_exception is not None:
raise self._pressure_exception
for pressure in self._pressure_updates:
yield SimpleNamespace(pressure=pressure)

Yields:
Namespace objects with a pressure attribute.
"""
if self._stream_exception is not None:
raise self._stream_exception
for p in self._stream_updates:
yield SimpleNamespace(pressure=p)
def get_pressure(self, name: str): # noqa: ARG002
"""Return pressure update or raise the configured exception."""
if self._pressure_exception is not None:
raise self._pressure_exception

if self._pressure_updates:
if self._index < len(self._pressure_updates):
pressure = self._pressure_updates[self._index]
self._index += 1
else:
pressure = self._pressure_updates[-1]
else:
pressure = 0

return SimpleNamespace(pressure=pressure)


@pytest.mark.parametrize(
Expand All @@ -117,7 +130,7 @@ def test_min_pressure_used_as_fallback_when_stream_errors(
assert: min_pressure is used as fallback to create runners.
"""
mgr = _FakeManager()
planner = _FakePlanner(stream_exception=planner_error)
planner = _FakePlanner(pressure_exception=planner_error)
cfg = PressureReconcilerConfig(flavor_name="small", min_pressure=2)
reconciler = PressureReconciler(mgr, planner, cfg, lock=Lock())

Expand Down Expand Up @@ -148,7 +161,7 @@ def test_fallback_preserves_last_pressure_when_higher(
assert: The higher last_pressure is used as fallback instead of min_pressure.
"""
mgr = _FakeManager()
planner = _FakePlanner(stream_exception=planner_error)
planner = _FakePlanner(pressure_exception=planner_error)
cfg = PressureReconcilerConfig(flavor_name="small", min_pressure=2)
reconciler = PressureReconciler(mgr, planner, cfg, lock=Lock())
reconciler._last_pressure = 10
Expand Down Expand Up @@ -422,15 +435,15 @@ def test_create_loop_syncs_runner_count_on_start(monkeypatch: pytest.MonkeyPatch
so no unnecessary runners are created.
"""
mgr = _FakeManager(runners_count=3)
planner = _FakePlanner(stream_updates=[3])
planner = _FakePlanner(pressure_updates=[3])
cfg = PressureReconcilerConfig(flavor_name="small")
reconciler = PressureReconciler(mgr, planner, cfg, lock=Lock())

# stop after stream exhausts to avoid infinite loop
original_stream = planner.stream_pressure

def _stream_once(name):
"""Yield from original stream, then stop the reconciler."""
"""Yield one stream batch, then stop the reconciler."""
yield from original_stream(name)
reconciler.stop()

Expand All @@ -441,6 +454,31 @@ def _stream_once(name):
assert mgr.created_args == []


def test_create_loop_request_mode_uses_get_pressure(monkeypatch: pytest.MonkeyPatch):
"""
arrange: A reconciler in request mode with no existing runners and pressure=2.
act: Run start_create_loop for one request cycle.
assert: It creates runners from get_pressure and does not require stream updates.
"""
mgr = _FakeManager(runners_count=0)
planner = _FakePlanner(pressure_updates=[2])
cfg = PressureReconcilerConfig(flavor_name="small", planner_pressure_mode="request")
reconciler = PressureReconciler(mgr, planner, cfg, lock=Lock())

wait_calls = {"count": 0}

def _wait(_interval: int) -> bool:
"""Stop after the first request-mode polling wait."""
wait_calls["count"] += 1
reconciler.stop()
return True

monkeypatch.setattr(reconciler._stop, "wait", _wait)
reconciler.start_create_loop()

assert mgr.created_args == [2]


def test_timer_reconcile_scales_down_with_soft_delete():
"""
arrange: A reconciler with 5 runners and a lower desired pressure of 2.
Expand Down
14 changes: 14 additions & 0 deletions src/charm_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,10 @@
PLANNER_DEFAULT_PLATFORM: Final[str] = "github"
PLANNER_DEFAULT_PRIORITY: Final[int] = 50

PLANNER_PRESSURE_MODE_CONFIG_NAME = "planner-pressure-mode"

LogLevel = Literal["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
PlannerPressureMode = Literal["stream", "request"]


@dataclasses.dataclass(frozen=True)
Expand Down Expand Up @@ -319,6 +322,7 @@ class CharmConfig(BaseModel):
path: GitHub repository path in the format '<owner>/<repo>', or the GitHub organization
name.
reconcile_interval: Time between each reconciliation of runners in minutes.
planner_pressure_mode: Planner pressure fetch mode (`stream` or `request`).
token: GitHub personal access token for GitHub API.
app_client_id: GitHub App Client ID for GitHub API.
installation_id: GitHub App installation ID for GitHub API.
Expand All @@ -338,6 +342,7 @@ class CharmConfig(BaseModel):
openstack_clouds_yaml: OpenStackCloudsYAML
path: GitHubPath | None
reconcile_interval: int
planner_pressure_mode: PlannerPressureMode = "stream"
token: str | None
app_client_id: str | None
installation_id: int | None
Expand Down Expand Up @@ -612,6 +617,14 @@ def from_charm(cls, charm: CharmBase) -> "CharmConfig":
f"The {RECONCILE_INTERVAL_CONFIG_NAME} config must be greater than or equal to 1"
)

planner_pressure_mode = cast(
str, charm.config.get(PLANNER_PRESSURE_MODE_CONFIG_NAME, "stream")
)
if planner_pressure_mode not in {"stream", "request"}:
raise CharmConfigInvalidError(
f"The {PLANNER_PRESSURE_MODE_CONFIG_NAME} config must be one of: stream, request"
)

dockerhub_mirror = cast(str, charm.config.get(DOCKERHUB_MIRROR_CONFIG_NAME, "")) or None
openstack_clouds_yaml = cls._parse_openstack_clouds_config(charm)

Expand Down Expand Up @@ -641,6 +654,7 @@ def from_charm(cls, charm: CharmBase) -> "CharmConfig":
openstack_clouds_yaml=openstack_clouds_yaml,
path=github_config.path,
reconcile_interval=reconcile_interval,
planner_pressure_mode=cast(PlannerPressureMode, planner_pressure_mode),
token=github_config.token,
app_client_id=github_config.app_client_id,
installation_id=github_config.installation_id,
Expand Down
1 change: 1 addition & 0 deletions src/factories.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ def create_application_configuration(
planner_url=state.planner_config.endpoint if state.planner_config else None,
planner_token=state.planner_config.token if state.planner_config else None,
reconcile_interval=state.charm_config.reconcile_interval,
planner_pressure_mode=state.charm_config.planner_pressure_mode,
)


Expand Down
Loading
Loading