diff --git a/images/chromium-headful/Dockerfile b/images/chromium-headful/Dockerfile index 6f1f7409..a40d012b 100644 --- a/images/chromium-headful/Dockerfile +++ b/images/chromium-headful/Dockerfile @@ -33,6 +33,12 @@ RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \ GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ go build -ldflags="-s -w" -o /out/wrapper ./cmd/wrapper +# Build supervisord eventlistener shim +RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \ + --mount=type=cache,target=/go/pkg/mod,id=$CACHEIDPREFIX-go-pkg-mod \ + GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ + go build -ldflags="-s -w" -o /out/kernel-images-supervisord-shim ./cmd/supervisord-shim + # webrtc client FROM node:22-bullseye-slim AS client WORKDIR /src @@ -378,6 +384,7 @@ RUN chmod +x /usr/local/bin/init-envoy.sh # copy the kernel-images API binary built in the builder stage COPY --from=server-builder /out/kernel-images-api /usr/local/bin/kernel-images-api COPY --from=server-builder /out/chromium-launcher /usr/local/bin/chromium-launcher +COPY --from=server-builder /out/kernel-images-supervisord-shim /usr/local/bin/kernel-images-supervisord-shim COPY --from=server-builder /out/wrapper /wrapper # Copy and compile the Playwright daemon diff --git a/images/chromium-headful/supervisor/services/supervisord-shim.conf b/images/chromium-headful/supervisor/services/supervisord-shim.conf new file mode 100644 index 00000000..5bc94034 --- /dev/null +++ b/images/chromium-headful/supervisor/services/supervisord-shim.conf @@ -0,0 +1,17 @@ +[eventlistener:supervisord-shim] +command=/usr/local/bin/kernel-images-supervisord-shim +events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL +; buffer_size defaults to 10 which overflows when several supervised +; services flap in quick succession. Bump it so a burst of crashes +; doesn't cause supervisord to drop events before the shim drains them. +buffer_size=100 +autostart=true +autorestart=true +; Effectively infinite restart attempts. Supervisord does not emit +; events about its own eventlisteners, so if the shim ever enters FATAL +; state we lose all service_crashed telemetry with no signal. The shim +; is tiny and side-effect-free; a transient failure should never reach +; the default startretries=3 cap. +startretries=999999 +stderr_logfile=/var/log/supervisord/supervisord-shim +; stdout is the eventlistener protocol channel; do not redirect. diff --git a/images/chromium-headless/image/Dockerfile b/images/chromium-headless/image/Dockerfile index 5348c3f2..2a26e0f0 100644 --- a/images/chromium-headless/image/Dockerfile +++ b/images/chromium-headless/image/Dockerfile @@ -34,6 +34,12 @@ RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \ GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ go build -ldflags="-s -w" -o /out/wrapper ./cmd/wrapper +# Build supervisord eventlistener shim +RUN --mount=type=cache,target=/root/.cache/go-build,id=$CACHEIDPREFIX-go-build \ + --mount=type=cache,target=/go/pkg/mod,id=$CACHEIDPREFIX-go-pkg-mod \ + GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ + go build -ldflags="-s -w" -o /out/kernel-images-supervisord-shim ./cmd/supervisord-shim + FROM docker.io/ubuntu:22.04 AS ffmpeg-downloader # Allow cross-compilation when building with BuildKit platforms @@ -256,6 +262,7 @@ RUN chmod +x /usr/local/bin/bake-certs.sh && /usr/local/bin/bake-certs.sh && rm # Copy the kernel-images API binary built in the builder stage COPY --from=server-builder /out/kernel-images-api /usr/local/bin/kernel-images-api COPY --from=server-builder /out/chromium-launcher /usr/local/bin/chromium-launcher +COPY --from=server-builder /out/kernel-images-supervisord-shim /usr/local/bin/kernel-images-supervisord-shim # Copy and compile the Playwright daemon COPY server/runtime/playwright-daemon.ts /tmp/playwright-daemon.ts diff --git a/images/chromium-headless/image/supervisor/services/supervisord-shim.conf b/images/chromium-headless/image/supervisor/services/supervisord-shim.conf new file mode 100644 index 00000000..5bc94034 --- /dev/null +++ b/images/chromium-headless/image/supervisor/services/supervisord-shim.conf @@ -0,0 +1,17 @@ +[eventlistener:supervisord-shim] +command=/usr/local/bin/kernel-images-supervisord-shim +events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL +; buffer_size defaults to 10 which overflows when several supervised +; services flap in quick succession. Bump it so a burst of crashes +; doesn't cause supervisord to drop events before the shim drains them. +buffer_size=100 +autostart=true +autorestart=true +; Effectively infinite restart attempts. Supervisord does not emit +; events about its own eventlisteners, so if the shim ever enters FATAL +; state we lose all service_crashed telemetry with no signal. The shim +; is tiny and side-effect-free; a transient failure should never reach +; the default startretries=3 cap. +startretries=999999 +stderr_logfile=/var/log/supervisord/supervisord-shim +; stdout is the eventlistener protocol channel; do not redirect. diff --git a/server/cmd/api/main.go b/server/cmd/api/main.go index c226e5a7..2f73dd24 100644 --- a/server/cmd/api/main.go +++ b/server/cmd/api/main.go @@ -30,6 +30,7 @@ import ( oapi "github.com/kernel/kernel-images/server/lib/oapi" "github.com/kernel/kernel-images/server/lib/recorder" "github.com/kernel/kernel-images/server/lib/scaletozero" + "github.com/kernel/kernel-images/server/lib/sysmon" "github.com/kernel/kernel-images/server/lib/telemetry" ) @@ -103,6 +104,14 @@ func main() { } telemetrySession := telemetry.NewTelemetrySession(eventStream) + // VM-internal failure telemetry. OOM kills come from /dev/kmsg here; + // service_crashed events arrive via POST /telemetry/events from the + // supervisord-shim child process. Failure to open /dev/kmsg is not + // fatal — the rest of the API should stay usable without CAP_SYSLOG. + if err := sysmon.New(eventStream, slogger).Start(ctx); err != nil { + slogger.Error("sysmon: kmsg OOM monitor disabled", "err", err) + } + // Optional S2 storage sink. var s2Writer *events.S2StorageWriter if config.S2Basin != "" && config.S2AccessToken != "" && config.S2Stream != "" { diff --git a/server/cmd/supervisord-shim/main.go b/server/cmd/supervisord-shim/main.go new file mode 100644 index 00000000..79959520 --- /dev/null +++ b/server/cmd/supervisord-shim/main.go @@ -0,0 +1,256 @@ +// Command supervisord-shim is a tiny supervisord eventlistener that +// translates PROCESS_STATE_EXITED (expected=0) and PROCESS_STATE_FATAL +// events into BrowserServiceCrashedEvent payloads and POSTs them to the +// local kernel-images-api telemetry endpoint. +// +// All schema-mapping and event publishing logic lives here; lib/sysmon +// does not handle supervisord events. Keeping the shim as the sole owner +// of the supervisord protocol means lib/sysmon stays single-purpose +// (kmsg only). +// +// Wire protocol per supervisord docs (http://supervisord.org/events.html): +// +// stdout: "READY\n" +// stdin: header line ("ver:3.0 ... eventname:PROCESS_STATE_EXITED len:54\n") +// stdin: payload of `len` bytes (no trailing newline) +// stdout: "RESULT 2\nOK" (always; ACK regardless of downstream success) +// +// The result frame intentionally has NO trailing newline: supervisord +// reads exactly the declared number of bytes after the header newline, +// and a trailing newline would leak into the buffer and corrupt the +// subsequent READY token, deadlocking the listener after one event. +// +// We always ACK with OK so supervisord doesn't quarantine us when the +// downstream HTTP target is briefly unavailable. The events are +// best-effort; if the API is down, we drop and log. +// +// All logging goes to stderr — stdout is the supervisord protocol channel. +package main + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "log" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + oapi "github.com/kernel/kernel-images/server/lib/oapi" +) + +const ( + defaultAPIBaseURL = "http://127.0.0.1:10001" + httpTimeout = 2 * time.Second +) + +func main() { + log.SetOutput(os.Stderr) + log.SetFlags(log.LstdFlags | log.Lmicroseconds) + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + go func() { + <-ctx.Done() + _ = os.Stdin.Close() + }() + + baseURL := os.Getenv("KERNEL_IMAGES_API_BASE_URL") + if baseURL == "" { + baseURL = defaultAPIBaseURL + } + + client, err := oapi.NewClientWithResponses(baseURL, oapi.WithHTTPClient(&http.Client{Timeout: httpTimeout})) + if err != nil { + log.Fatalf("init oapi client: %v", err) + } + + in := bufio.NewReader(os.Stdin) + out := bufio.NewWriter(os.Stdout) + + for { + if _, err := out.WriteString("READY\n"); err != nil { + log.Fatalf("write READY: %v", err) + } + if err := out.Flush(); err != nil { + log.Fatalf("flush READY: %v", err) + } + + header, payload, err := readEvent(in) + if err != nil { + if err == io.EOF { + return + } + log.Fatalf("read event: %v", err) + } + + // Try to publish but always ACK supervisord. + ev, ok := mapEvent(header, payload) + switch { + case ok: + if perr := publish(ctx, client, ev); perr != nil { + log.Printf("publish telemetry event: %v", perr) + } + case isCrashEvent(header["eventname"]): + // We subscribed to this event type but couldn't map it. + // Most likely cause: supervisord emitted a from_state we + // don't have a public phase for. Logging means a future + // supervisord behavior change shows up in stderr instead + // of silent telemetry loss. + log.Printf("skipped crash event: eventname=%q from_state=%q processname=%q expected=%q", + header["eventname"], payload["from_state"], payload["processname"], payload["expected"]) + } + + if err := writeResultOK(out); err != nil { + log.Fatalf("write RESULT: %v", err) + } + } +} + +// writeResultOK ACKs a single event. See the file header for why the +// frame body has no trailing newline. +func writeResultOK(out *bufio.Writer) error { + if _, err := out.WriteString("RESULT 2\nOK"); err != nil { + return err + } + return out.Flush() +} + +// readEvent reads one supervisord event: a header line followed by a +// payload of declared length. +func readEvent(in *bufio.Reader) (map[string]string, map[string]string, error) { + headerLine, err := in.ReadString('\n') + if err != nil { + return nil, nil, err + } + header := parseFields(strings.TrimRight(headerLine, "\n")) + + lenStr, ok := header["len"] + if !ok { + return nil, nil, fmt.Errorf("missing len in header: %q", headerLine) + } + n, err := strconv.Atoi(lenStr) + if err != nil { + return nil, nil, fmt.Errorf("invalid len %q: %w", lenStr, err) + } + + buf := make([]byte, n) + if _, err := io.ReadFull(in, buf); err != nil { + return nil, nil, fmt.Errorf("read payload: %w", err) + } + payload := parseFields(string(buf)) + return header, payload, nil +} + +// parseFields parses supervisord's "key:value key:value" tokenization. +// Values are split on the first colon; supervisord does not escape colons +// in values, but in practice the values we care about (process names, +// states, ints) never contain them. +func parseFields(s string) map[string]string { + out := make(map[string]string) + for _, tok := range strings.Fields(s) { + i := strings.IndexByte(tok, ':') + if i < 0 { + continue + } + out[tok[:i]] = tok[i+1:] + } + return out +} + +// phaseForExited maps the supervisord state a process exited from to the +// public lifecycle phase. EXITED in supervisord always originates from +// RUNNING (post-startsecs); STARTING-during-startsecs-violation routes +// through BACKOFF→FATAL, not EXITED. We still defend against STARTING +// here in case a future supervisord version changes the state machine, +// and we treat anything else as "unknown" so the caller logs and skips +// rather than inventing a phase. +func phaseForExited(fromState string) (oapi.BrowserServiceCrashedEventDataPhase, bool) { + switch fromState { + case "RUNNING": + return oapi.BrowserServiceCrashedEventDataPhaseRunning, true + case "STARTING": + return oapi.BrowserServiceCrashedEventDataPhaseStartup, true + default: + return "", false + } +} + +// isCrashEvent reports whether the supervisord eventname is one we +// subscribed to. Used by the main loop to log when a target event was +// dropped instead of silently skipping it. +func isCrashEvent(eventName string) bool { + return eventName == "PROCESS_STATE_EXITED" || eventName == "PROCESS_STATE_FATAL" +} + +// mapEvent decides whether to publish and constructs the event payload. +// Returns ok=false for events we deliberately skip (intentional stops, +// non-crash event types, or unknown lifecycle transitions). +func mapEvent(header, payload map[string]string) (oapi.PublishEventRequest, bool) { + var phase oapi.BrowserServiceCrashedEventDataPhase + switch header["eventname"] { + case "PROCESS_STATE_EXITED": + // expected=0 means the exit was not in `exitcodes` — i.e. a + // crash. expected=1 means clean shutdown (operator-initiated + // stop, or a configured exit code). Skip the latter. + if payload["expected"] != "0" { + return oapi.PublishEventRequest{}, false + } + p, ok := phaseForExited(payload["from_state"]) + if !ok { + return oapi.PublishEventRequest{}, false + } + phase = p + case "PROCESS_STATE_FATAL": + // FATAL is reached exclusively by the BACKOFF→FATAL edge after + // supervisord exhausts startretries. The from_state is always + // BACKOFF here, and the semantic is "gave up trying to start". + phase = oapi.BrowserServiceCrashedEventDataPhaseGaveUp + default: + return oapi.PublishEventRequest{}, false + } + + name := payload["processname"] + if name == "" { + return oapi.PublishEventRequest{}, false + } + + data := oapi.BrowserServiceCrashedEventData{ + ServiceName: name, + Phase: phase, + } + if pidStr := payload["pid"]; pidStr != "" { + if pid, err := strconv.Atoi(pidStr); err == nil { + data.Pid = &pid + } + } + + category := oapi.PublishEventRequestCategory(oapi.TelemetryEventCategorySystem) + sourceEvent := "service.crashed" + return oapi.PublishEventRequest{ + Type: string(oapi.ServiceCrashed), + Category: &category, + Source: &oapi.BrowserEventSource{ + Kind: oapi.LocalProcess, + Event: &sourceEvent, + }, + Data: data, + }, true +} + +func publish(ctx context.Context, client *oapi.ClientWithResponses, body oapi.PublishEventRequest) error { + resp, err := client.PublishTelemetryEventWithResponse(ctx, body) + if err != nil { + return err + } + if resp.StatusCode() >= 300 { + return fmt.Errorf("status %d: %s", resp.StatusCode(), bytes.TrimSpace(resp.Body)) + } + return nil +} diff --git a/server/cmd/supervisord-shim/main_test.go b/server/cmd/supervisord-shim/main_test.go new file mode 100644 index 00000000..dc48df1f --- /dev/null +++ b/server/cmd/supervisord-shim/main_test.go @@ -0,0 +1,193 @@ +package main + +import ( + "bufio" + "bytes" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + oapi "github.com/kernel/kernel-images/server/lib/oapi" +) + +func TestWriteResultOKHasNoTrailingNewline(t *testing.T) { + // Regression: supervisord's eventlistener protocol reads exactly the + // declared byte count after the header newline. A trailing newline + // here misaligns the following READY frame and deadlocks the + // listener after the first event. + var buf bytes.Buffer + bw := bufio.NewWriter(&buf) + require.NoError(t, writeResultOK(bw)) + assert.Equal(t, "RESULT 2\nOK", buf.String()) +} + +func TestParseFields(t *testing.T) { + got := parseFields("processname:mutter groupname:mutter from_state:RUNNING expected:0 pid:1234") + assert.Equal(t, map[string]string{ + "processname": "mutter", + "groupname": "mutter", + "from_state": "RUNNING", + "expected": "0", + "pid": "1234", + }, got) +} + +func TestReadEvent(t *testing.T) { + payload := "processname:cat groupname:cat from_state:RUNNING expected:0 pid:2766" + header := "ver:3.0 server:supervisor serial:21 pool:listener poolserial:10 eventname:PROCESS_STATE_EXITED len:" + + strconv.Itoa(len(payload)) + "\n" + in := bufio.NewReader(strings.NewReader(header + payload)) + + hdr, pl, err := readEvent(in) + require.NoError(t, err) + assert.Equal(t, "PROCESS_STATE_EXITED", hdr["eventname"]) + assert.Equal(t, "2766", pl["pid"]) + assert.Equal(t, "cat", pl["processname"]) + assert.Equal(t, "0", pl["expected"]) +} + +// crashedData unwraps the BrowserServiceCrashedEventData payload from a +// mapped PublishEventRequest. The shim builds the request with Data set +// to a concrete struct (not interface{}); this helper keeps the test +// assertions short. +func crashedData(t *testing.T, body oapi.PublishEventRequest) oapi.BrowserServiceCrashedEventData { + t.Helper() + data, ok := body.Data.(oapi.BrowserServiceCrashedEventData) + require.True(t, ok, "Data is %T, want BrowserServiceCrashedEventData", body.Data) + return data +} + +func TestMapEventExitedUnexpectedFromRunning(t *testing.T) { + body, ok := mapEvent( + map[string]string{"eventname": "PROCESS_STATE_EXITED"}, + map[string]string{ + "processname": "mutter", + "from_state": "RUNNING", + "expected": "0", + "pid": "1234", + }, + ) + require.True(t, ok) + assert.Equal(t, string(oapi.ServiceCrashed), body.Type) + require.NotNil(t, body.Category) + assert.Equal(t, oapi.PublishEventRequestCategory("system"), *body.Category) + require.NotNil(t, body.Source) + assert.Equal(t, oapi.LocalProcess, body.Source.Kind) + require.NotNil(t, body.Source.Event) + assert.Equal(t, "service.crashed", *body.Source.Event) + + data := crashedData(t, body) + assert.Equal(t, "mutter", data.ServiceName) + assert.Equal(t, oapi.BrowserServiceCrashedEventDataPhaseRunning, data.Phase) + require.NotNil(t, data.Pid) + assert.Equal(t, 1234, *data.Pid) +} + +func TestMapEventExitedUnexpectedFromStarting(t *testing.T) { + // A crash during startup must surface as the "startup" phase, not + // "running" — operators triage these differently (config bug vs + // runtime bug). + body, ok := mapEvent( + map[string]string{"eventname": "PROCESS_STATE_EXITED"}, + map[string]string{ + "processname": "envoy", + "from_state": "STARTING", + "expected": "0", + "pid": "55", + }, + ) + require.True(t, ok) + assert.Equal(t, oapi.BrowserServiceCrashedEventDataPhaseStartup, crashedData(t, body).Phase) +} + +func TestMapEventExitedExpectedSkipped(t *testing.T) { + _, ok := mapEvent( + map[string]string{"eventname": "PROCESS_STATE_EXITED"}, + map[string]string{ + "processname": "mutter", + "from_state": "RUNNING", + "expected": "1", + "pid": "1234", + }, + ) + assert.False(t, ok, "expected=1 (clean exit) must not produce an event") +} + +func TestMapEventExitedFromBackoffSkipped(t *testing.T) { + // supervisord's state machine does not normally produce EXITED out + // of BACKOFF — the BACKOFF→FATAL edge fires instead once + // startretries is exhausted. If a future supervisord version routes + // it differently, we must not silently invent a phase: skip and let + // the caller log. + _, ok := mapEvent( + map[string]string{"eventname": "PROCESS_STATE_EXITED"}, + map[string]string{ + "processname": "chromium", + "from_state": "BACKOFF", + "expected": "0", + }, + ) + assert.False(t, ok) +} + +func TestMapEventFatalFromBackoff(t *testing.T) { + body, ok := mapEvent( + map[string]string{"eventname": "PROCESS_STATE_FATAL"}, + map[string]string{ + "processname": "chromium", + "from_state": "BACKOFF", + }, + ) + require.True(t, ok) + data := crashedData(t, body) + assert.Equal(t, oapi.BrowserServiceCrashedEventDataPhaseGaveUp, data.Phase) + assert.Nil(t, data.Pid, "FATAL transitions do not carry a live PID") +} + +func TestMapEventFatalIgnoresFromState(t *testing.T) { + // FATAL is reached exclusively via the BACKOFF→FATAL edge per + // supervisord docs, so the from_state lookup is intentionally not + // consulted for FATAL events. This test pins that behavior so a + // future refactor doesn't reintroduce a silent drop if supervisord + // ever omits from_state. + body, ok := mapEvent( + map[string]string{"eventname": "PROCESS_STATE_FATAL"}, + map[string]string{"processname": "chromium"}, + ) + require.True(t, ok) + assert.Equal(t, oapi.BrowserServiceCrashedEventDataPhaseGaveUp, crashedData(t, body).Phase) +} + +func TestMapEventUnrelatedSkipped(t *testing.T) { + _, ok := mapEvent( + map[string]string{"eventname": "PROCESS_STATE_STARTING"}, + map[string]string{"processname": "x", "from_state": "STOPPED"}, + ) + assert.False(t, ok) +} + +func TestIsCrashEvent(t *testing.T) { + assert.True(t, isCrashEvent("PROCESS_STATE_EXITED")) + assert.True(t, isCrashEvent("PROCESS_STATE_FATAL")) + assert.False(t, isCrashEvent("PROCESS_STATE_STARTING")) + assert.False(t, isCrashEvent("PROCESS_STATE_RUNNING")) + assert.False(t, isCrashEvent("")) +} + +func TestMapEventUnknownFromStateSkipped(t *testing.T) { + // If supervisord emits a crash transition out of a state we have no + // public mapping for (e.g. STOPPED, which shouldn't happen with the + // events we subscribe to), drop the event rather than invent a phase. + _, ok := mapEvent( + map[string]string{"eventname": "PROCESS_STATE_EXITED"}, + map[string]string{ + "processname": "x", + "from_state": "STOPPED", + "expected": "0", + }, + ) + assert.False(t, ok) +} diff --git a/server/go.mod b/server/go.mod index 1bdae079..d7ae2a36 100644 --- a/server/go.mod +++ b/server/go.mod @@ -10,6 +10,7 @@ require ( github.com/creack/pty v1.1.24 github.com/docker/docker v28.5.1+incompatible github.com/docker/go-connections v0.6.0 + github.com/euank/go-kmsg-parser/v2 v2.1.0 github.com/fsnotify/fsnotify v1.9.0 github.com/getkin/kin-openapi v0.133.0 github.com/ghodss/yaml v1.0.0 diff --git a/server/go.sum b/server/go.sum index b26c03c9..bb8925ce 100644 --- a/server/go.sum +++ b/server/go.sum @@ -52,6 +52,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/euank/go-kmsg-parser/v2 v2.1.0 h1:G3QuOjQgrC1lNUgArlLLnvq/8S9kksQW/fk26LfPDew= +github.com/euank/go-kmsg-parser/v2 v2.1.0/go.mod h1:829LX1BxwHvmThOJ2AIy+b42Ku7VdX7lgVQFwmo5zdY= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -216,6 +218,7 @@ github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKk github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= diff --git a/server/lib/oapi/oapi.go b/server/lib/oapi/oapi.go index 9b350bf0..a0a6e5e2 100644 --- a/server/lib/oapi/oapi.go +++ b/server/lib/oapi/oapi.go @@ -578,6 +578,81 @@ func (e BrowserPageTabOpenedEventType) Valid() bool { } } +// Defines values for BrowserServiceCrashedEventType. +const ( + ServiceCrashed BrowserServiceCrashedEventType = "service_crashed" +) + +// Valid indicates whether the value is a known member of the BrowserServiceCrashedEventType enum. +func (e BrowserServiceCrashedEventType) Valid() bool { + switch e { + case ServiceCrashed: + return true + default: + return false + } +} + +// Defines values for BrowserServiceCrashedEventDataPhase. +const ( + BrowserServiceCrashedEventDataPhaseGaveUp BrowserServiceCrashedEventDataPhase = "gave_up" + BrowserServiceCrashedEventDataPhaseRunning BrowserServiceCrashedEventDataPhase = "running" + BrowserServiceCrashedEventDataPhaseStartup BrowserServiceCrashedEventDataPhase = "startup" +) + +// Valid indicates whether the value is a known member of the BrowserServiceCrashedEventDataPhase enum. +func (e BrowserServiceCrashedEventDataPhase) Valid() bool { + switch e { + case BrowserServiceCrashedEventDataPhaseGaveUp: + return true + case BrowserServiceCrashedEventDataPhaseRunning: + return true + case BrowserServiceCrashedEventDataPhaseStartup: + return true + default: + return false + } +} + +// Defines values for BrowserSystemOomKillEventType. +const ( + SystemOomKill BrowserSystemOomKillEventType = "system_oom_kill" +) + +// Valid indicates whether the value is a known member of the BrowserSystemOomKillEventType enum. +func (e BrowserSystemOomKillEventType) Valid() bool { + switch e { + case SystemOomKill: + return true + default: + return false + } +} + +// Defines values for BrowserSystemOomKillEventDataConstraint. +const ( + Cpuset BrowserSystemOomKillEventDataConstraint = "cpuset" + Memcg BrowserSystemOomKillEventDataConstraint = "memcg" + MemoryPolicy BrowserSystemOomKillEventDataConstraint = "memory_policy" + None BrowserSystemOomKillEventDataConstraint = "none" +) + +// Valid indicates whether the value is a known member of the BrowserSystemOomKillEventDataConstraint enum. +func (e BrowserSystemOomKillEventDataConstraint) Valid() bool { + switch e { + case Cpuset: + return true + case Memcg: + return true + case MemoryPolicy: + return true + case None: + return true + default: + return false + } +} + // Defines values for BrowserTargetType. const ( BrowserTargetTypeBackgroundPage BrowserTargetType = "background_page" @@ -835,16 +910,16 @@ func (e ProcessKillRequestSignal) Valid() bool { // Defines values for ProcessStatusState. const ( - Exited ProcessStatusState = "exited" - Running ProcessStatusState = "running" + ProcessStatusStateExited ProcessStatusState = "exited" + ProcessStatusStateRunning ProcessStatusState = "running" ) // Valid indicates whether the value is a known member of the ProcessStatusState enum. func (e ProcessStatusState) Valid() bool { switch e { - case Exited: + case ProcessStatusStateExited: return true - case Running: + case ProcessStatusStateRunning: return true default: return false @@ -2198,6 +2273,104 @@ type BrowserPageTabOpenedEventData struct { Url string `json:"url"` } +// BrowserServiceCrashedEvent A managed service exited unexpectedly. Intentional stops (e.g. operator-initiated shutdown) do not produce this event — only unexpected exits and terminal restart-give-up transitions do. +type BrowserServiceCrashedEvent struct { + // Data Per-crash payload for `service_crashed` events. Exit code and signal are not exposed by the underlying process manager on this channel, so only the service identity, the lifecycle phase the crash occurred in, and (when available) the PID are reported. + Data *BrowserServiceCrashedEventData `json:"data,omitempty"` + + // Source Provenance metadata identifying which producer emitted the event. + Source BrowserEventSource `json:"source"` + + // Truncated True if the data field was truncated due to size limits. + Truncated *bool `json:"truncated,omitempty"` + + // Ts Event timestamp in Unix microseconds. + Ts int64 `json:"ts"` + Type BrowserServiceCrashedEventType `json:"type"` +} + +// BrowserServiceCrashedEventType defines model for BrowserServiceCrashedEvent.Type. +type BrowserServiceCrashedEventType string + +// BrowserServiceCrashedEventData Per-crash payload for `service_crashed` events. Exit code and signal are not exposed by the underlying process manager on this channel, so only the service identity, the lifecycle phase the crash occurred in, and (when available) the PID are reported. +type BrowserServiceCrashedEventData struct { + // Phase Lifecycle phase the crash occurred in. `startup` means the process died before it ever reached a healthy running state. `running` means a previously healthy process died unexpectedly. `gave_up` means the process manager exhausted its restart attempts and stopped trying; no further `service_crashed` events will fire for this service until something restarts it. + Phase BrowserServiceCrashedEventDataPhase `json:"phase"` + + // Pid PID of the crashed process. Absent when the process manager gave up after exhausting restart attempts and is no longer tracking a live PID. + Pid *int `json:"pid,omitempty"` + + // ServiceName Program name of the crashed service (e.g. `chromium`, `mutter`, `kernel-images-api`). + ServiceName string `json:"service_name"` +} + +// BrowserServiceCrashedEventDataPhase Lifecycle phase the crash occurred in. `startup` means the process died before it ever reached a healthy running state. `running` means a previously healthy process died unexpectedly. `gave_up` means the process manager exhausted its restart attempts and stopped trying; no further `service_crashed` events will fire for this service until something restarts it. +type BrowserServiceCrashedEventDataPhase string + +// BrowserSystemOomKillEvent The Linux kernel OOM-killer terminated a process inside the VM. Sourced from `/dev/kmsg`. Fires for any process killed by the kernel due to memory exhaustion, including Chrome renderer subprocesses that are not supervised. +type BrowserSystemOomKillEvent struct { + // Data Per-kill payload for `system_oom_kill` events. + Data *BrowserSystemOomKillEventData `json:"data,omitempty"` + + // Source Provenance metadata identifying which producer emitted the event. + Source BrowserEventSource `json:"source"` + + // Truncated True if the data field was truncated due to size limits. + Truncated *bool `json:"truncated,omitempty"` + + // Ts Event timestamp in Unix microseconds. + Ts int64 `json:"ts"` + Type BrowserSystemOomKillEventType `json:"type"` +} + +// BrowserSystemOomKillEventType defines model for BrowserSystemOomKillEvent.Type. +type BrowserSystemOomKillEventType string + +// BrowserSystemOomKillEventData Per-kill payload for `system_oom_kill` events. +type BrowserSystemOomKillEventData struct { + // Constraint Why the kernel decided to OOM-kill. `none` means global memory exhaustion; `memcg` means a cgroup memory limit was hit; `cpuset` / `memory_policy` are NUMA/policy-driven kills. Absent on kernels older than 5.0 which did not emit the structured `oom-kill:` line. + Constraint *BrowserSystemOomKillEventDataConstraint `json:"constraint,omitempty"` + + // MemFreeKb Free system memory in KiB at the time of the kill, derived from the `free:N` field in the kernel's Mem-Info dump. Assumes a 4 KiB page size. Does not include reclaimable caches, so a small value with a large `mem_total_kb` may still mean the system was not under hard pressure. Absent if the kernel did not emit a parseable Mem-Info section. + MemFreeKb *int `json:"mem_free_kb,omitempty"` + + // MemTotalKb Total system memory in KiB at the time of the kill, derived from the `N pages RAM` line in the kernel's Mem-Info dump. Assumes a 4 KiB page size. Absent if the kernel did not emit a parseable Mem-Info section. + MemTotalKb *int `json:"mem_total_kb,omitempty"` + + // Pid PID of the killed process. + Pid int `json:"pid"` + + // ProcessName Comm of the killed process as reported by the kernel (max 15 chars, truncated by the kernel). + ProcessName string `json:"process_name"` + + // RssKb Resident set size of the killed process in KiB (sum of anon-rss, file-rss, and shmem-rss). This is the physical memory the process was using at the time of the kill. + RssKb int `json:"rss_kb"` + + // TopTasks Top processes by resident-set-size at the moment of the kill, sorted descending. Sourced from the kernel's `Tasks state` table. Empty if the kernel did not emit the table. Capped at 5 entries to bound payload size. + TopTasks *[]BrowserSystemOomKillTask `json:"top_tasks,omitempty"` + + // TriggerPid PID of the triggering process. Absent if the kernel did not emit the standard `CPU: N PID: N Comm:` header line. + TriggerPid *int `json:"trigger_pid,omitempty"` + + // TriggerProcessName Comm of the process whose allocation request caused the kernel to invoke the OOM-killer. Often the same as `process_name` (the kernel killed the requester) but can differ when the kernel chose a different victim. Max 15 chars, truncated by the kernel. + TriggerProcessName *string `json:"trigger_process_name,omitempty"` +} + +// BrowserSystemOomKillEventDataConstraint Why the kernel decided to OOM-kill. `none` means global memory exhaustion; `memcg` means a cgroup memory limit was hit; `cpuset` / `memory_policy` are NUMA/policy-driven kills. Absent on kernels older than 5.0 which did not emit the structured `oom-kill:` line. +type BrowserSystemOomKillEventDataConstraint string + +// BrowserSystemOomKillTask A single process entry from the kernel's `Tasks state` dump. +type BrowserSystemOomKillTask struct { + // Name Comm of the process (max 15 chars, truncated by the kernel). + Name string `json:"name"` + + // Pid PID of the process. + Pid int `json:"pid"` + + // RssKb Resident set size in KiB at the moment of the kill. + RssKb int `json:"rss_kb"` +} + // BrowserTargetType CDP target type of the page that produced the event. type BrowserTargetType string @@ -3940,6 +4113,62 @@ func (t *KnownBrowserTelemetryEvent) MergeBrowserCaptchaSolveResultEvent(v Brows return err } +// AsBrowserSystemOomKillEvent returns the union data inside the KnownBrowserTelemetryEvent as a BrowserSystemOomKillEvent +func (t KnownBrowserTelemetryEvent) AsBrowserSystemOomKillEvent() (BrowserSystemOomKillEvent, error) { + var body BrowserSystemOomKillEvent + err := json.Unmarshal(t.union, &body) + return body, err +} + +// FromBrowserSystemOomKillEvent overwrites any union data inside the KnownBrowserTelemetryEvent as the provided BrowserSystemOomKillEvent +func (t *KnownBrowserTelemetryEvent) FromBrowserSystemOomKillEvent(v BrowserSystemOomKillEvent) error { + v.Type = "system_oom_kill" + b, err := json.Marshal(v) + t.union = b + return err +} + +// MergeBrowserSystemOomKillEvent performs a merge with any union data inside the KnownBrowserTelemetryEvent, using the provided BrowserSystemOomKillEvent +func (t *KnownBrowserTelemetryEvent) MergeBrowserSystemOomKillEvent(v BrowserSystemOomKillEvent) error { + v.Type = "system_oom_kill" + b, err := json.Marshal(v) + if err != nil { + return err + } + + merged, err := runtime.JSONMerge(t.union, b) + t.union = merged + return err +} + +// AsBrowserServiceCrashedEvent returns the union data inside the KnownBrowserTelemetryEvent as a BrowserServiceCrashedEvent +func (t KnownBrowserTelemetryEvent) AsBrowserServiceCrashedEvent() (BrowserServiceCrashedEvent, error) { + var body BrowserServiceCrashedEvent + err := json.Unmarshal(t.union, &body) + return body, err +} + +// FromBrowserServiceCrashedEvent overwrites any union data inside the KnownBrowserTelemetryEvent as the provided BrowserServiceCrashedEvent +func (t *KnownBrowserTelemetryEvent) FromBrowserServiceCrashedEvent(v BrowserServiceCrashedEvent) error { + v.Type = "service_crashed" + b, err := json.Marshal(v) + t.union = b + return err +} + +// MergeBrowserServiceCrashedEvent performs a merge with any union data inside the KnownBrowserTelemetryEvent, using the provided BrowserServiceCrashedEvent +func (t *KnownBrowserTelemetryEvent) MergeBrowserServiceCrashedEvent(v BrowserServiceCrashedEvent) error { + v.Type = "service_crashed" + b, err := json.Marshal(v) + if err != nil { + return err + } + + merged, err := runtime.JSONMerge(t.union, b) + t.union = merged + return err +} + func (t KnownBrowserTelemetryEvent) Discriminator() (string, error) { var discriminator struct { Discriminator string `json:"type"` @@ -4010,6 +4239,10 @@ func (t KnownBrowserTelemetryEvent) ValueByDiscriminator() (interface{}, error) return t.AsBrowserPageNavigationSettledEvent() case "page_tab_opened": return t.AsBrowserPageTabOpenedEvent() + case "service_crashed": + return t.AsBrowserServiceCrashedEvent() + case "system_oom_kill": + return t.AsBrowserSystemOomKillEvent() default: return nil, errors.New("unknown discriminator value: " + discriminator) } @@ -17915,328 +18148,352 @@ func (sh *strictHandler) StreamTelemetryEvents(w http.ResponseWriter, r *http.Re // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+z9+3Ibt5YojL8Kir+pijRDUrKT7Jlt1/lDkeQdTXzRT5KTmWzlI8HuRRJbTaADoCXR", - "KU+dhzhPeJ7kK6wF9IVE8yYptvfnqqnZjti4rTsW1uWPTqJmuZIgrem8+KOjweRKGsD/+IGnF/B7Acae", - "aq20+1OipAVp3T95nmci4VYoefAPo6T7m0mmMOPuX/+iYdx50fn/HVTzH9Cv5oBm+/jxY7eTgkm0yN0k", - "nRduQeZX7Hzsdo6VHGci+bNWD8u5pc+kBS159ictHZZjl6BvQTP/YbfzVtlXqpDpn7SPt8oyXK/jfvOf", - "EynYZHqsZnlhQR8l7vOAKLeTNBXuTzw71yoHbYUjoDHPDCyucMRGbiqmxizx0zGO8xlmFYN7SAoLzLjJ", - "pRU8y+b9TreT1+b9o+MHuH82Z3+nU9CQskwY65ZYnrnPTvEfQklmrMoNU5LZKbCx0MYycJBxCwoLM7MO", - "jk2AOHzNhDyjkc+6HTvPofOiw7XmcwSoht8LoSHtvPh7eYbfyu/U6B9A1PeDVncG9FEujnmWnd56hC9A", - "UrIfr67OWcKzjE25TDNI2WiOh7kBLSHriRmfgOnxXDCDhLUMypTbteQS2c6JG+ZIRBU6gQ0nwJGXNOJj", - "t2N1IRNuHTgWz3alC2BijGdxO2RjAVnK7rhh5SiWFuAQa8QHYJmYCWvc8TwwR0plwBEnNkIouBVmxQyM", - "5bOcCcneS3HPZiLRykCiZIqzjZWecdt50RHS/uW7anohLUwAWZT+8kcHZDFDxOZi4HBSw6yxWsjJEglY", - "EyYsAbkhNZx4rG3BeOege0gqOZ9niqdsrDQbhs0OGbh5TYRACo0iZjCLgPEXnmW9JFPJDQvfObZzaCOK", - "1A6yM5FlogZUf0JZzEYEQrceLSIixPAuB3l0fsbKr87SsMjMyRJImVZOaOxBf9Jnw1yrBIxxfD7ssqHl", - "N3CZaABppsoO92s7CHghtICx0fUd5PzvTKROKo0FaDbWatbCbOHrmUjTDO64huiixnJbRKCKbB00MaOv", - "WKLS+iwlAS7QVO0gC3At1+s2cLqC4hy5XVqe3Cxv8fjknF0U0jFQHz+50jwBpiHXYByI5ARh85/8ll/i", - "OJJTxn3LuMUf3WiU0pKor89eOTY3rDDA3AqSz9xEiZLuZ5TkmtspaGanXDIj+Q0MEm5QDiAt4LzHU61m", - "wE7g9kqpzLBzraxKVMbuhAZGLN2/lkuk7nb4SvMZbKBZ8DRj/LjLHPXpmTKWtEhDfywsobJiJt8S5S8t", - "8ito1RtxAymjDxnxCLsTdipIT2VCRumg2xkXEnXKWz6D5blrmAgfOvhClynNYJbbOSPKRMHApZLzmSpM", - "+bGJkrDbzQancZ9FzkJfx09Dv52lcdqj/66xY3R3hc6Wh7+/eO2O7M4exIifbSyyGKMucFgDzLV90nIN", - "kHSb+I6xWtNGWBDay5KQhD3L+AgyRBRuH5nKIgeSDORmLhOW8MJAXN7lXAcrMsvejTsv/r6RBq8kwsff", - "lhQMTtnYDFISbgX/avpLwKyx3EpBlNtkyi9VdgsXYIrMttlELKFPmXHfMm6tI22mgaOe4MwxqnAgVIVN", - "1AweZBG17OurcdRqHHn0DBA9A40we2xDaRVWtreZAgk1zKbYMdpNqPB1AMaCOPMUewsyVZqN+Uxk875T", - "WmmRgDZMOjBnDpG5VrciBd0zOSRiLBJmublBUWaYkFYxOxWGGbAvGLgrZa6FAXbLteDSGifuNAQOSVSW", - "8dxAGAhCs1vQximGUZHcgGV7t8/ZAbv9dr/LuEwZl3MnuidMKssSdYsKkQSOA+6JctrkjfUH6rI840Ky", - "d8cX+0wYZxso7UiTGzZUTosPSQkH2pj6nXUc8gPMbp83//NbRwmFlsaKzJHDBMC6W2i3g1NGaKm7vQmL", - "ph1JEGO5to6TYoJjyZDF6+PAmWrLCyE91lCH36JZ566gYy6yQpc27OnFxbuLwfHR+dXxj0eD928v373+", - "+eiH16fD/T47GjkLyw0yReIs3a2My6vFc7Chn2b4gs6smQYHYpSXheGjDNwPeGfus6Hfaexr6Q+1ZwDY", - "sAKG2/XQyRNV2GpcKlKkJBpftwucVgD9jWF3XFg2KtIJ2D4b8hGXqZKQDl/4T1jCZQKZu/l6XZjzCTDJ", - "b8UExSC/43NnhvdwzSa9+WM7QUZHcmCkTXa6nXKxKEk5voteFjyWuTFi4mBSs1DYu5z/XkDXmbfjgtS3", - "KXLHFcwJVtPTMAYNMoE4Su9gZISFwVSZiO77UZFlWkLhbgoaPDyJ5Z2KQECkK+fPuZ1GrkHcTjefn/3/", - "C9ClSQn3SVak0WWXDIKarNzhypLmx0pKSGy71wTuvbMtyYRjJGK5pDBWzUCzy5Ofuuw84/M7LSZT22Xn", - "RZ6DBdD77ibi5oaUkcjEW8ovMLpUKC9zre7n5FAShv38ZlkVfDUJlk2CNB94uD66JZDmJ8Ik2xJEWo6B", - "tLrmr0E1O+eCLjf4tZjNIBXcQjZnuYYEUscHw9phh8HzaNxNxFgNfPYwa3TptF8N0ZVUV+H56QlvR+uz", - "2uKCAdrY/uN77/zE7i+bOPBmYAyfwCBRRYzH6P7r5nZM5D92FmHG505Jo/aLrAsCnT2p0PS3uKdAAzex", - "2/Iv0/ninCCdEmJDYvRBkinjDBn8inhfSGEFEi79URlnIRU58ecgmXI5QQMEnUyimDENaCNCSnYGGLSg", - "nb2MmhLlhFUaWKruJDOqvlqiiix1NrnHMZ9wIQ15xyTcsbBufQtoVg1flL+xVDhrTge4sryY5WSI0VmV", - "tHBvB6Wp5A8cnJT+d2Tbypzas/NcOCNr7p8OmJkW1h1hv2lF1UHZ6XYWIVX/E+4JnSILO1rPfnU6XiS3", - "kgJWMaSSRmWAD1+tvoMRfesg4j72xqzSzMmyYjK1dXcm3CeQE1GR7/J0JmylMO6UUyNWyMQi0ZPMMKQg", - "UjFGQ8+S2DRTnoPplw5Vv/7R+dkxJ2T4v/T9nYFnmdl3pOVuiIZlcAtZlzmYdhnXE0PXNfS5DNATU81d", - "bvtqqh097pVnK3+pT01zZkJC17sku/4og0JnkXW8B9fZ9f590l0fvLVEIxnXwDheYmJe2G003iJWvyq8", - "doVHsPJM+Nj6LoqIbb2LOPKYhEPnY3fRd+4oO8K2WVYyLNeTYuZmZokCnZCZTgc0fXZOTxNMyWzuLi/S", - "06Nn2Tbua3jzly+CC/5bYpKIl6fhz2/4v2sXqUqoIE0hi2688QXWjitLlBVxn3qAohvEbnnmrqo8u+Nz", - "w67Js3HdeRAUo68Hy3t5XXss+HSAqqRcyxPC0tMBs1N82NJw19zjI2ys4dcJ0nZjL3XptO92kLeW5Q7q", - "lWBAuG+qPQvJRspOg/DOuZ2a9fd4XGdZYvy2JDNeq8nGCjlTE9K2lUbM1KQbfu8LOVbVf91xLbsMbNLf", - "7z+Clgkb/apj1uqYTE2eSMM0kPB56Zet1MQKMdxqBbo5uiznxuDtRKtiMmWFHIvMoiceRQm9ffe993WI", - "jndVeI9Vwwbwd0bmLhzA05eMZxlDJzpb1AbG2XLANXPyt88ugbwhJoekfIQcF1nGHCGQTffnyK1XGLS1", - "iJ5l7KyXV4SQ7gZyq0FFSzvyH3kxFe5WyGlVeFaQazMlhXVXDGkVgv/45LwXNIO/0rOz4EGmG7LlegK2", - "S7EHZIB7dzfeRXKVTB1L302Fj4agnagkKbS7EEYsbpwq6s12WMZf64EvNUc9bSau2xVPQbfOmqqEcEXf", - "1ebvuhs14PsG8GRaO110HclvBwZ+X17ljZLKKukvsUIm7paIr1cVuCjMMAnmRpc+c/uCtNyAVXkPyaM+", - "MgqEDUSm9w+0wiX4D+qBRp7DaJ2aOyMKD/oqOn+gTT9RbYk9Y/Ge5j0x1TlNOChnlo/2V60YlMEGnH2F", - "I67cgFVRGhoyuOWSnt+mwhApv6TXB/fBGOM4Spw4XsDfiHW6pYuj/BbsndI3NW/ZaqFQQ1YdsM0jVyS4", - "Qn3V9f+WXkCtbkFyR6QzsBxNAo+5uaNmYnR/YdcMvBei5Pxl0wfi5lZ4cK69UKLkwEAZ/yzZppuGCN66", - "9Cp9KAjqOOHcCJm22SfhQH30dQZ/Wyyoy6ux0k/vhWufDSkwb8BzMXzBfsL/YEfnZ8GhtefkjL4FcqnS", - "H3sTkKDRxgo7Z0O4tyAdIQxfMCH/Qe8Cfj/lb302zFTCs4EPPxy+YGZuLMyY/wPThZQOYzxTcmJECo3t", - "Np1qad7pdqr9u5/CQh0nW2sLRd8nA6m0E1vESFlHD0GbETE4aUV8cOD55IBUxdlJA9+BFxZ4C5G/gmN+", - "tDb/EZxuMO2HsLpYYhiMnpzSSDbjucPuHdcpRh70hKcUt3sn2lRhywALUjLsZ3f1NeilqjlBycpjo8Ky", - "GZ+zETAu5+w/L9+9RROpYfUsHQbj+yni+zgTyc3aG0+B1x73abAkeG4LZ+XdCl4RIUq7Kopu5ytOdH9f", - "LzqtFx1RwWuAWHrs6047Qh750mMgg8SqSPjm8eUlC7/irT94cfHATkBmaCm12ASTWFzzm9fM8kkj9nJh", - "NoelIs9BY1gvSZof3l9dvXvbZUdddnL2c4sRErXGfxZGoP/ZiS2fOtOycJdZjY+20envY3PDHcZu3PcS", - "pXQqJLfNU7mzOCjm4h4yE3czzVdMPN994gXiu++4lboVtglDK+85NRL8CeZrJdYNzEeK6/TPlldhb1+l", - "1UbS6gbmTyirGsh4ZEnldr4EtZ9gTq7qyv77yRMiAZQkyKnbYpf9wJMbk/PE3ZvjYmQHcRgEF3p/p/hA", - "nxSGvLyUHjJHMsk1GNMiXjYXlzj5anF59vb8/VWXXZ3+19XRxWm70Fw0yOABEuIy0SrLLsHaDNK1ssLg", - "18zQ515ihJsLH9vqk1wZUcvVw0dlISfdP0++LJ/sq6TZSNIQBgceyU8odFow9Mjix8mXQcQMoNXZfa8k", - "VZ/dRJHD1TOR+2oCxlHtJoYBrjdvXW/+2Ot5l8YOApDWWmcQqhjwXmEkslkGIcoAN3k4QZAVm5xExeDW", - "WGr+KEstJgYRhZSo84f2G1qG8ErZ+lrcgjME10SzskzcArsVcFeFFC2EqLqr8LjIgvD9xrBfYHRxdVy6", - "Qd7Cjdrvsx/9d0pm85cYwBEk8lhpnCUDYxilOT5IusbO9lWotgpVh+KBQ/FThcm24mP7eMXgvm4EKy4d", - "oD1ecZV7/HVJ6stO8j67bHiwy5A602VGMc6s5tIggwQn8CgTOUu4RDLHgC3vSSxjeDEwd1htabiVx3gD", - "gK8PTl7m73hw8qZMXgUpx7Aymi8d98FM/jUkeXs+f7rA5FVYeXRu/4wClHeVKy99iYEQnawpLZ+C+dvk", - "2pYPSxumtryhx+KTGv+3SI0rn61Qg5FV4cHCsUKmjO2zK7TXrJ4Hwef92qlWeQ4pK6QVWXijHpQS1V3R", - "tBa3YPrsSgO36AgXspdrNXF33FDYBSNDLbA9L3EHIs0wgGECg4zPVWHD5WCfccMKqSETKMRpZTsF+SAR", - "1AaxrzKoVQYFbNe1zGPLoJVoWSeEmsTQFvp/gX8vX86r0+ADT4KcMCgD98vHxfKlLvzSr7/JLYxaD5b1", - "YekeFGdS2FdcZGs5OggoyhtwNvoIfMpCJj7Qfh/KLgub+cosa5nFIWAwRpA9Ea/EcLIdpxgLeTtdzcBO", - "FebQlsTkA2Qs5OTZpPN5FyMFcPQN2KPCqiNreTLdwMWIm1h/2ougajbiiaiWazCIhh5ggIsw09LBCPdT", - "XhhLD/JZdWEgjwrm/Js+e6vYuNBUcmZRXd6JLPOqsEwE9Az6GHwYg8JXZlzLjCUin5YjW7HzJAqsQZ0+", - "W71f/XXgidmpMiJmR6aBitkdaGD4alDkZdCDz34fF1k2R4WndCja1OSqug6MrPiIavACHmzZLpwqwvd8", - "0Ro4JW4Ozq60KOEw4TlGgZC5fNy0arGihQGL/oWFILTgYrCaJzduNm80sLEGMw23dmFYroS0jyosvgqK", - "rQXF08uIh8iHwHCbXpSxqNrClZhZfgPIKrVU09Lv3eSHTYC6xOCxTa6HT1WVr9X9lYMWKhUJM+W3wQMQ", - "HhNvfbjEY7DRwo6+ctFaLqrw8kRMFEPJdjyUy8jj+g/cwF++64FMVAopO3/7tw1JrITVaG5hrcXr1l5x", - "xrekKM7SDNY+mgelItIQVrvwZM7Z94eHM8N+LwRYzznk65WKCdkbZ2IytcxXl8TI6Ie94yy8mH5lk2U2", - "qbu+HptBPPG8VjwVcrLyrrRMRRmNCtc6n8N+Nm6UBnAg5pkGns4dUDwBYWSLs8I43vvcpVAqlmuhNBuG", - "A/sphjhH/SFR2P0uGxY6G3bZMGSeuH+XCSNDymoZavA5mA4Aw1rW+Es2jFAg5jrlXFPRaJarvMiQNDBN", - "g1uWcAMPTDhvBflXTbGWBTzFPdG1bDVmHjkWhApXrENUnYvCiMUMMAylmEQqodbwRXXU4gGub0NGC2b0", - "1X7zjhoJ9sWL04uLwfG7t29Pj6/O3r0dXJy+en95erJ9IWTH85FCyPhCEu5MSouJkBz9KguyoPVxxK1a", - "Y/X4wv6k/Qv/6dU8h9r9GFdYyo6sB/z7xMifpLqTFDNomJBYl4yd+Gy0LnsFNpl22X/9eNFlVOmjyy7t", - "PAMzBXfZO5vxCXTZG0gF77JXyo25gnt75a56XVZj6W5VLarL3nApxrjDcw1jWuOdnYImWTdTeoPKs43a", - "zjWq6FYEuTKmxIMw9HTYVFUE9GE2eEtO0fYytL6Lr9JzrfT0SHgisbmEjEcWmCHbc205hTItFDV2sxiT", - "B0FUgExrmULb7LueZbRc0tiDJWQT9d1Kfk+O91pl1Vn4po+1NIRMsU8HZuuhIVKY5pl2FlzGi6ica+OE", - "Sa7B6VmSKpjMHQWXMAMNVFZrFbugj8vLe+P3a4qMWmuwMEOcT+hJoaW4vX9v4IaFUqZucizPTnrrb6dX", - "XXb+7vKqpXy1MnYQZE4cZyOVzlE/uFkOzt9flXeerjscv+Ui46MMWvQRHS1Or+9Ix2WYVzqCsfJFScIo", - "RAMeDE3lGrARjLqAR1K9XVZI8XsBjZrq1QvEVzX7cDXrybjbFGGVwFkSCJtpYOrtsIUK9s0gNCQgbqsL", - "2yu36Zovr/wQyd8hxXvCaVgXn8SQKkOGJD1gPY5Gr53qq0rfQKUTvJ5Mpy+i45GVuiOxKGY8+Bu0WMlE", - "rIGEEgXuLXtz9uaUaoz8qXrd76yu2DdRWN5KUUEBrDJJZmLWJmjLQ4cJS1CR9nOQOZjaWdZli02+vt7a", - "Pnt18kiNfcI0LTf/6Fy19Px3P3VZ2c5tf1etV9bfDoy4Ur2d8wmcqNkxJdq+VjzdwCN58u5NY0Co8OXI", - "x03YT8sZcS5UeQ+r6NW6z69aq1VrYdhmqmYDn0aN/rzH9+OtRs1j+/HSfFACKyLAKK5gFgoJMXphpXxT", - "IVl4XeXWV2FZIuWxA0IXCx1bcYt4DWQfYg0pMGDP2WWIKqzgtN9n7w2woTVUWeWu+b4bCXFeLKPfONla", - "pn2N4bibpm9S8G5L+uYzDxZvlKJ3E4PDq5coC/oWsBRKmGkqxngvqy7Kt8IUHPuFjUQm7LzPTnkybQyg", - "+Au6lz7r+VXdofXXV60/QRY0Q7ifQg54qnS4Xl8ispgVnskaNLJ3/Ppy35NomS1zDhpPLRNgV2IG2J7s", - "6PzswUplccdf9clmNOQA9mdQ0JP4Nn3MyzL0ThZyVhqECdLq+VKgzp4vuHuIYr8hHlkOGksu7kczXOqg", - "HKRgucjM9ik9gS1qgGPcWi1GhQWzhoPwSMs8NOXpQEPibAYh88KupuMGkHzdhARSejrDskg4SXB5YchD", - "1zeYcYpDeD4/fn0Zp3NU35EsoPq6JlE63FOE8bjac5YPQiLEHb6+3I+r4iWa9BelLSsthpoP+Peq+HED", - "RGVhx2jWtYi1gowir2LyGLWuz7FaDPVeOLDfS5XttIFRkuRrxf5rrifukurNrnGRsXMu3PXh9fH5nyj3", - "/Va/yvs18j7Jn0TM18H/yOI9S/IdxamnzYo0iTIfKk59lYWoFBFpNX3g49fH51WNKzEOfrjWoq2DuNBw", - "N5qyY+/CvBulYEqVtou+k3dvmPsgIv1q67T1f5Ep6JZtX+CPm278pVe81I6NvGK+4kEZOH8lZkJOekdZ", - "pu569BQUTzkVH6C9IhnXwFs2RAUnmPm94E25Xs297hm1PiMGXbkjMKXZrUhBhZ9aKqA+rfKqb80JLsLe", - "E+gvXChmZO2svNZrLMXX356rG/GioysLwx/JxVVu56taWqOWFH+aC2wDAZ+58wptvoosvxTX1dsy9WYz", - "zqtX//Z9uxb5EPn+beghut9nx1xrAVgXuyyCO6ZGR0Ki9BlhGVnLfCnoLsOuJKFkdd1TtVis/cFcvgCA", - "r7y+mtcr+D8Fx8eQsV22wm5atup4i19sW5H/Ldyx1VX5WdnRtrwVrynMTy3mV1gNvmH80pGoL+6I/l4r", - "Rf/Sh3/TDiJF+VsaIW9dcf/R6ur/ueXyKxqw6tFq21O0S80SqqhoY1ZY/a4Q+uZhVErLK1NZ2n/B68ym", - "/BaoPRHqq/Jp2TRpp/G0UDYwFobVpqcXByz1jXFh7EymkDvrlGoG11M5XjLOjJCTDJj7glI86bk8VUDt", - "70ao88RDe9x9fY7YVq4/5ZPEFR+9y0GueCSTcFcaHJaPsC85yQUHS4WDydbwRRRCFs2Voj8gDSN90jiz", - "T2FeJoQa8kYpEGGqPBxf8c9tIbSJMapR0mtdzo23X5rZNjVDpqRupDln58UycfrsWElTzEC7+x0lGi3Y", - "TdhnIdTWn2K1BovFhIR1thNHT7fg2WNk7Swj7quRtJqZLB8NiFSfnol2sJFwa3FL5mqpt423kBwvYlC5", - "Z0EkaiWBooHlfFulX7W/iPXqkXCXzcul+OhJLAErbBZxj1D0eeZliPumtBJRMMQ3EzUrwlQ111L7HIt0", - "sdKmWEEitUOugno9fo6O6rDr+8YsNMCpE3en2xnx5GaiVSHTgf+LAX0rEhg4DQ/a/WHKNaTVf2MsfbTV", - "Sth1KA9zzC1MlLsvHis5FpMd6uolNMW8VnPG17vGoAtsROIobVRvNhbJ5eW52NrrsHiOuT/FchQlddTp", - "oW+xl/G5uyQkVtwKOw+tMIElhbFqBpqlWDLuBRNy5ACPjXiwkXHQWje16UyP5yLUrnSKqex+0/Nt9VOE", - "SDLlzKjsFmolexyJUNdtNzBbKu6XiTEk86TsTYSqTsix5sbqIkEolyN9UD29yvaUZENq6jNkAUdON2Hf", - "V+x39ITg9h0QmSpsXli2h50vfY9LrZXex11H2jn7tJWyXvYT7vE9vc+WSwUQ72HbCtNlNzBP1Z00XV+J", - "eR83503rJ9xYPen9oIyaDF2S+nRfnTwl+s7R7btIe3v1Owpl7izERL0+PndA+rhCXrbsYTu5Q4NCxIKT", - "LuUVpy6FPNEvv0pJd2lKY134gcpnOX1bCjb8N1Xwdxct1szPUe5KSFc1YbzxlMKYF5klceHsq71yMr/2", - "Ps10dHX845q59rDfF93bsABwbueM4MqGf3wc7qNVzaTqqfwlibGwlgbLhTRMWMPw4V1aKgjUZ1fK78TZ", - "/KkwVI+4GnorOO2uy+aqYLOCcipT3MJ9nolEWDZ0Zxu6GYaIpmGjsVRpFm5EDruQQVUUNIkQRNlbraGl", - "FnVTn72bOTO+OjrC2wZEvSAEWlWOFLbPLusf4N7cFxTYQV/grPXs6RtwyLdCQzavT8ezLKwtwNDU7m9j", - "J9KrH3D+pRWTDLhvuBiHRew24ne0qRnXaitEEYuufFHMAotSn/UtEfuKqobRm/MIi9qmVAlTFLPygMD+", - "7//+PyHA3oTmglNuwFeKWOZ83zU62lEORy7LhHK1AU0947mh6kTokj8YiwxIzx7kKhPJ/KDU/we5Vu7n", - "g1SYPONz5hTHy9L35SfExEXHqN4l5FDBrfChpPWins2dUDPH2kxRiy9eS/FdTpiowXKxemJ9ZWNVPgjw", - "p0h7bet/8PVGEQAO1LUWgO4//PnDh6KYDcYZnxjCjwPR+qtbOHNAYcwox/Zjb1RhwGdWbnlXGxXWxoLj", - "cEpGvzrcB6sB39lrcMpgbDvdjhaTqfvfmUjTLNjwdMO94zqN4gmNjpbklCt/e6B+Wt4wqlZ1Roq7qeQd", - "P010ganK0sENzE3seCndCN3P7nzu23pxOZp1m57RsphRszm/HOrDzotni5z+llrnu6uRmIFnrByCQe7X", - "XfYERPpz/Bdr60AW+mds2tTsv3eZKdrFrIVIc2z65INttqTReIbNsVfvSZi80UZu1w743Y4vJKiPKmt8", - "czF+FIwyX2Rbe9olXQlJYYFxqoNDWdoo6vvsagpsSIV0yAaiNiRexF/LapacoivIIbhc9ZxGOyCgHYS3", - "Ixqbc81nYEGb/rU8veeJzeZMyfJ3GtnIO8M7PBpCI6y+fCvSeItqYuWZkxnrdOyywPrY7aSaTzYbfqL5", - "ZHH0TN3CZqPfqFtYHI2Nvwa+fdmqwefuw59gXhtLt6R1A6knUH0Y2EFSaKPWWiSXYI/xw/roDEjBrRzo", - "PvIkXHMjLtfrDH6aJQpr6OEafhvwpplDnZMKlCVoGrhtnDwcJCa5q0nXHNPpiSu4tyV4Frk8nvPd7Rxr", - "4BZOMO1f6fluynOmUlhhaaRhduY+ZHsqsZgzo7FFGqYB/vv33+/32Unt8vTv33+PRhy3FrSb7v/5+2Hv", - "33/749vudx//Jf6Qa6eRl46RUZmTNtUmQlOoBI++sMhB/1/XV8tzK8WAeQIZWDjndrobHNccIWw8xWUe", - "f+MXkKDum+y2+5j7+WzJwa3DIrWTsKMsn3JZzECLxN3CpvM8dFWo4Z/3Phz1fj3s/bX327/9y2YxgSdk", - "fm54x1xICAA05loVbjDt6bsqJLIl+hOL6g40t7B+Sv8101jCV7IfP7A93/ZCFlnGxBhft1KwkOAz8H50", - "0TuRxghqcTX8bOX+o6Bd1EBPY3A7sdlibJdGNlndMQGagrt81O3Qw0VT5cR9spThMgJ7ByDDRpyhjZYG", - "3n889Tr5T23SvXffYpjUTEgxcxs9jOFkZYFc/yxklROQZS+Zxb2F1xvyKRCE3F5mZS0XM1PKTv8X1nAh", - "fwQ6RgqrZtyKxFnc7gwjbiDFsja4IMqXDOTEn4Pf0zmeHR4eHtbO9X30YA+5ZbgjbHXJiEvKdxpjdFkm", - "DJqVf7/vsvlvdZM+50KbEnchk/9uKjLaxETISZ+9caaetx0ZtywDbix7TnWw8QGj3OnilmsAmfH7M/r1", - "OQKv+o/F06z8kXDZoOFYP/n3Bti0mHHZy8QNsB/gg8B8Q30LFTUjhu/4nA7ChDQWONaLyIQE7p3iucp8", - "j/lfsDusWw2dBGaQgx4YmCClETtAPkAmG8z8E8VEqmacdO2RuPF540jfb8mXZcAn7msJg2e0i2VuWMuf", - "S+ds3mIP26+x5ZaQtmhfmAzn4eUfaVBMtG+QvaHtsWeNvT5be+1sVe6lG25Th9jCxKvcLqd0lzvP+PwO", - "pfCmyiBeDat2O6ymxNITy1etqM3p7GCqrHHwn/yW0z9xgtrcdM3EP065YRxr8bvfv8n5BL7psm98FMg3", - "dLv8JrSrY7dcY+snf3Wc5Rm8YNcdfseFxdfd/kRZtffN1NrcvDg4APqmn6jZN/svmQZbaMlqn+O7997+", - "y+tO3X/ezCygQLKkQYd/WaLDNySt/RnxCuNLrIdAi2BeM2HYXw4bEv7bhnxfT2sI/A3pweCGtySHUL5t", - "gQqq0y2/7AQqXwhBwYqjnoSd3VTBx1d4jVeM8ZtevidSZDphsiqUipvboxCNfRIjKejIfi4tlyn2n8eN", - "lalV9YNFPLmpiiWklpP5x9YNZ6PmE6vewKAObUgb/SrizzyNmE2/QIxAXokMzuRYLcsjYQap0Kt3hfoL", - "H73K61xLWT/VmiDmVPkMDRKqV1TG+5dRSCm30PN5oMv1kqJyxx2LbrcjYQ3Vtumy606q7+51z/3fdcdd", - "bK47PX3X0z33f9edeJUkyWP7/oEbaLZYF+EJbxkSG9+Kg826TCTiAwxGcwsROrkUH1Cw4M99n4sWtiFg", - "kybEeEa/u8Zi3UAHNRx6oLeR0yU+wbREOr4q32ioKzu0lYbdhPz4eBwauG9Ih7vislxqV6RuRyVxt5gP", - "35vnUPeBHV+cHl2ddrqdXy7O8H9PTl+f4j8uTt8evTndIBSP4qtaDRasorX4BtmC3xPh/muG1n3KCunr", - "GJShrYvdsEL9Fy+3KTiIEg+dWSAMoTXE2PCMWX6vpJrNX2ADSYov9ZVAq9mN1cBn7G6KwaYpt3yID2xK", - "z9CyULLENdoQbisjyNQd2yMPN22JXN/+XX/YDodhl2mYcE29rtXYLczyIrQQErbPjnmWge5Vf/QAwOf9", - "d5dX7KDc/UEtwoiCZqWxmp4lMepJGILsS2YA2HBhL+V9FAujminPoc9+5plIy7ISCW4mNGA1jE+4u3vQ", - "1AHAoXhr4oNyvzGhcFh4EUUbKa0wTgp/xvNcUPMMnouBW2vNw/ZRLhx4iKS6HR+iNcAQrUFQ/itnOKYh", - "l24EWSvlZGleNsBeM0eaN9qm09haU921wxcbM5fxXQNvDa2egL5FC2lxfKYmm41+rSZhbC2gih4A18xw", - "Vn2PjyGxefA5YtNZfoJ5bA7ywJfh9htPR88VjayQbqTL+fZN5BvTbIzvtm7c3Xg31N16ztZmq7eL3KUj", - "Z22qpWZ3O3cWjE269XzLc9XaA+3SgKnTbXZQ2agObdVNp9vWfGLHLh+1CUM99q1r3Tfm8AVgty+v2+m2", - "FuTbsfRhmHGhrNfGNa+a3Lxc3Wn74lnlNEm+RQmWcpTi6TY58mFcLT9069zb5Tm2gGNLvlx3KSNj22QX", - "eoNGk3z+Fs1msgw/djtKwuaxrouK6WN3m2E1bbjhwBjzbDu0zjLbjY1w/3YTVGJow3ExgtpiaJyrt5ig", - "YoUtBi2R2s618rYaG5h9+/XqvLUTYnaZIW6RbT+4NMS2HxoxujacpEU1bzd62SDabvySjbHj8B34ucUK", - "23B040q0qchcuMBsPmzRht1wZNSY3nLsjku3XfiwzsdrYSw6miJOGa353F2Bl108QpLHERNQpA2es/JF", - "cdWuSjdq5G201HyRtN9MTXy1odJXXCvdvjJqerHO1qT0qlu4t611kVrqvlyJma8SWO6IqihSXtym/tiW", - "p6r60jEPEwYZnPuIzovSvl10SW8aahoCuXYPMW2bYePQ0qWIvu2iMR4xKgFD3B4Yj5AKY7lMoPFI9f1T", - "RyG4PW8VhfDwp3nvSa7e4d0/ubQLUIw7l9eRZxXmECiMWbUTmW4601bkunucXArGDtbF+4Gx2CtCyfKV", - "Y124XLdjdLJuYkoe33jOxbexsEC3dooYhN7d1OXSFo+nf6OSBezdT2XfhWW5rm7WUu0ZlSKBslt9f/3L", - "n7qJnuWc22TqQ/F2w3hbLN5JewxeKSief3e4fUTeSWskHrafVfSM0GWFAfJaT8VkCsZWHbtoSNVEBMnH", - "K1n/lvKXw+63h93n33efHf4W3yKC1juV1uFr7CN1NIwLyhHTgGUbUARXGcZKV0GYBxrwmMJQUjTEJY3P", - "eKryfpYDPavVqd5ayAbzBfar84d3OMxqM5RWx3jKc4r7lXCHxSYa4QqU9eZgOQWejousS7l54S9ZC3m2", - "hkCetIY+lmTz7fPDzQIhF+Phd9O8a4IUg9YNaotS0eeGIhMXS0fWSNSh+7BL33INzPI8J/tqdRzUCkVa", - "BnbP1mnUG5hjIVfDjAOO1+ibK9j4+q99eJ+b3cxnI0XZ+biQb8DglggFW0bAeO1bZoo8V9q/uN2nyiqV", - "Xcs9A8D+69kzPMt8xlIYY8s0Jc1+n/lgn6qpz3XnAkNArjtddt1B9wD989jqjP51lPk/vfr+utO/phA/", - "igIThmIUE9wgz4xyu0zUbORVlvFx8TTfv9kQPYD/hav92xUf4bRbAHRBWiN0o/KaKiee3kPyaPFc3B1v", - "hjGDc+nkiFSFySIp2lxPmqGBf4/UGKCZuJ4UZYXYzamKm4FWqhnYFz9G4UP2fCVJ7A3ihrJci1uRwQRa", - "xA43g8In2q6eMhRgdF+7qWSRofYIMn45W5DOHnmtR0CHxG4zhSwrQe50QRGvf5fcxbLhlb5xPFxdVvd4", - "Pbpg38/o32tpESowvHiA9TYXyNt28vojFtPtcfbHx0WEncpboZXEi0cZq4fli3zBqhroa9CoKH8p3m67", - "ELt2BLZH0hE617Lhg8LoeJ3pSoSV54gU91x1Hzwtz992GYxXwIZ7YQfxuE1/VOY+WdlqLAWtB6O/fBcP", - "qvnLdz2QbnjK6FM2KsbjlgqHFFW36WSqsO2TfWzH3k+iSnnbDn2XYuKULFKvLKum1ai3iTKDnzeEWufq", - "9OJNZ/W89dAe//lPZ69fd7qds7dXnW7nx/fn6yN6/NoriPgCTdFdtQmasZydX/13b8STG0jbwZCozMQr", - "h1rQM0GZ71kxozKcq2Jeux2t7tbN5T7ZMlAbZ+3SRldA7DLnd7IOsI0KvkRU93JNZp5lyl3tBtbO12vB", - "I/814yw3UKSqV55+7/zqv/cXBWtVr6KqsXMLpJFa1GUcaaGs1yLi6EJTP0S9YfEuKF1ayX22+zIfo9Wg", - "m3jdQZ6f1RzGfOQEEmfGzbaKH/JYWt67yxJZZydxUet/jxaVu8SKV72y1m6kslxtP6UftyhE2tKH0pnj", - "A27jfmIqHojYqJOZH7aFq7iV1cpGmNtUIqqV1SkMadl2qZQXgzzWxfzUWDHD2MXj8/esQH96DjoBafkE", - "ol0VVqjR06A+Q1nHAKspJ91K4Fpno3Q7M5i1Rf9WO9ZgEPNsBjNnI9Luy8Dg1mahK/Q/1SiqqSRdSOnQ", - "R8duK/TYjthUyN2Uzgm33EmyOy3IAbpAehR4j02e4rXRNzIs0voq64sVlvP+tvbMD7IX3XZ8kqNx0y2f", - "0H1hQbYRSZUVhR8w/3m/s6lLxR9FA68iu7exnS5PQ7Qp0+B7ybgTBQz6jAmll4qdPRSb5cNaRSzuFFET", - "FOLvdK+bW1oKwXasEE133Ug0lIKUJheGXePA604by7r9R7QAOcJ96LOq1ZFNpoW8aVYRwgSWMi1mQyam", - "2GXE/8P8ENhmHgsW05ShohoBQHruXgznjohxXymsaWVTfsGSnU3h8/VadLVyTliFsSop2A01P+sFELtY", - "GbMb5o/mN/uqtG3V0eiEnhP6D64lvCZNYHVN+U1LUlAZAtDxNKGxkBjPvom1UNUaCKPabIW1bhcyg5b/", - "bMqiCbXfGxmvG9s21W79oB03uwBntLnq+4zBvIqauYDJJuV+Nnue+ZGeZcrSDxPvK1hRKKHFYf8LOuq3", - "mWjDx3ua6xvjWzWMnZDUEh70nL/FnNEX0wCFbgDsOpTt8vCgS0SvqdnTJIyopG5W9tn2MTezfHC/+v3j", - "R6XFByWxbgyuxfhMFdL2GUVxuPsl/t0wzBbtMgkT3vi7w0NcwdEO1pSJ+NntONlg/VTdycjyRR5f/CEB", - "C2Vtoc193+u4ouqzVBZAai61PVNsPeXGUQRLVaG2lFoiTUGuyYOlaIfqKckPWvsU7r9r2fYrkcE56JnA", - "Ys9mt/1PtCryuH8Kf/Iphpr9rXHJ3zaXNVKu6S/ffbe/XXUmdSdjzyFur/gTPoCE/b5v2e8meY+UgpdX", - "sKVXT3pgw5fndNfKSSvyUOtlxrYspM4LA/WsdCppnEPieD8tXexb+ujrD8ZYXyzmoq/n/zdiqw7XMmV9", - "8ShAnAnzyvzCbfKoxbDKSmV4a8aigfEMfse44hbWuzdLbvfzsXJsNt8g5KU1gAch8MCSWtgEJR6gclHZ", - "tuEjh+Jx7jj2FrQWKZhQm95DYL+O8+eH63ylUc9hePuP+PxqBiwVsX+kwl646UDQZ/KSCLj9fa7aR/19", - "quxdvhI6KwEy4/eYcC4+wJl880P7DjDY1/g0+Tc/bIiRxTpLzzYMQLm0Kn8ooSmdgJtnPb+czXxvA+xB", - "pfKy4+xE8wTGRcbMtLDOCvIJ1TMMo0LXkpAYB6B1kVtIfZtXB6z4s8A2FeWIg92GnrCcXJX5LG8hU/m2", - "sXlXWLWLhlat6qxyEr9WYoMtZG1H6tgHx9HKopDN3HksuPl7q++1V/XgDME6jJzO1U45dhymuG0xhnq7", - "YqJr6qqJcRKvubE9XLl3duKj0Qof9H15eRr8Rt5dJgxV16KAlqWGQFs8r7kzBs/abytx2BYkv1A0gMoF", - "3QkNvpUfOVUw0R2LB+W1ggIecwxkiuehtiAhFEsunL7PjvRIWM11yP33dpahPldUSKBKm9fAeEqT9dmr", - "pdYqq6obdGNlCXDHoHvovCGyKZsrQhoqVoXmWf/q8/0PFv5ygvPWAqa6bLmoQbRcbsOd9nn4ziqE/Ofl", - "u7el6ywG7UwYD6XVpRqocg05oxeh36xaHIMrocX3b3miTmCXYAPNeP1UOolbG4NZJ7mpWHfVHGzz3mDY", - "CKzRGqzRFaxRC9ZfxHToJka78wGOWzYQe1rfZYn7y/DOtcOLYltDh0jjpDwTLc7FX5oNjZsNlAMwm80z", - "HH79lJSlUfa7rHYkQl893zwg3bxyUlIWYd2q94TvOLGz8vL6KePGLulVdhIaBWLr46DfmmChW2O8Ud0W", - "NyZ/fDpHlHYWajhv7UZ7WKXTG5gbq9UNmGh1wmjsQ7yC4k5ZMSFcr9pHyAqqZcc4SXTvLsXuJP1rebLU", - "bQc7qnKD6SqYD3WQhjq1+9RhxcmtEE5+LX38rxMBbi20XLhkKlxzaus1IMX28G//69DBxSft7PevZa1i", - "Jpbhd1Cb56Ql7pROe05WpvRC5gNKy5MLaTXvua9oQXMtnRUgORUiQvVGP+e8MA5PzjChvZGEdntZgbpo", - "j55uS18BR4oIVyyMTspgqowtS/q3FJJSA8cwCaymRezMM+VOXTvLfZ4rJqTjBMdx7jL7ks2EsfwGyOxB", - "PYkWBcJsxJMbk/MEKiJgh332TmZzL8JMDAJsz4gMpM3mDThdy+ozpI19AlV5MzvsP4tSfUtv+NaeCr9o", - "YaHsArEbo6/GViNcIRQ+Cwvu2gziI3Zmo9c4383Rt9VjZ9gHjx2dn3W6nVvQhrZz2H/WP0S/Xw4S+/t1", - "vu0f9r/1Zb/wIAchm+SAOsKQzyeJOH3egJ5QS0f8kkgA7oXBJ30lwXRZkTvlwxYmjeSj3Ap32cpB3wqj", - "dNolJsOSnIW0IkPIlV+fwO2VUplh1x0096SQk+sOZq1iR31hmBqhzZSGxrdUGxLdID5xConJ4ZA8GCm6", - "/WwyDau88h1xfLWWH1Q6p1DGqktIlaR78A9DTkbSmJEX0gDNBesiHIlgaBWbIVh9rcK/X3d6vRuhzA0l", - "LfR6vjdYb5IX153f9nfPM6ANxcmq+s7xJ6UaYc4arvP88DDin8b9E76pJ3Z5NI/sxYqVH7ud72immOVR", - "rnjwAw88STVzP3Y7328yDusXSJ75UVhjczbj7mLTeU90WW4x44VMph4JbvN+zzisot6yn9I6rigM6F7o", - "SVItA1jIWQsDjHpTscoFVQY8jHj5c99RVfdarmUXtj23XMtt2eUYNNbeDlBgMy75hK6TN/5S22yFiVTM", - "TkPrqUvf4617LbHJZg+LM0NazkjnKOcPZIi+zOOT84OQm6zkPuofbJMO6bVEf0WA5VrOPq/aYu3K3HHV", - "ELOoNkF+n/0UMsH8T5LPwFzLPZ9v5LXpsVI3AoyH43WH2nZi8Vv/ojItZ6C/9q/lJQALpY+pL1i1k/5E", - "qUkGJWEf0EtHmS0Z/k4g9YWT3fl/4EYkR4WdvrsF/aO1+Wno4UgwiG4YHUXuY/M+n2iegilHeaX6ht/7", - "MhJCSXMO+tzRSefFt8+7nXOVF7k5yjJ1B+krpd/rzOCb3nJZ585vHx9LrgVa+WJF2yLZubO0S7gizxRP", - "e1W3uB6XaS9868SeMhFD5z0Oo4Kams2cBCmnYB9EzrhOpuLWcTjcW2zVZqcwY4VMQbODqZrBAYmQqluf", - "ObguDg+/TRwr4L+gey3dfVA7GTerr0ByW8gdDI1Scl7LP9HQIHiVgtEcyfTCw3iVTJoVmRU5djlUetYL", - "vrI2m6PW8681XbP6xhkfhH6ECSYIcNuovdCcPl5G95XKHE7x1dgqlmc8AV/+OqBrO6wvPBAc9X7lvQ+H", - "vb/2B73f/njWff799/HH7Q8iH2Arw6Ut/loRZGgo4WMPC5lTJkvFPuWu97DXWEg1nXEpxmAsquj9uhdi", - "JKTjxHVWfbk9X484djNZacDVsLubFfcsFo9aUgORAqTdiLQjrimZAztm8vRTy70lEVRis0bke9w4gWT2", - "60KwPKKXhv4ufTAKNl5c6p2GLFrJ1EKTk4UOe4Ye2Xz7vdC+vM+O/K+o+SkKx5kz5C2zgmfZ3HfRmKqs", - "bDl8n2SFccTrzJ8uM4pJxbDJPIW+s1LYGJZwST6KDPgtYIeEENRgrMpNcCKMhTbW178PzfvKXteirDpB", - "3srQlI8ak17LUKK5MPjUiF1Tp56rUqD8HXcvrPyAmJpB5VTcajcwpy6JHlzXMrxf5nzuZvHPCgwb7/es", - "FjlzpqNMKIIYML1cpuJWpAXP/DQxyfsDGoLNLoq7m4ErfabLK1WN4HYzRnDKlgYAn5L3SkagjpFRBqjT", - "9AKbLTRoDMzWRFzVmvGJ8BXp/bgjmqhbVuhsGdj6k2LoUsyKjNIFievqvWvjjsQlHJG76sCJ+nY0XQBP", - "j2uurRi0HgtdzbatiK2Fu1fZfdUviXpqiW8eDF13aPIsl3kmS16+NnCib7Adnk3n5BORftwDuiv5o9fT", - "5xZRU/SAhc9GYP1CDtngTN8AX2VD1DiayqDXJ8LQcqvVjZHzKOvXCl/F+IzicW9FaApQ3pY/G4z/KFJf", - "gkPd1av7NdHcbPUbt/qwshBaLRj5HQQq9STslo9UznLjoaaeW1ZbehXC0AO52KdwIm5DKzgyTDPgBtC2", - "qnfYWdNEL2bxlC0hn4g0l5se7yg33ESfibrErVR1EwlNHPGwQDETsEQwg7IXeauQ+BvYRo3Lp1SP8WKa", - "cd7FqAM6aXmIx4Di38A2Ahu85UHCIqy0ifHR7KEdB25Za/OJyHy5O/eDrEMPBXeyT0vqb0IJyQZ2glYs", - "I94rSWM2wVijb/kKOerr9FXr4DM+yszae38Zbk9+8irvo1Zs7FrGSohRiBiWuco1TEHSvXm5VlmXGYBr", - "6TYTrzfGuK3c6BNh+2MNkIK5sSrvKz05uHf/L9fKqoP7Z8/oH3nGhTygyVIY96ckz30411RJpU098MPH", - "Mobzuhu1DyZPPCgwbcB4FxphQaXRFw9fAO+J2GGp3/yO3IAIRWr5nKwF0vF1XxLS5QaEX29a0iaqrvgN", - "VCl8T2UxLmUifvQ4WqlxMCz1IKfM2Wql9d7NJcVSbYBiXT8pQo95ji+SnFUICkFoa9CpsqxdiFGOJbv1", - "eYjZ3FlvB8rxdsiNdH+zNRuvJkmb1mLDz9eo4ujNwEaSo2/sK1mmJpgCaUVyY9ieVNYn4JKLs0ZBbART", - "fiscSfM5u+V6/pLZAr10vo95YOAQMzVSdlo7Cj03hpxLzND0vkv/1N2tR6uGkB986Wm4NPfKOdAUrhbY", - "p7gP9CJRsFCI7A6icBhiw8iB0etpyIFb9pb1ehR0dcjoBYEMcnpDGMYk5GVIdXwi9qsl3+4qHT15fSY+", - "JNpMZSsQerh1lvEW1lwI+m0Rjj7g8onwshjP+SAnBwURfjZay52NnBqrsOBjhNtlWlVJNjw3Mvf/KAx5", - "vhiejFKrfCIyljsDzao8x9SKBNgeBSR0r6V/k61eY7pOcGBaln+O69ZsPl8M2IgPQk72/a25XEiUpaYY", - "3PPEZvNrics1XqY08FRIp8vd7dndxzGKOqwxpALKhc6GuJ4XO5yNwNgejMdK22tZdWQqyyaHWcMrhZsZ", - "DTV3seETYJSe8IOTjQ4JoY2jnvEMQ02tupbDYE4Offl9LucIaTZXBUsVhkBLcDs+Cu3unUnibUGMz3Bf", - "47vkCJgvqNO/xncGDJxp4oq6n+tClvVu8dnqRS3+po4bj4EuPa930TiWixjrR1GiZDYn7HvVBzKlwNgy", - "BYdi1q+l1VyaYN6+YGLMOD7t6Cr8x+0bH5vcBrnOnFqsmI4ZkQIDbM0a8tpmXEhHD7g2BQIn4GnV/Ukq", - "2Xt+f+/fu3Ktcj5xCrl/Lc81jNG0duC5xU7xOcdEzmEVXfCvQ0oFOvAwGuJ7no9uJbbJILwu9qwWkwk4", - "O+laEg6Ik4REfPq8zCp8P6asApSPS/59xEABCgsa1MPbFuI7rl71/sPn3jRjl9iM5+z//u//wzDG28CM", - "SysSLKF7fnR1/CNbjp6LV7z1Xw1aAiVrO6A3bjb845qCGK87L+pxkr99HG64IRwd3Y1H6ybbmDmhgZZJ", - "/J60XGV/yPawksgB1RE5AJv09/sMDS6qNh0CqpcJiELKTTe8z2I2a5kgsiiNRSWKG2FLDU5tMmm0INaK", - "OJLTepiPQS9k2H3iNFZSYMGNaoo+RobQMarMgJVxR/v99UEoDw4Refr4DYwZd0MGXnYuQ9Ny3f9gbCw6", - "BdO+wCB4h43YGQw29UmJXjh7UWD6zIuzEH/lKzFguWzf3qgKHPSD3f8zB7XW4WjBG8jc+D18bqdQOzb0", - "YX4HtAo+7A/3Kd106OCWDyqWGJJWQBFJ6PbxDOGwdsrL+Brj9B1+cKd5nsNSK/O16PJVnpxyj7Dxxevy", - "9cerd/DKvZLCK9V36QvqsgzkhPzzCSdes+z54Xf/QVX2uhXrOQQmGOxLYRQoIzwCaBejDFqqIjdhucJo", - "qxKsAgTx9aAaSxnZWuT0WLlAkyVV7DkdWRbM8ZlEWBkd7u3+RuWWP6snqoYl5OXly8rcLKnAzZzB4ttV", - "/yGG/XeHf10/zm0wE8nSdeBxHssXrYdwfWiFE6DB5f4XZXkZ052yfMoRxPWbxxHaM3RtT0uDBq/yPju3", - "aYnmWWGWYB8KWR3UtG8ZZR8J5/Za9akcnJH2OH8yRfvVQ7LlMrLe+1fWcFdqAPmTUeyDY5dbjuNIY2wO", - "Eg3cwqDsgoBkUsQihvDDsjbNU4UNNVfZilSerSqlQ+f8jNwLdFLGMeerAn/ASwpObG6AlxP88KnxQqvU", - "25nt/C5dooSOmD6Ms75bP+6tsq9UIdNHfNDGnTPejrdgB69A2Ssydz9vbGGhtH8CRCE+ShypO+ksZsdd", - "gw8CCwJNwMYKUNlCS8M4+/XsnJV3gdodIlwNyhIxVVGzQBr95RgSv/6J0L+KHCPyNZ+BBW2w+UFbu7+S", - "c9AGtaq09Z1pEA6Ftzs37vcCUBzQnS6Ud2vSQLfuxFhXLu63rZSzh+uDHr0c1MMZy0pISFh1AH+JdOmR", - "VRch7jZAhBYutHF6NTbdgGDD3XfPcl27AM/C4zDaoW6u/ZV0fS1XEDb71diUqfEYtGFGTKQYi4Rj6vmY", - "G7r+0YLefr2WKdT/5P7NNd0AP4jcO1x4MhVwi81SwS7OgmwUj8yqcZWD0ZfCVt0/llt/lcfFCIY++1FM", - "pqDpv8oOwszMeJbV3RGjwjLLb4BlSk5A969ljzBh7Av2Pw7bNAV71mU+8d8hFlK29z/fHh72vj88ZG9+", - "ODD7bqAvbNAc+G2XjXjGZeJMKTfyADHA9v7n2fe1sYS45tB/7wZ8hiHfH/b+ozFoaZvPuvjXcsTzw953", - "5YgWjNSoZYDTdOroqEqah39VdZc8qDrd2m+0ZfyHiRWk31Yqeu59kFi8WvBr/X9ENC6480rxiA6XULvB", - "i8WmaChbiW8qE1ASeLAudTX/XDTsdjZh1U59maDQyqv1av8CyeZvYBvd5kPzoCXslWSTCWPRTjetdFM1", - "vd9NmXyZlFKdOkIq1fUto9okXyCtYLYuYp4SCZdpA9ukt13fQmPvJwyNfYyrG4aiVu6OLxBPeAJs5Yyv", - "XKuYWQNPy0t3lJcvgKf+yr0ZK+NiwSR0838u3KwSC7ZXtax5kC2Boj+ax/WFEQtmjTWe60riMECCflAr", - "md7K3cuV658uCamlRP7O1TVqFeF9ytAXiMhLsMuMXq92f4DV9M1U5CWG6QW0PQgL65yY2kOpzx1Xuoov", - "IYXgQ/U1zJSXAZTL1m+pOhHMg0eLHiktkpYn+hSMHazpEuC+8V22Swnmq6Z5g3aT/gDdzq6v+f4lv9rq", - "1uUYCAqPVokBsVQWYfjSRV2kOMPY22t1dgiuzZVFZjg6XigGDfslUz0ZYU3l21xKX1mkrzbmIO/mo7HG", - "tqSf1hsp1CrlVDESajM+eKTIllX8sCNh/yryiqxrCPynIXJeL3i0QKJL9O6dK2sIflvXaBtfXMv1jLHe", - "RdrwiF7LBZdoe7kj7+N8NOZqjaK6msKi66VUIRvEDX0ypo1H+bQVa327eaCP707l94bFjLC8ryOnXg+/", - "6VXj9vvb1VAOeHgScXHkYfhPLjIWybVFbNwtFiRauAnU+vs81R0g0kJoc9zuWDwVjx1tev1eit8LiPW9", - "qbjyzoNjo3i1xXrtNpmyx67x94mIjQ5Td1L7Qk1yUrPEEFoHfwSQf/RlzIGKlCzSm8orcltwUqDjwXsa", - "vN+hxOMq38N6V8N3scL6hCgKdv7CEXWJDXxCXHnM27eIpAPKkWt1JVHP5lfmlD77E3G16BaycG9pt1F/", - "0Lr3gEu82vrWOZGc06qFjRrX7sI+hxB7d/IUT/1H5796l5enPV8+qHcVbUXxBlLBfbX1MfaIwdYbPiVx", - "b1GI7Tde7sIr3ZKoizzKffwSyZR6BS1C2Zc8IbFbUqy7zK8OMsKiPJs4PE9qxhdfcn7+ie/e76qGBKE7", - "Y2tjxkbvlL98913bNrGbYcu2VrZzJObbROM/0B27ozejLAn1patRdEs5zRniIatQrUxNzEEF2PgTnZr4", - "HvotcniBIHx3oVWUGwSNJ/Gqvm20p3t8mbHKMnUXjzxodLSu9VxcRDMmeJRpe2LMaO9MGOa3toIx27XK", - "NuvUzh5frfpgkFObms4n02iv1WRDVeYI67PWXjHN4DZNOZSXl6fEIHnG53ea0t6oaOQG5VXL5l/n5WiW", - "OGGLb6FjDWZa69WKqLm3jE+4kIZu4iELQRcSSzhLJVmmEp5NlbEv/vr8+XPKTsVZp9xgBzmDovqbnE/g", - "my77xs/7DSX0fOOn/KbsFBOqNPiuij4WA2esNoelcm2hZdXILZBXzHHiQVCd+5i0w1Pc7JbW+kRZD5F9", - "OIDGk1VK4H6O5VCrI2DZgUvcOVFEhDg9g5BMQu5ov+j7BltuoSer71Ou8InooLGDNgqoqhlr/81nUQY3", - "UbOZkxJmLpOpVlIVJlS9DQg2Ob+TazF8iV89KYpxiU+LY7+FNiTjz5+4+MkybvkK5P7h/4F38xvRrCAU", - "RfRPAkvRrL+XVzOvNAlLS74oRPqQy8JOCHWn+Swrlb776YuML3CiREzcTdMqFszWdoqjwgBrae6CPvun", - "oTo6z1e6e7wAJawvwdn51X/3RtRKYT3xGctt0e6KDCKfvvqzae+J9RgdKqbC/C9fZJSyRwAz4XjtqE/F", - "BjYNfvVPI3XwOJ/YfqIttNlPP8yxdQe5375Yj1ul+RjR2Uo6VIVd54irgKcKu9Ij94nk0QM8S+XZ3LAN", - "fUwBuqqweUE98jMxhmSeZPD1AeXpHlBqVK0Ku+Aw05BgudDJQfUIG5eulDl8Eb5/0kTtcpX1tWUX0z39", - "wE+Xov2JaluUid25hluBd0ZGyIWU3YoUVO0doYZ1n1zWKsVC9lkd8Stfz8pHK7+6rjfZpypkvol/o5pr", - "EWp1+1eBcnjbQxYKvfgzFu99OOr9etj7a++3f/uXnUQjAuxgln/34HSCiiJ9zGNDwJW/9l4JiU3qe0ex", - "Rs9iBsbyWe6EHFa4Is9uNTUN7rO/FVxzaYHi5UbALl4df/vtt3/tr34BaWzlkuJRdtqJj2XZdSNuK88P", - "n69ibCwuJ7KMCSwWOdFgTJfl2M+CWT0n3yfVeGyC+wKsnveOxu6H5VK4xWRCuaLYVgM7QArJqob5ofui", - "nhMTVIcoY9meRWLZPn7BCadUitcgL1ID9Q0kSiZIe7TmD154xjYP7U9R5gOsUihhNcr0XAqyX+LX0LhS", - "l7t8tAQ7nmX1aZtgW+qAGgm9e2rl21xkpe59topFvRD4AitEIQTKKu6VXOuzd1Ryti7rctDs7ARbIGJt", - "84kwFrs0YslqJ0H6y1hW+Sokq/zpcVxbY3fzyofCfdqC4VblTfVD4DYJz8CqD6DVge9nv7JNCN0V3EQ/", - "v6GihW4GLPyhmJul65DLdZrh9WXMfry6OmdW8/FYJExJJmyfHfMsC7VCjs7PqES2MG7KO6et7vgNMGHZ", - "CBJeGGDvpbjRfGzp19B5PPGNnW7ANymZhyIGIefk5zfRUh90zEt38iv1K2jV2SSsEb/vWdVzp2QeVumj", - "IOcshVmuLKkNPzPCFQJUayDqLyMO5Gq8XYCxSoPxZTNp6vIoZSeCao2uk7/qDk0IhGZzM2Q1oEUj0gwI", - "oTS2NHN+fsOk8qVEsHK28bbNFLKUcYe26Cu7fDhuQD4RamjidZixkMHM2T5rC+3UGzKVo5ql9vosfPzd", - "4XdMjGvfUdXuqkhqtPXM38Belft5Qu9Xucil5Tbqdr+KH3BX2225u1X7/GXlygVxxrVvgkH5roSQVkSg", - "Vku4hQlV4oV7ByzhCMNg/Yh6HRU2Uukcq8lSUHf6Mtzk6lNosJzGCV1SgqEO/WYr1DPf1x8NpzHmJFXL", - "2JInXjDs7s+SDLg2oVhT7ZSx7kUOek0ieoIOvRR4US5TL7T55/lwd6biT5UxHSvZuYoRiljfHLBrKD/Q", - "4fPDZ006vONEiDU/SkWTL314lRt36MYJ6wY8Fqm+JLHr/q+U0V79bCcizwv76aj7s6fmbbOFnmZDBj5t", - "ONHlKgXTUPq19I+4MXYm/4HdMbgkzzsTIRO0WoAeAnyXDvrIMG6MmEigNqdSWSW9CSxkooFjS6bQ0z2U", - "HucyZWMu3ShVoCXnmE7lIMNjQ6KkBOoLHmeOUSZMJf7p/eKJHvFoLVziEz3iVeeUt5CpPEqkuEEMS82t", - "z4LMaesPUQDNpnc03wZEskh+Sw9tix5nkNS89hZY882pmplIuM9OeTJlY81nFIiL5R+UnrGhSF+wPwz8", - "/vH6Wqbc8hfsD/AA6zmAu79fX8uhk/UNgixblCVgTK8kY4IhaIOun0QrYxYEgE+Ne8k4e82N7SEOemcn", - "dAfFbj1eB9Uo2nHNLc9EihdEDaaYhWtn4LATrXLaFAX1UMfKCc9NMOiGIh1SjwzsiOPv0CBuIaXfhKEq", - "CnbKJXvG+BR4GkKOM7dXAyDx0254a7sD7RhbYN5s2ad8VIzHoPvsOBP4le+taTVPbiKzOW5OwUJicb99", - "9gqjr2sMTcnoUi2ADF1O1bKV3elR5ZCBYf0GAAtMB3pw4uhOOFhNeY4h/thKDyRokbBhU0gMqd9nCPf2", - "JwdvBI/mOPYnbJtBTQnZnvt8ju17HKVQkznOUpUUM5Bu1NDOcxhSAyqa8RvDhtRvw9GL0rOy4ETVDMZr", - "33/FbZ3gx8TvXWYgg8TvhyaPdqdDYmkeb21VtwtHbqGTBZoqC8LZd5pSmhmQKTukHPEoakJLt035qcuM", - "ajLFLc8KioefgWMRrSHBOgK0FHdrCGxYFZ6Q6DGgekNq0NCny9PYSEK/3kC6fXEpHIsnYNywS3wQ7F06", - "IvFk6Ub/vwEAAP//O92XVQ+aAQA=", + "H4sIAAAAAAAC/+z9+XIjN5YojL8Kgr+JsDRDUiovPber4v4hS1VtjWvRr6SyZ7rljwQzD0m0kkAaQFKi", + "HXXjPsR9wvskX+AcIDciuWhxleeriInpspjYzo6Ds/zeS9QiVxKkNb3nv/c0mFxJA/gf3/P0PfxagLEv", + "tVba/SlR0oK07p88zzORcCuUPPqnUdL9zSRzWHD3r3/RMO097/3/jqr5j+hXc0Szffz4sd9LwSRa5G6S", + "3nO3IPMr9j72e6dKTjOR/FGrh+Xc0ufSgpY8+4OWDsuxS9BL0Mx/2O+9VfaVKmT6B+3jrbIM1+u53/zn", + "RAo2mZ+qRV5Y0CeJ+zwgyu0kTYX7E88utMpBW+EIaMozA+0VTtjETcXUlCV+OsZxPsOsYnAHSWGBGTe5", + "tIJn2WrY6/fy2ry/9/wA98/m7O90ChpSlglj3RLrMw/ZS/yHUJIZq3LDlGR2DmwqtLEMHGTcgsLCwmyD", + "YxMgDl8LIc9p5LN+z65y6D3vca35CgGq4ddCaEh7z/9RnuGX8js1+ScQ9X2v1a0BfZKLU55lL5ce4S1I", + "SvbD1dUFS3iWsTmXaQYpm6zwMDegJWQDseAzMAOeC2aQsNZBmXK7lVwi2zlzwxyJqEInsOMEOPKSRnzs", + "96wuZMKtA0f7bFe6ACameBa3QzYVkKXslhtWjmJpAQ6xRvwGLBMLYY07ngfmRKkMOOLERggFt8KsWICx", + "fJEzIdkHKe7YQiRaGUiUTHG2qdILbnvPe0Lav3xbTS+khRkgi9Jffu+BLBaI2FyMHE5qmDVWCzlbIwFr", + "woQlIHekhjOPtT0Y7wL0AEkl56tM8ZRNlWbjsNkxAzeviRBIoVHEjBYRMP7Ms2yQZCq5YeE7x3YObUSR", + "2kF2IbJM1IDqTyiLxYRA6NajRUSEGN7lIE8uzln51XkaFlk4WQIp08oJjQMYzoZsnGuVgDGOz8d9Nrb8", + "Bi4TDSDNXNnxYW0HAS+EFjA2ur6DnP+didRJpakAzaZaLTqYLXy9EGmawS3XEF3UWG6LCFSRrYMmZvQV", + "S1Ran6UkwBZN1Q7Sgmu5Xr+B0w0U58jt0vLkZn2Lp2cX7H0hHQMN8ZMrzRNgGnINxoFIzhA2/8GX/BLH", + "kZwy7lvGLf7oRqOUlkR9Q/bKsblhhQHmVpB84SZKlHQ/oyTX3M5BMzvnkhnJb2CUcINyAGkB5z2da7UA", + "dgbLK6Uywy60sipRGbsVGhix9PBarpG62+ErzRewg2bB00zx4z5z1KcXyljSIg390VpCZcVCviXKX1vk", + "76DVYMINpIw+ZMQj7FbYuSA9lQkZpYN+b1pI1Clv+QLW565hInzo4At9pjSDRW5XjCgTBQOXSq4WqjDl", + "xyZKwm43O5zGfRY5C30dPw39dp7GaY/+u8aO0d0VOlsf/uH9a3dkd/YgRvxsU5HFGLXFYQ0w1/ZJyzVA", + "0m/iO8ZqTRuhJbTXJSEJe5bxCWSIKNw+MpVFDiQZyM1KJizhhYG4vMu5DlZklr2b9p7/YycNXkmEj7+s", + "KRicsrEZpCTcCv7VDNeAWWO5jYIot8mcX6psCe/BFJntsolYQp8y475l3FpH2kwDRz3BmWNU4UCoCpuo", + "BTzIIurY1xfjqNM48ugZIXpGGmH22IbSJqzsbzMFEmqYTbFjdJtQ4esAjJY48xS7BJkqzaZ8IbLV0Cmt", + "tEhAGyYdmDOHyFyrpUhBD0wOiZiKhFlublCUGSakVczOhWEG7HMG7kqZa2GALbkWXFrjxJ2GwCGJyjKe", + "GwgDQWi2BG2cYpgUyQ1YdrD8mh2x5TeHfcZlyrhcOdE9Y1JZlqglKkQSOA64Z8ppkzfWH6jP8owLyd6d", + "vj9kwjjbQGlHmtywsXJafExKONDG3O+s55AfYLb8uvmf3zhKKLQ0VmSOHGYA1t1C+z2cMkJL/f1NWDTt", + "SIIYy7V1nBQTHGuGLF4fR85UW18I6bGGOvwWzTp3BZ1ykRW6tGFfvn//7v3o9OTi6vSHk9GHt5fvXv90", + "8v3rl+PDITuZOAvLDTJF4izdvYzLq/Y52NhPM35OZ9ZMgwMxysvC8EkG7ge8Mw/Z2O809rX0hzowAGxc", + "AcPteuzkiSpsNS4VKVISja/bBU4rgP7KsFsuLJsU6QzskI35hMtUSUjHz/0nLOEygczdfL0uzPkMmORL", + "MUMxyG/5ypnhA1yzSW/+2E6Q0ZEcGGmTvX6vXCxKUo7vopcFj2VujJg5mNQsFPYu578W0Hfm7bQg9W2K", + "3HEFc4LVDDRMQYNMII7SW5gYYWE0Vyai+35QZJmWULidgwYPT2J5pyIQEOnG+XNu55FrELfz3edn//8C", + "dGlSwl2SFWl02TWDoCYr73FlSfNTJSUktttrAnfe2ZZkwjESsVxSGKsWoNnl2Y99dpHx1a0Ws7nts4si", + "z8EC6EN3E3FzQ8pIZOIt5WeYXCqUl7lWdytyKAnDfnqzrgq+mATrJkGajzxcH90SSPMzYZJ9CSItx0Ba", + "XfO3oJpdcEGXG/xaLBaQCm4hW7FcQwKp44Nx7bDj4Hk07iZirAa+eJg1unbaL4boRqqr8Pz0hHdP67Pa", + "YssAbWz/8b13fmL3l10ceAswhs9glKgixmN0/3VzOybyHzuLMOMrp6RR+0XWBYHOnlRo+lvcU6CBm9ht", + "+ef5qj0nSKeE2JgYfZRkyjhDBr8i3hdSWIGES39UxllIRU78OUrmXM7QAEEnkygWTAPaiJCSnQEGLWhn", + "L6OmRDlhlQaWqlvJjKqvlqgiS51N7nHMZ1xIQ94xCbcsrFvfAppV4+flbywVzprTAa4sLxY5GWJ0ViUt", + "3NlRaSr5Awcnpf8d2bYypw7sKhfOyFr5pwNm5oV1RzhsWlF1UPb6vTak6n/CPaFTpLWj7exXp+M2uZUU", + "sIkhlTQqA3z46vQdTOhbBxH3sTdmlWZOlhWzua27M+EugZyIinyXLxfCVgrjVjk1YoVMLBI9yQxDCiIV", + "UzT0LIlNM+c5mGHpUPXrn1ycn3JChv/L0N8ZeJaZQ0da7oZoWAZLyPrMwbTPuJ4Zuq6hz2WEnphq7nLb", + "V3Pt6PGgPFv5S31qmjMTEvreJdn3RxkVOous4z24zq7375Pu+uCtJRrJuAbG8RIT88Luo/HaWP2i8LoV", + "HsHKM+Fj67soIvb1LuLIUxIOvY/9tu/cUXaEbbOsZFiuZ8XCzcwSBTohM50OaIbsgp4mmJLZyl1epKdH", + "z7Jd3Nfw5q9fBFv+W2KSiJen4c9v+L9rF6lKqCBNIYvuvPEWa8eVJcqKuE89QNENYkueuasqz275yrBr", + "8mxc9x4ExejrwfpeXtceCz4doCop1/GEsPZ0wOwcH7Y03Db3+Agba/h1grTd2UtdOu37PeStdbmDeiUY", + "EO6bas9Csomy8yC8c27nZvs9HtdZlxi/rMmM12q2s0LO1Iy0baURMzXrh9+HQk5V9V+3XMs+A5sMD4eP", + "oGXCRr/omK06JlOzJ9IwDSR8XvplLzWxQQx3WoFujj7LuTF4O9GqmM1ZIacis+iJR1FCb99D730do+Nd", + "Fd5j1bAB/J2RuQsH8PQF41nG0InO2trAOFsOuGZO/g7ZJZA3xOSQlI+Q0yLLmCMEsun+GLn1CoO22uhZ", + "x852eUUI6e8gtxpUtLYj/5EXU+FuhZxWhWcFubZQUlh3xZBWIfhPzy4GQTP4Kz07Dx5kuiFbrmdg+xR7", + "QAa4d3fjXSRXydyx9O1c+GgI2olKkkK7C2HE4sapot5sh2X8tR74UnPU02biul3xFHTnrKlKCFf0XW3+", + "vrtRA75vAE/mtdNF15F8OTLw6/oqb5RUVkl/iRUycbdEfL2qwEVhhkkwN/r0mdsXpOUGrMoHSB71kVEg", + "7CAyvX+gEy7Bf1APNPIcRuvU3BlReNBX0fkDbfqJakscGIv3NO+Jqc5pwkE5s3xyuGnFoAx24OwrHHHl", + "BmyK0tCQwZJLen6bC0Ok/IJeH9wHU4zjKHHieAF/I9bply6O8luwt0rf1Lxlm4VCDVl1wDaPXJHgBvVV", + "1/97egG1WoLkjkgXYDmaBB5zK0fNxOj+wq4ZeC9Eyfnrpg/Eza3w4Fx7oUTJgYEy/lmySzeNEbx16VX6", + "UBDUccK5ETLtsk/CgYbo6wz+tlhQl1djpZ/eC9chG1Ng3ojnYvyc/Yj/wU4uzoND68DJGb0EcqnSHwcz", + "kKDRxgo7Z2O4syAdIYyfMyH/Se8Cfj/lb0M2zlTCs5EPPxw/Z2ZlLCyY/wPThZQOYzxTcmZECo3tNp1q", + "ad7r96r9u5/CQj0nW2sLRd8nA6l0E1vESNlGD0GbETE4aUV8cOT55IhUxflZA9+BF1q8hcjfwDE/WJv/", + "AE43mO5DWF2sMQxGT85pJFvw3GH3lusUIw8GwlOK270TbaqwZYAFKRn2k7v6GvRS1ZygZOWxSWHZgq/Y", + "BBiXK/Yfl+/eoonUsHrWDoPx/RTxfZqJ5GbrjafAa4/7NFgSPLeFs/KWgldEiNKuiqK79xUnur8vF53O", + "i46o4DVCLD32dacbIY986TGQQWJVJHzz9PKShV/x1h+8uHhgJyAztJQ6bIJZLK75zWtm+awRe9mazWGp", + "yHPQGNZLkub7D1dX79722UmfnZ3/1GGERK3xn4QR6H92YsunznQs3GdW46NtdPq72Nxwi7Ebd4NEKZ0K", + "yW3zVO4sDoq5uIPMxN1Mqw0Tr+4/cYv47npupX6FbcLQxntOjQR/hNVWiXUDq4niOv2j5VXY2xdptZO0", + "uoHVE8qqBjIeWVK5na9B7UdYkau6sv9+9IRIACUJ8tJtsc++58mNyXni7s1xMXIPcRgEF3p/5/hAnxSG", + "vLyUHrJCMsk1GNMhXnYXlzj5ZnF5/vbiw1WfXb38z6uT9y+7hWbbIIMHSIjLRKssuwRrM0i3ygqDXzND", + "n3uJEW4ufGqrT3JlRC1XDx+VhZz1/zj5sn6yL5JmJ0lDGBx5JD+h0OnA0COLHydfRhEzgFZnd4OSVH12", + "E0UOV89E7qsZGEe1uxgGuN6qc73VY6/nXRr3EIC01jaDUMWA9wojkc06CFEGuMnDCYKs2OUkKga3xlKr", + "R1mqnRhEFFKizh/ab2gdwhtl62uxBGcIbolmZZlYAlsKuK1Cilohqu4qPC2yIHy/MuxnmLy/Oi3dIG/h", + "Rh0O2Q/+OyWz1QsM4AgSeao0zpKBMYzSHB8kXWNn+yJUO4WqQ/HIofipwmQ78bF/vGJwXzeCFdcO0B2v", + "uMk9/rok9XUn+ZBdNjzYZUid6TOjGGdWc2mQQYITeJKJnCVcIpljwJb3JJYxvBiYO662NN7LY7wDwLcH", + "J6/zdzw4eVcmr4KUY1iZrNaO+2Am/xKSvD+fP11g8iasPDq3f0YByveVKy98iYEQnawpLZ+C+bvk2p4P", + "Szumtryhx+KzGv93SI0rn61Qg5FV4cHCsUKmjB2yK7TXrF4Fwef92qlWeQ4pK6QVWXijHpUS1V3RtBZL", + "MEN2pYFbdIQLOci1mrk7bijsgpGhFtiBl7gjkWYYwDCDUcZXqrDhcnDIuGGF1JAJFOK0sp2DfJAI6oLY", + "FxnUKYMCtuta5rFl0Ea0bBNCTWLoCv1/j38vX86r0+ADT4KcMCoD98vHxfKlLvwyrL/JtUZtB8v2sHQP", + "inMp7Csusq0cHQQU5Q04G30CPmUhE7/Rfh/KLq3NfGGWrcziEDCaIsieiFdiONmPU4yFvJuuFmDnCnNo", + "S2LyATIWcvJs0vm8i5ECOIYG7Elh1Ym1PJnv4GLETWw/7fuganbiiaiWazCIhgFggIsw89LBCHdzXhhL", + "D/JZdWEgjwrm/Jshe6vYtNBUcqatLm9FlnlVWCYCegZ9DD6MQeELM25lxhKRT8uRndh5EgXWoE6frT6s", + "/jryxOxUGRGzI9NAxewWNDB8NSjyMujBZ79PiyxbocJTOhRtanJVXQdGVnxENfgeHmzZtk4V4XvetgZe", + "EjcHZ1dalHCY8RyjQMhcPm1atVjRwoBF/0IrCC24GKzmyY2bzRsNbKrBzMOtXRiWKyHtowqLL4Jib0Hx", + "9DLiIfIhMNyuF2Usqta6EjPLbwBZpZZqWvq9m/ywC1DXGDy2ye3wqarydbq/ctBCpSJhpvw2eADCY+LS", + "h0s8Bhu1dvSFi7ZyUYWXJ2KiGEr246FcRh7Xv+cG/vLtAGSiUkjZxdu/7UhiJawmKwtbLV639oYzviVF", + "cZ5msPXRPCgVkYaw2taTOWffHR8vDPu1EGA955CvVyom5GCaidncMl9dEiOjH/aO03ox/cIm62xSd309", + "NoN44nmteCrkbONdaZ2KMhoVrnU+h/182igN4EDMMw08XTmgeALCyBZnhXG897lLoVQs10JpNg4H9lOM", + "cY76Q6Kwh302LnQ27rNxyDxx/y4TRsaU1TLW4HMwHQDGtazxF2wcoUDMdcq5pqLRLFd5kSFpYJoGtyzh", + "Bh6YcN4J8i+aYisLeIp7omvZZsw8ciwIFa7Yhqg6F4UR7QwwDKWYRSqh1vBFddTiAa5vQ0YLZvTVfvOO", + "Ggn2+fOX79+PTt+9ffvy9Or83dvR+5evPly+PNu/ELLj+UghZHwhCXcmpcVMSI5+lZYs6HwccavWWD2+", + "sD/p8L3/9GqVQ+1+jCusZUfWA/59YuSPUt1Kihk0TEisS8bOfDZan70Cm8z77D9/eN9nVOmjzy7tKgMz", + "B3fZO1/wGfTZG0gF77NXyo25gjt75a56fVZj6X5VLarP3nApprjDCw1TWuOdnYMmWbdQeofKs43azjWq", + "6FcEuTGmxIMw9HTYVVUE9GE2eEdO0f4ytL6LL9Jzq/T0SHgisbmGjEcWmCHbc2s5hTItFDV2sxiTB0FU", + "gMxrmUL77LueZbRe0tiDJWQTDd1Kfk+O9zpl1Xn4Zoi1NIRMsU8HZuuhIVKY5pnuLbiMF1E518YJk1yD", + "07MkVTCZOwouYUYaqKzWJnZBH5eX98bv1xQZtdZgYYY4n9CTQkdxe//ewA0LpUzd5FienfTW315e9dnF", + "u8urjvLVythRkDlxnE1UukL94GY5uvhwVd55+u5wfMlFxicZdOgjOlqcXt+Rjsswr3QCU+WLkoRRiAY8", + "GJrKNWAjGHUBj6R6+6yQ4tcCGjXVqxeIL2r24WrWk3G/KcIqgbMmEHbTwNTbYQ8V7JtBaEhALKsL2yu3", + "6Zovr/wQyd8hxXvCaVgfn8SQKkOGJD1gPY5Gr53qi0rfQaUTvJ5Mp7fR8chK3ZFYFDMe/A1arGQi1kBC", + "iQJ3lr05f/OSaoz8oXrd76yu2HdRWN5KUUEBbDJJFmLRJWjLQ4cJS1CR9nOQOZrbRdZn7SZfX25tn706", + "eaTGPmGajpt/dK5aev67H/usbOd2eF+tV9bfDoy4Ub1d8BmcqcUpJdq+VjzdwSN59u5NY0Co8OXIx004", + "TMsZcS5UeQ+r6NW5zy9aq1NrYdhmqhYjn0aN/rzH9+NtRs1j+/HSfFQCKyLAKK5gEQoJMXphpXxTIVl4", + "XeXWV2FZI+WpA0IfCx1bsUS8BrIPsYYUGHDg7DJEFVZwOhyyDwbY2BqqrHLbfN+NhDi3y+g3TraVaV9j", + "OO6u6ZsUvNuRvvnMg8UbpejdxODw6iXKgl4ClkIJM83FFO9l1UV5KUzBsV/YRGTCrobsJU/mjQEUf0H3", + "0mcDv6o7tP7yqvUHyIJmCPdTyAFPlQ7X20tEFovCM1mDRg5OX18eehIts2UuQOOpZQLsSiwA25OdXJw/", + "WKm0d/xFn+xGQw5gfwQFPYlv08e8rEPvrJWz0iBMkFav1gJ1DnzB3WMU+w3xyHLQWHLxMJrhUgflKAXL", + "RWb2T+kJbFEDHOPWajEpLJgtHIRHWuehOU9HGhJnMwiZF3YzHTeA5OsmJJDS0xmWRcJJgssLQx76vsGM", + "UxzC8/np68s4naP6jmQB1dc1idLhniKMx9WBs3wQEiHu8PXlYVwVr9GkvyjtWWkx1HzAv1fFjxsgKgs7", + "RrOuRawVZBR5FZPHqHV7jlU71Lt1YL+XKttpB6MkybeK/ddcz9wl1Ztd0yJjF1y468Pr04s/UO77rX6R", + "91vkfZI/iZivg/+RxXuW5PcUp542K9IkynyoOPVVFqJSRKTV9IGPX59eVDWuxDT44TqLto7iQsPdaMqO", + "va15d0rBlCrtFn1n794w90FE+tXW6er/IlPQHdt+jz/uuvEXXvFSOzbyivmKB2Xg/JVYCDkbnGSZuh3Q", + "U1A85VT8Bt0VybgG3rEhKjjBzK8Fb8r1au5tz6j1GTHoyh2BKc2WIgUVfuqogPq0yqu+NSe4CHtPoL9w", + "oZiRdW/ltV1jKb799lzdiNuOriwMfyQXV7mdL2ppi1pS/GkusA0EfObOK7T5KrL8s7iu3papN7txXr36", + "t+/b1eZD5Pu3oYfo4ZCdcq0FYF3ssgjulBodCYnSZ4JlZC3zpaD7DLuShJLVdU9Vu1j7g7m8BYAvvL6Z", + "1yv4PwXHx5CxX7bC/bRs1fEWv9i3Iv9buGWbq/KzsqNteSveUpifWsxvsBp8w/i1I1Ff3An9vVaK/oUP", + "/6YdRIrydzRC3rvi/qPV1f9jy+VXNGDVo9W2p2iXmiVUUdHOrLD5XSH0zcOolI5XprK0f8vrzOZ8CdSe", + "CPVV+bRsmrTTeFooGxgLw2rT04sDlvrGuDB2LlPInXVKNYPrqRwvGGdGyFkGzH1BKZ70XJ4qoPZ3E9R5", + "4qE97r48R+wr15/ySeKKT97lIDc8kkm4LQ0OyyfYl5zkgoOlwsFka/giCiGL5krRH5CGkT5pnDmkMC8T", + "Qg15oxSIMFUejq/457YQ2sQY1SjptS3nxtsvzWybmiFTUjfSnLPzYpk4Q3aqpCkWoN39jhKNWnYT9lkI", + "tfXnWK3BYjEhYZ3txNHTLXj2GFk764j7YiRtZibLJyMi1adnonvYSLi1uCVztdbbxltIjhcxqNyzIBK1", + "kkDRwHK1r9Kv2l/EevVIuM1W5VJ88iSWgBU2i7hHKPo88zLEfVNaiSgY4puJmhVhqpprqXuONl1stCk2", + "kMgl6KVI4FRzM98gaBdc8hmkWOFQJMDgTlisWgZ3OebcZyunw50VgVTlS0hTdJSjJm6VHlQpBmV/Y5Yq", + "FHG+BU1d/P3f//1/KH6xWgXXNdTvGfTChwXi1XIwE0sYFLkvPkn9lFL1QGkWAc8XcdYpzjx1jBIC12OL", + "sy5k7F9LETfYrKTY2ntZRJG9vBMW4wap/7KYOaJzGhur9t45xVxmGhYyBZ1hq6jgtyHO0WXRomTOpYQM", + "jQWkbsojILYi4WZXfXIQiSkkq8TZvXNuyKNLOw/vkExIsiUO0KAv8ywO6aHj/Aw3qiFXWGAtwgs4c6xM", + "4w5LD9kYWa/Ix2wBXJrQhB0PngoHF7KiBEa2amefoO3E2Rx4Zuersk8T1nkZsrH/7zAhZ7mGpVCFyVbl", + "mMYKTRE0nvEljOIbCpgoq+kwJ0tCtZiygA912aaqjFY7XL5gsqpM1UUoVKHKXYyqS3tAK5VYNGoBdl6r", + "UGPKC0vJQATOXr/n4dDr9/yJop2n8uhV//ysLNxJewwgGLKTSZUqE4ONW4wV+XrZriiYhDNPWaakG1rW", + "3+FUR/fi/Kwj0tYDUPLoe4NWM80XzbY1/hgBnr5BGhYJFMXCGdeLwlrQ7l/UyGtAD0kDnovxLsXS6nvq", + "e67YJIqw29g7tfhRZNmG+kmvhSzuGG2JvXv3ZnAjsgzrm6H2wnoSJRKELBuV/fRmyC7rXYvHRyksj24W", + "ZjYONxRHZlxW7IBTl6LIr+k1xQIWSq9KhNIlPURxeO+1fxxCf5CfE0uxcluKO1PkDlAmLkv20atr8Pui", + "VrvVKgJrpNRi5HD86Go1jov9tarbXEupNnfeXZk4UdJYzUWMj36eNykaEpHS/Tsw1JCNpZIQhP4sUxOe", + "rdP8CzZewCKpKZdkplWRhy8R5UgSc2FfsHGSFwbsmB3hOKVXo1xlIlnRhf3thzcnR/SHQarFEiRyYCVk", + "lfRbNkxl+C4+55J9Nzz2b0CpSMvK/76phC4SarQyVmqBR3s+ZpmQ0FQT7rCYOLBInIagfdIfql12tCpc", + "jKYaYHQziXRt0AChj6IHiZDsR/F96HpRf9h3m+uzFDRmyJXBFWM3+/O3Y89fQtZQ95Vhb2AxOJdTxdJi", + "kQ/ZiTHFAhwmvsV1qDyD+A2G7Cw4P0ICioYk42KBNYsTZ0aEavNmwbPMv7Vh3DNnmbsCIdZGVlmejW4m", + "Y6y4bKyjUYd+gjgd1qHcLYXmG5tznVIDISz857HpZUcgwjruOGUE487KAxpf4qveqrDG4/WtRaSV++XB", + "qHiL8DTs/ckboqIHoONpoLDNfvEqLZgv8Tnoxw5z4lQtFvHZGMZDkGXcUpoHC37Hnn3nbHVt+jUF0fis", + "IzvMmChK34NB654ZsKRh4rvyaD4wBe6bSyUH2pg+m4oM6F9ooc4XsHD/eThkV87W9Dnj+XxlRFJJv7qR", + "58i8wK7MHUTU1eIlH1lubkyMTnNWmQoTrG2JpxwYsAM8pV9qoRa1doBEsYZg76Ykv37L5mmQ6vjKbYHu", + "CWPmXw1eLnK72kSU3hnlvj3laNJzy77DUBXs8a3YRBX43EFaC4kdibVsL7+vNeP2iRzO785pju/WW9Jb", + "LWYz0KNtDOC/q10od2FF36FIpk6SjU8vPjxnb5097v7HMcTzsU/GrOmWCN7DHndmsJLQ5soA41mmKJmy", + "fK+pFWPw+7aKCblUN2T2VhbykL2bWn9Jwfclbti4vpMxO6hN45molugI+hADBRIuWSqmU9D1LnE4KKFt", + "+p8dTJcisWIxZG924f9o7/12Sbo67EjelSJiV5MMCWo/a+ykfDDzGKHYrG1chVpgzTbbHe8PkZvbOGGj", + "Dthd6Da16LpU2qH1FCHRY3Q7Lmtu5E1+7XqGMjmT3cXLu0VbLcbrzwe9fm/CkxtnyMp05P8SrrO3St+A", + "dn+Ycw1p9d9YrSRqIYZdhwLcp9zCTDlpearkVMzu422jKVa1qt6+oyCmtWGrZ3ddCC95XZcEnou947ra", + "51j5U6znqf9Y8xsMMr4CzXhixVLYFeECPRGFsWoBmqHFb54zIUmBnFycs4RnmQnvgmtuiNAdyGnvsr/4", + "YAEpecQdRJI5Z0ZlS6gVRXckkmt1t8KB2Vr7lMpJ6P1QHFvQTTUvrxPVSG//Ud7LQMlwQxuzgCMn1D72", + "e76j/BOC+9T3rFeFzQvLDjI167NbrmWf6qsd4q6dAClmc8vgLoHcx51gYaCyI+ET7vEDZcCUSwUQH2Bj", + "YNNnN7BK1a10NxHsdXeIm/PBC0+4sXpZ0aMyLz30oR9SRNDsKdF3gYG1bdo7qEeBUG2kVtbp69MLB6SP", + "G+Rlxx72kzs0KOSEoass6MS6FPJEvx73L53RmMacEUANCtCXHwQb/pt6pA7ZuWTNCkhqIawPhhHGX4xT", + "mPIisyQudAHsoJzMr31IM51cnf6wZa4DpZnxkTHYYs0ZxQRXNv794/gQ4xaYVAOVvyAxFtbSYLmQBj3h", + "6Nd3upJc8VfK74QpzVJhqONbNXQpOO2uz1aqYIuCqtaluIW7PBOJsGzszjZ2M4wRTeOG0VS61HYih/uQ", + "QdV2KYkQhFc4Q9bQUm3dNGTv6H5bfoHwtgFRzwmBVpUjhR2yy/oHuDf3BaXO0Rfvgqle6+rtDDWhIVvV", + "p+NZFtYWYGhqjo8NS6j9gPOvrZhkwOmZ3MZhEfPk+h3t+lDeaStEEXvq3faBReGlk/d7IvYV9WUgk3aC", + "bcNSFh4EygMCPiJ7D5Lxfjd60aLKqOucvwBjvOhct1Hjj2XlaiOaesFzQ/Xf8c3kyN3cSc96f+FRqf+P", + "cq3cz0epMHnGV8wpjhdldKGfEEvDOUb1QXcOFdwKn6xfb5vU3AmaqfWZohZfvFvNu9y/5FewbPenGTZe", + "rlQ+CvCnWiba1v/gOzohAByo+70SCO4//PnDh6JYjKYZnxnCjwPRdrd3OHNAYcwoP3Wa+40qDPjadXtG", + "w0wKa2Ppxzglo1/pUktWA3oka3DKYGrdtUHM5uixFWmaBRueXgduuU6jeEKjo6P8z5W/PeA3zBtG1arO", + "SOn1e/iyiJ9EF5irLB3dwMrEjpdSzI372Z3PfVtv30Gz1twm6wE4LReILBYjsqNoOdSHvefP2pz+FjMX", + "8GokFuAZK4dgkPt11++EkQ7I/8kSpXSKD2/l2yJCLHQojs4UKbz1X/eZqUWudz03dQeR5thW36cz7kmj", + "8RpGp169J2FyKmDkY3+3Rxi5SaObpVYt+qSyxu/hqPBtDLWnXdKVkBQWPT+577bNSdQP2dUc2JhKlZMN", + "RI2evYi/ltUsOeWvkdd0va8kjXZAQDsIb0c0NueaL8CCNsNr+fKOJzZbMSXL32lko7IX3uHREJqgw2Ip", + "0vhLKbHywsmMbTp2XWB97PdSzWe7DT/TfNYevVBL2G30G7WE9mh8GXFiYtvgC/fhj7CqjaVb0raB1HW9", + "PgzsKCm0UVstkkuwp/hhfXQGpOA2DnQfeRKuPcGuv+gHP80ahTX0cA2/DXjTzKGSdAXKEjQN3DZOHg4S", + "k9zVpFuO6fTEFdzZEjxtLo9X1ez3TjVwC2dYWFXp1f2U50KlsMHSSMPszH3IDlSCr2B4yj7DmI9//+67", + "wyE7q12e/v2779CI49aCdtP9P/84Hvz7L79/0//247/EU2XsPBLiODEqc9Km2kRou5/g0VuLHA3/dbvz", + "160UA+YZZGDhgtv5/eC45Qhh4yku8/gbfw8J6r7Z/XYfc/Ser4UQ67BI7STsJMvnXBYL0CJxt7D5Kg99", + "a2v454PfTgZ/Px78dfDLv/3LblnXZ2R+7njHbJVcATTmOhVuMO3puyrpvCO/HtuWjTS3sH1K/zXT2CRN", + "sh9+Ywe+sbAssoyJKT4KpWAhwWeww+iityKNEVR7Nfxs4/6joG1roKcxuJ3Y7DC2SyObrO6YAE3BXT7q", + "duhx21Q5c5+s1RCagL0FkGEjztD2EYRcW0+9Tv4znqkyHcpiIupCSLFwGz2O4WRjCzIfeI8BBFW37vbe", + "Qnw8+RQIQm4vizIE0CyUsvP/iVWyyR+BjpHCqgW3InEWtzvDhBtIMYIVF0T5koGc+XPwOzrHs+Pj4+Pa", + "ub6LHuwhtwx3hL0uGXFJ+U5jFQSWCYNm5T/u+mz1S92kz7nQpsRdqJV6OxcZbWKGr9VvnKnnbUfGLcuA", + "G8u+pk6D+IBR7rS95XooSPlQ/DUCr/qP9mk2/ki4bNCww2vEp83mxYLLQSZugH0Pvwms6KaXUFEzYviW", + "r+ggTEhjgWNF3kxI4N4pnquMPEjsZ3xZdauhk8CMctAjAzOkNGIHyEfIZKOFf6KYSdWsRFELsGt83jjS", + "d3vyZZlSj/taw+A57WKdG7by59o5m7fY4+5rbLklpC3aF5Yb8/DyjzQoJro3yN7Q9tizxl6fbX++7FLu", + "pRtuV4dYa+JNbpeXdJe7yPjqFqXwrsog3m+gdjuspsQg/UhkYdrhL6HaxUf/wZec/klR/tXcdM3EP865", + "YRy7nbrfv8r5DL7qs698nt1XdLv8yrtNv2JLrrG5vr86LvIMnrPrHr/lwuLr7nCmrDr4am5tbp4fHQF9", + "M0zU4qvDF0yDLbRktc8xs+jg8MV1LxZ0QCVTKFU3adDhX9bo8A1Ja39GvML4JpYhSDWY10wY9pfjhoT/", + "piHft9MaAn9HejC44T3JITTIaFFBdbr1l51A5a3wXezp5EnY2U0VfHwPrXhNbr/p9XsixSMSJqsICNzc", + "ASXBHZIYSUFH9nMZwneo2VQZuVE/WMSTm6pYyb9yMv/YuuNs1N530xsY1KENaaMjcPyZpxF07xeIEcgr", + "kcG5nKp1eSTMKBV6865Qf+GjV3md62icojpLcDlVvkCDxAcxhYoqZQR3yi0MfKW99UjbqNxxx6Lb7URY", + "nx/XZ9e9VN/e6YH7v+ueu9hc9wb6dqAH7v+ue/GImXhczvfcQCN5YirCE946JHa+FQebdZ1IxG8wmqws", + "ROjk0gfc4M9DX+0rbEOA2SHWJsRNcbTra4v1Ax3UcOiB3kVOFFTVkazxqnyjwQytGXQ139qF/Ph0SumK", + "O9PhfXFZLnVfpO5HJXG3mE99WOVQ94Gdvn95cvWy1+/9/P4c//fs5euX+I/3L9+evHm5QxoDZTB0GizY", + "p6D9BtmB3zPh/ivk3BTSV4otiweUb6PQ7FDu5TYFB1Fptyrglpch+zxjlt8pqRar55iUQymsvtdSNbux", + "GvjCB0iOU275GB/YlF6gZaFkiWu0IdxWJpCpW3ZAHm7aErm+/bv+uBsO4z7TMOM6zZzloqZuYZYXoUm7", + "sEN2yrMM9KD6owcAPu+/u7xiR+Xuj2oRRlSWwCdthGoIwhBkXzADwMatvZT3UWw9ZeY8hyH7iWciLQv3", + "JriZEI1rGJ9xd/egqQOAQ6hz4ssefGVCa4bwIoo2UlphnBT+gue5oPbEPBcjt9aWh+2TXDjwEEn1ez5E", + "a4QhWqOg/DfOcEpDLt0IslbKydJ85Dt5b5sjzU/pw/rYqpH49uFn5bflDBR9NfLW0OYJ6Fu0kNrjMzXb", + "bfRrNQtjawFV9AC4ZYbz6nt8DInNg88Ru87yI6xic5AHvixosvN09FzRqLvT72ViCaOlgNsdkfxaLOEn", + "AbctTFfT7IzvMNM60kMb8Wqqrcf0fcHPaiPaswkpbOgzu9Nk51LYegPlaqqqt/0+85Wd/7dMuvd863PV", + "GrDfp8V9r9/sUb1Tp6+qX3m/q73vPfso1yYMHS/37ibamMO32Nq/gVmv39ny5J7NZcKMrcYJO3cVaHLz", + "ev38/dsTlNMk+R5FrstRiqf7VCEN42oV+Paubrg+xx5w7KhI1l+rebNvOaFaeHuoM7F3DQ83Ryupdt98", + "ZZ965m4Gq7dovZOB+rHfUxJ2D7lt68eP/X2G1ZTyjgNjPLzv0Drn7jc2IoT2m6CShjuOi9H1HkPjwmWP", + "CSqO3GNQi+L3Wa4tdfYZG2TO/uvVWfxeiLnPDHHDcP/BpT24/9CI7bfjJB0Wwn6j1+2y/cavmTr3HH4P", + "fu4wBncc3biZ7SoyW/eo3Ye1TekdR0Zt+j3H3nPprnvnjsOj6u6+BbOoivRrYSw62SIOKa35yl3/191b", + "QpK3FZNvKGl3uGtybulCjrwLl+o2UussU7N2vmStMejGiPF2F4dZ+aJg4c52Vt3vqCp+JRa+B025I+rR", + "QzmBu/qiO57p6kvHvGsYYHHho1nfl7Z92x2/a5htCGK7f3ht1ww7h9WuRTPuF4nyiBEZGN73wFiMVBjL", + "ZQKNB7rvnjoCw+15rwiMh4cleC96FYPg/smlbUEx7ljfRp5ViEegMGbVvch015n2Itf7xwimYOxoW6wj", + "GIudiJUsX3i2hQr2e0Yn2yamokM7z9l+FwwL9GuniEHo3U1dLu3xcPw3KojL3v1YdvVdl+vqZivVnlOh", + "azDh5XO4/dVT3UTPcsFtMvdhiPfDeFcc4ll3/GEpKL7+9nj/aMSzzijEITufUqoipH1WUB0xYHMxm4Ox", + "VZ1CGlK1qEby8UrWvyP95bj/zXH/6+/6z45/iW8RQesdatvwNfVRShqmBeXHacC6ACiCq+xqpasA1CMN", + "eExhKCEc4pLGZ3tVOU/rQa7V6lRML2TC+XJ71fnDGyRm9BlKKWQ85TnFPEu4DVWSqlANyvhzsJwDT6dF", + "1qe8xPCXrIM8O8M/zzrDPkuy+ebr492CQNu5APfTvFsCNIPWDWqL0vBXhqIy242JaiTq0H3cp2+5Bmax", + "VMz2GLANirQMal9s06g3sKJqU8w44HiNvruCja//2oc2utnNajFRVJkAF/Ltfd0SoRz4BBivfctMkVeV", + "ke5SZZXKruWBAWD/+ewZnmW1YClMsaSvkuZwyHygU9Uy/rr3HsNfrnt9dt1DnwT989TqjP51kvk/vfru", + "uje8pvBGioAThuIzE9wgz4xyu0zUYuJVlvE5ATTfv9kQOYH/hav92xWf4LR7ALQlrRG6UXlN9Uhe3kHy", + "aLFs3B1vgfGSK+nkiMTCp5FKGHrWDIv8R6S+As3E9awo+4/tTlXcjLRSzaDG+DGKZiVRrMDmhrJci6XI", + "YAYdYoebUeGTjDdPGdr7uK/dVLLIUHsEGb+eKUlnj0QqIKBDUruZQ5aVIHe6oIh3V0luY5UAlMZyp9Vl", + "9YDXIysO/Yz+rZoWofZ17QNst7lALrvJ6/dYPLvH2e8f2wh7KZdCK4kXjzJOEcte+nYI8Wo7FeWvxRru", + "F17YjcDuKEJC51Y2fFAIIa8zXYmw8hyR1lGb7oMvy/N3XQbjlYzgTthRPGb1ItRyCgWpO8rrYkThaPKX", + "b+MBRX/5dgDSDU8ZfcomxXTa0T+HIgp3nUwVtnuyj93Y+1FU6X77oe+SynEj9cqyJ0eNepsoo+rdDaHW", + "u3r5/k1v87z1sCb/+Y/nr1/3+r3zt1e9fu+HDxfbo5n82huI+D2aovfVJlRtj11c/ddgwpObZtnEdkx0", + "ZuJ9qcqq+onKigU1edoU79vvaXW7bS73yZ5B6jhrnza6AWKXOb+VdYDtVOwmorrXO/756nUwsna1XQue", + "+K8ZZ7mBIlWD8vQHF1f/ddgWrFWtjqq+0BJII3WoyzjSQtOINuLoQlM/BIZNtVMb9kDp2krus/sv8zHa", + "a7CJ13vI8/Oaw5hPnEDizLjZNvFDtPbcu8sSWV2VzEN1v9jwS6z2NSg7uUX6ltT2U/pxi0KkcUGM7RNH", + "3Mb9xFR0eq2uux+2h6u4k9Ust8W+XZlPayWFCkNatlsq5cUoT2JNy4wVC4zbPL34wAr0p+egE5CWzyDa", + "s3eDGq36OYhm9cI5N76vyS42CpXw7Yh8rnYcCqKGeqy0+zIoukODR90tFxVObSPStuoVQNuP66JuxKZC", + "3k/pnHHLnSS71YIcoC3So6QDIfPCxjtv7mRYpPVVtpfyL+f9ZeuZH2Qvuu34BE/jpls/ofvCguwikioj", + "DD9g/vNhb1eXij+KBl5Fte9jO12+LOveavCdyuutS3y2iNJrhd4eis3yYa0iFneKqAkK8Xe6180trYWf", + "O1aIpvruJBpKQUqTC8OuceB1r4tl3f4jWoAc4T7sW9X6DyTzQt40Kyhh8k6ZErQjE1PcNuL/YX6IiUpX", + "1A6PpgzV5AgA0nN3O5Q9IsZ9lbSmlU3hVmt2NqUO1Ovw1UpZYQXKqpxiP9Q7rRd/7GNV0BDOFc/t9t0M", + "uirD0Qk9Jwwf3INiS4rE5o6lu5bjoBIMoOMpUlMhMZZ/F2uhqrMQRnXZClvdLmQGrf/ZlAUjar83sn13", + "tm2q3fpB99xsC85oc9X3GYN5FarzHma7lDra7XnmB3qWKctezLyvYEORiA6H/c/oqN9noh0f72mur4xv", + "BDx1QlJLeNBz/h5zRl9MAxT6AbDbUHafhwddInpLvaImYUQldbOq0b6PuZnlo7vN7x8/KC1+UxJr5uBa", + "jC9UIe2QURSHu1/i3w3DTNk+kzDjjb87PMQVHO1gS4mMn9yOkx3WT9WtjCxf5PHFHxKwUNZV2t33vY0r", + "qi7+ZfGn5lL7M8XeU+4cRbBWEWtPqSXSFOSWHGCKdqiekvygrU/h/ruObb8SGVyAXggsdG3ut39s8BP3", + "T1HvH0qv1OxvjUv+vnm8kVJVf/n228P9KlOpWxl7DnF7xZ/wASTs90PHfnfJ+aT0w7yCLb160gMbvjyn", + "960atSEHt15ibc8i8rwwUM/Ip3LOOSSO99PSxb6nj77+YIy11WIu+nrtg0Zs1fFWpqwvHgWIM2FemZ+5", + "TR61EFhZpQ1vzVgwMV69wDGuWMJ292bJ7X4+Vo7NVjuEvHQG8CAEHlhODFtsxwNU3le2bfjIoXiaO45d", + "gtYiBRPq8nsIHNZx/vXxNl9p1HMY3v4jPr+aAUsF/B+pqBluOhD0ubwkAu5+n6v2UX+fCnGKm6GzESAL", + "fofJ9uI3OJdvvu/eAQb7hp4cb77fESPtGlPPdgxAubQqfyihKZ2Am2c7v5wvfF+HbIW9RPFVXBWWzTRP", + "YFpkZQNkn0y+wDAqdC0JiXEAWhe5hZQtRQoKgRV/Ftinmh5xsNvQE5bSq7K+5RIyle8bm3eFFctoKCvd", + "5xY7vdXKi7BWxnqkhn9wHG0siNmsG4DFRn/t9L0OFkoqq6RIymAdRk7naqc80cqYsoFvvccW0fWQfTC+", + "U91rbuwAVx6cn/lotMIHfV9evgx+I+8uE4Yqi1FAy1ojyT2e19wZg2ftl4047AqSbxVMoFJJt0LDIIMl", + "ZN6pgkn+WDgprxVT8JhjIFM8D7VECaFYsnX6ITvRE2E116HugbezqAetL6JQlQzQwHhKkw3Zq7W2Mpsq", + "O/RjJRlwx6AH6LwhsmGpSjCoBsoOh2PvDfpXX+vgqPWXM5y3FjDVZ+sFHaKlghvutM/Dd1Yh5D8u370t", + "XWcxaGfCeChtLlNBVXvIGd2GfrNicwyuhBbfu+aJOshegg004/VT6STubChrneT2/djKprK795TFBrKN", + "lrKNbrKNOrj+IqZDF1ranQ9w3LPx7NP6LkvcX4Z3rnu8KHY1s4g0jcoz0eFc/Jln2SDJVHJDIKtu4TVg", + "NhuHOPz6KSlLw4bifNWOhAnlUahxQrp71aikLEC7V98N323j3srL66eMG7umV6tOrBoM2KDfmmChW2O8", + "wfEeNyZ/fDpHlHZa9av3dqM9rMrrDayM1eoGTLQyYzT2IV498l5ZMSFcr9pHyAqqZcc4SXSHbTUzvhpe", + "y7O1TkOhn+Ii5EMdpaFG7yF1l3FyK4STX0sf/+tEgFuLeuxKpsI1p7ZeA1LsAP/2P48dXHzSzuHwWtaq", + "hWILAge1VU5a4lbpFBuHpvRC5gNKy5MLaTUfuK9oQXMtnRUgORVhQvVGP+e8MA5PzjChvfmWxCbUHo2i", + "LtqfqN/RU8GRIsIVi8KTMpgrDFqmdgYdRbTUyDFMAptpEbsSzblT185yX+WKCflP6qiLmRMv2EIYy2+A", + "zB7Uk2hRIMwmPLkxOU+gIgJ2PGTvZLbyIszEIMAOjMhA2mzVgNO1rD5D2jgkUJU3s+PhsyjVh6CMXftJ", + "/KyFhbIDxv0YfTO2GuEKoehbWPC+jTA+Ylc6eo3Dap1lS0F2jj0A2cnFea/fW4I2tJ3j4bPhMfr9cpDY", + "27D3zfB4+I0veYYHOQrZJEfUDYd8PknE6fMG9AwwMwS/JBKAO2HwSV9JMH1W5E75sNakkXyUpeBlr3+l", + "0z4xGZYjLaQVGUKu/PoMlldKZYZd99Dck0LOrnuYtUpdpw1TE7SZUjaBqdKhLia6QXziFBKTwyF5MFJ0", + "+9lkHlZ55bsB+Uo136t05RvHlx1SqiTdo38acjKSxoy8kAZotqyLcCSCoVVsgWD1dRr/cd0bDG6EMjeU", + "tDAY+L5og1leXPd+Obx/ngFtKE5W1XeOPynVCHPWcJ2vj48j/mncP+E7xXtSeTSP7Ha1zo/93rc0U8zy", + "KFc8+p4HnqR6wR/7ve92GYdFEyTP/CisL7pYcHex6X0guiy3mPFCJnOPBLd5v2ccVlFv2UtqG1cUBvQg", + "9GOplgEsYq2FAUZ9uVjlgioDHia8/HnoqKp/LbeyC9ufW67lvuxyChrrjgcosAWXfEbXSd/guNUGFKmY", + "vQxtty59f7v+tcQGowMsTA1pOSOdo5w/kCH6Mk/PLo5CbrKSh6h/Js6ShvRaor8iwHIrZ19ULcHuy9xx", + "1RCzqHZB/pD9GDLB/E+SL8BcywOfb+S16alSNwKMh+N1j1qWYuFf/6IyL2egvw6v5SUAC2WfqSdatZPh", + "TKlZBiVhH9FLR5ktGf5OIPVFo935v+dGJCeFnb9bgv7B2vxl6F9JMIhuGB1F7mPzIZ9pnoIpR3ml+obf", + "+doVQklzAfrC0Unv+Tdf93sXKi9yc5Jl6hbSV0p/0JnBN731kta9Xz4+llwLtPKnFW1tsnNn6ZZwRZ4p", + "ng6qTnkDLtNB+NaJPWUihs4HHEbFRDVbOAlSTsF+EznjOpmLpeNwuLPYps7OYcEKmYJmR3O1gCMSIVWn", + "QnN0XRwff5M4VsB/Qf9auvugdjJuUV+B5LaQ9zA0Ssl5Lf9AQ4PgVQpGcyLT9x7Gm2TSosisyLHDo9KL", + "QfCVddkctX6Hnema1TfO+CD0I0wwQYDbRu2FXdq0v1KZwym+GlvF8own4Et/B3Tth/XWA8HJ4O988Nvx", + "4K/D0eCX35/1v/7uu/jj9m8iH2Ebx7Ut/r0iyNBMw8ceFjKnTJaKfcpdH2CftZBquuBSTMFYVNGHdS/E", + "REjHidus+nJ7vhZz7Gay0YCrYfd+VtyzWDxqSQ1ECpD2I9KOuKZkDuwWytNPLffWRFCJzRqRH3DjBJI5", + "rAvB8oheGvq79NEk2HhxqfcyZNFKploNXlrdBQ09svnWg6F1+5Cd+F9R81MUjjNnyFtmBc+yle8gMldZ", + "2W75LskK44jXmT99ZhSTimGDfQp9Z6WwMSzhknwUGfAlYHeIENRgrMpNcCJMhTbW1/4PjQvLPt+irDpB", + "3srQkJCasl7LUJ66MPjUiB1j556rUqD8HXcvrPyAmJpB5VTcajewog6RHlzXMrxf5nzlZvHPCkyrQqYD", + "q0XOnOkoE4ogBkwvl6lYirTgmZ8mJnm/R0Ow2UHy/mbgRp/p+kpVE7z7GSM4ZUfzg0/JeyUjULfMKAPU", + "abrFZq3mlIHZmoir2lI+Eb4ifS/viSbqFBa6ega2/qQYuhSLIqN0QeK6et/euCNxDUfkrjpyor4bTe+B", + "p6c111YMWo+FrmbLWsRW6+5Vdp71S6KeWuObB0PXHZo8y2WeyZqXrwuc6BvshmfTOflEpB/3gN6X/NHr", + "6XOLqCF8wMJnI7B+JodscKbvgK+yGWwcTWXQ6xNhaL3N7M7IeZT1a4WvYnxG8bhLERoilLflzwbjP4jU", + "l+BQt/Xqfk00N9scx60+rCyEVgtGfgeBSv0Y++UjlbPceKip55bVll6FMPRAtns0zsQytMEjwzQDbgBt", + "q3p3oS0NBGMWT9kO84lIc73h8z3lhpvoM1GXuJWqbiKhiSMeWhQzA0sEMyr7sHcKib+BbdS4fEr1GC+m", + "GeddjDqgk5aHeAwo/g1sI7DBWx4kLMJKuxgfzf7hceCWtTafiMzXO5M/yDr0UHAn+7Sk/iaUkGxgJ2jF", + "MuK9kjRmF4w1erZvkKO+Tl+1Dj7jo8ysvfeX4fbkJ6/yPmrFxq5lrIQYhYhhmatcwxwk3ZvXa5X1mQG4", + "lm4z8XpjjNvKjT4TdjjVACmYG6vyodKzozv3/3KtrDq6e/aM/pFnXMgjmiyF6XBO8tyHc82VVNrUAz98", + "LGM4r7tR+2DyxIMC0waMd6ERFlQaffHwBfCeiB3Weu3fkxsQoUgtn5O1QDq+7ktCutyB8OsNW7pE1RW/", + "gSqF76ksxrVMxI8eRxs1DoalHuWUOVuttN27uaZYqg1QrOsnRegpz/FFkrMKQSEIbQs6VZZ1CzHKsWRL", + "n4eYrZz1dqQcb4fcSPc3W7PxapK0aS02/HyNKo7eDGwkOfqmxpJlaoYpkFYkN4YdSGV9Ai65OGsUxCYw", + "50vhSJqv2JLr1QtmC/TS+R7ugYFDzNRE2XntKPTcGHIuMUPT+y79U3e/Hq0aQn7wpafh0jwo50BTuFrg", + "kOI+0ItEwUIhsjuIwnGIDSMHxmCgIQdu2Vs2GFDQ1TGjFwQyyOkNYRyTkJch1fGJ2K+WfHtf6ejJ6zPx", + "IdFmKluB0MOts4z3sOZC0G+HcPQBl0+El3Y854OcHBRE+NloLXc2cmpswoKPEe6WaVUl2fDcyNz/ozDk", + "VTs8GaVW+URkLHcGmlV5jqkVCbADCkjoX0v/Jlu9xvSd4MC0LP8c16/ZfL4YsBG/CTk79LfmciFRlppi", + "cMcTm62uJS7XeJnSwFMhnS53t2d3H8co6rDGmAooFzob43pe7HA2AWMHMJ0qba9l1Y2qLJscZg2vFG5m", + "NNTcxYbPgFF6wvdONjokhBaWesEzDDW16lqOgzk59uX3uVwhpNlKFSxVGAItwe34JLT6dyaJtwUxPsN9", + "je+SE2C+oM7wGt8ZMHCmiSvq/K4LWda7xWer57X4mzpuPAb69LzeR+NYtjE2jKJEyWxF2PeqD2RKgbFl", + "Cg7FrF9Lq7k0wbx9zsSUcXza0VX4j9s3Pja5DXKdObVYMR0zIgUG2JY25LUtuJCOHnBtCgROwNOq+5NU", + "cvD13Z1/78q1yvnMKeThtbzQMEXT2oFniV3yc46JnOMquuBfx5QKdORhNMb3PB/dSmyTQXhdHFgtZjNw", + "dtK1JBwQJwmJ+PR5mVX4fkxZBSiflvz7iIECFBY0qoe3teI7rl4N/ofPvWnGLrEFz9n//d//h2GMt4EF", + "l1YkWEL34uTq9Ae2Hj0Xr3jrvxp1BErWdkBv3Gz8+zUFMV73ntfjJH/5ON5xQzg6uhuP1l22sXBCAy2T", + "+D1pvcr+mB1gJZEjqiNyBDYZHg4ZGlxUbToEVK8TEIWUm354n8Vs1jJBpC2NRSWKG2FLDU5tMmm0INaG", + "OJKX9TAfg17IsPvEaaykwIIb1RRDjAyhY1SZARvjjg6H24NQHhwi8vTxGxgz7oaMvOxch6blevibsbHo", + "FEz7AoPgHTdiZzDY1CcleuHsRYEZMi/OQvyVr8SA5bJ9e6MqcNAPdv/PHNXapqMFbyBz4w/wuZ1C7djY", + "h/kd0Sr4sD8+pHTTsYNbPqpYYkxaAUUkodvHM4TD2jkv42uM03f4wa3meQ5rbdy3ostXeXLKPcLG71+X", + "rz9evYNX7pUU3qi+S19Qn2UgZ+SfTzjxmmVfH3/7P6jKXr9iPYfABIN9KYwCZYRHAO1ikkFHVeQmLDcY", + "bVWCVYAgvh5UYykjW4ucHitbNFlSxYHTkWXBHJ9JhJXR4Y44cmtq9mf1RNWwhLy8fFGZmyUVuJkzaL9d", + "DR9i2H97/Nft49wGM5GsXQce57G8bT2E60MnnAANLve/KMvLmO6U5XOOIK7fPE7QnqFre1oaNHiV99m5", + "TUs0zwqzBvtQyOqopn3LKPtIOLfXqk/l4Iy0x/mDKdqvHpIt15H1wb+yhrtSA8ifjGIfHLvccRxHGlNz", + "lGjgFkZlFwQkkyIWMYQflrVpnipsqLnKXqTybFMpHTrnZ+ReoJMyjjlfFfgDXlJwYnMHvJzhh0+NF1ql", + "3s7s3u/SJUroiOnDOOvb7ePeKvtKFTJ9xAdt3Dnj3XgLdvAGlL0ic/fzxhYWSvtvgCjER4kjdSudxey4", + "a/SbwIJAM7CxAlS20NIwzv5+fsHKu0DtDhGuBmWJmKqoWSCN4XoMiV//TOi/ixwj8jVfgAVtsPlBV7u/", + "knPQBrWqtPWdaRAOhbc7N+7XAlAc0J0ulHdr0kC/7sTYVi7ul72Us4frgx69HNTDGctKSEhYdQD/GenS", + "I6suQtxtgAgtXGjj9GpsugPBhrvvgeW6dgFehMdhtEPdXIcb6fpabiBs9ndjU6amU9CGGTGTYioSjqnn", + "U27o+kcLevv1WqZQ/5P7N9d0A/xN5N7hwpO5gCU2SwXbngXZKB6ZVeMqB6M/C1v1f19v/VUeFyMYhuwH", + "MZuDpv8qOwgzs+BZVndHTArLLL8Blik5Az28lgPChLHP2f9y2KYp2LM+84n/DrGQsoP/9c3x8eC742P2", + "5vsjc+gG+sIGzYHf9NmEZ1wmzpRyI48QA+zgfz37rjaWENcc+u/9gM8w5Lvjwf9oDFrb5rM+/rUc8fXx", + "4NtyRAdGatQywml6dXRUJc3Dv6q6Sx5UvX7tN9oy/sPECtLvKxU99z5ILF61/Fr/HxGNLXdeKR7R4RJq", + "N3ix2BQNZSvxXWUCSgIP1rWu5p+Lht3PJqzaqa8TFFp5tV7tf0Ky+RvYRrf50DxoDXsl2WTCWLTTTSfd", + "VE3v76dM/pyUUp06QirV9S2j2iR/QlrBbF3EPCUSrtMGtknvur6Fxt5PGBr7GFc3DEWt3B1/QjzhCbCV", + "M75ybWJmDTwtL91RXn4PPPVX7t1YGRcLJqGb/3PhZpVYsIOqZc2DbAkU/dE8rj8ZsWDWWOO5riQOAyTo", + "R7WS6Z3cvV65/umSkDpK5N+7ukatIrxPGfoTIvIS7Dqj16vdH2E1fTMXeYlhegHtDsLCOiem9lDqc8eV", + "ruJLSCH4UH0NC+VlAOWyDTuqTgTz4NGiR0qLpOOJPgVjR1u6BLhvfJftUoL5qmneoN2lP0C/d9/XfP+S", + "X21173IMBIVHq8SAWCqLMPzZRV2kOMPU22t1dgiuzY1FZjg6XigGDfslUz0ZYU3l21xLX2nTVxdzkHfz", + "0VhjX9JP640UapVyqhgJtRsfPFJkyyZ+uCdh/13kFVnXEPjfhsh5veBRi0TX6N07V7YQ/L6u0S6+uJbb", + "GWO7i7ThEb2WLZdod7kj7+N8NObqjKK6mkPb9VKqkB3ihj4Z08ajfLqKtb7dPdDHd6fye8NiRlje15HT", + "YIDfDKpxh8P9aigHPDyJuDjxMPxvLjLa5NohNm7bBYlaN4Faf5+nugNEWgjtjtt7Fk/FY0ebXn+Q4tcC", + "Yn1vKq689eDYKV6tXa/dJnP22DX+PhGx0WHqTmpfqEnOapYYQuvo9wDyj76MOVCRkja9qbwit5aTAh0P", + "3tPg/Q4lHjf5Hra7Gr6NFdYnRFGw858cUZfYwCfElce8fW0kHVGOXKcriXo2vzIv6bM/EFdtt5CFO0u7", + "jfqDtr0HXOLV1rfOieScVi1s1LR2F/Y5hNi7k6d46t97/zm4vHw58OWDBlfRVhRvIBXcV1ufYo8YbL3h", + "UxIP2kLssPFyF17p1kRd5FHu45+RTKlXUBvKvuQJid2SYt1lfnOQERbl2cXheVYzvvia8/MPfPd+VzUk", + "CN0ZOxszNnqn/OXbb7u2id0MO7a1sZ0jMd8uGv+B7th7ejPKklB/djWKbimnOUM8ZBWqlamZOaoAG3+i", + "UzPfQ79DDrcIwncX2kS5QdB4Eq/q20Z7useXmaosU7fxyINGR+taz8U2mjHBo0zbE1NGe2fCML+1DYzZ", + "rVX2Wad29vhq1QejnNrU9D6ZRnutZjuqMkdYn7X2imkGt2nKoby8fEkMkmd8dasp7Y2KRu5QXrVs/nVR", + "jmaJE7b4FjrVYOa1Xq2ImjvL+IwLaegmHrIQdCGxhLNUkmUq4dlcGfv8r19//TVlp+Ksc26wg5xBUf1V", + "zmfwVZ995ef9ihJ6vvJTflV2iglVGnxXRR+LgTNWm8NSubbQsmrkFsgr5jjxIKjOfUra4SludmtrfaKs", + "h8g+HEDjySolcD/HcqjVEbDswCXunCgiQpyeQUgmIXd0X/R9gy230JPV9ylX+ER00NhBFwVU1Yy1/+az", + "KIObqMXCSQmzkslcK6kKE6reBgSbnN/KrRi+xK+eFMW4xKfFsd9CF5Lx509c/GQdt3wDcn/3/8C7+Y1o", + "VhCKIvpHgaVott/Lq5k3moSlJV8UIn3IZeFeCHWn+Swrlb778U8ZX+BEiZi5m6ZVLJit3RRHhQG20tx7", + "+uy/DdXReb7Q3eMFKGF9Cc4urv5rMKFWCtuJz1hui25XZBD59NUfTXtPrMfoUDEV5n/5U0YpewQwE47X", + "jfpU7GDT4Ff/baQOHucT20+0hS776fsVtu4g99uf1uNWaT5GdLaRDlVhtzniKuCpwm70yH0iefQAz1J5", + "NjdsRx9TgK4qbF5Qj/xMTCFZJRl8eUB5ugeUGlWrwrYcZhoSLBc6O6oeYePSlTKH34fvnzRRu1xle23Z", + "drqnH/jpUrQ/UW2LMrE717AUeGdkhFxI2VKkoGrvCDWs++SyTikWss/qiN/4elY+WvnVdb3JPlUh8038", + "G9Vci1Cr278KlMO7HrJQ6MWfsfjgt5PB348Hfx388m//ci/RiAA7WuTfPjidoKJIH/PYEHDlr4NXQmKT", + "+sFJrNGzWICxfJE7IUfN+dGzW01Ng4fsbwXXXFqgeLkJsPevTr/55pu/Dje/gDS2cknxKPfaiY9lue9G", + "3Fa+Pv56E2NjcTmRZUxgsciZBmP6LMd+FszqFfk+qcZjE9zvwerV4GTqflgvhVvMZpQrim01sAOkkKxq", + "mB+6L+oVMUF1iDKW7Vkklu3jnzjhlErxGuRFaqC+g0TJBGmPzvzB956xzUP7U5T5AJsUSliNMj3XguzX", + "+DU0rtTlLh8twY5nWX3aJtjWOqBGQu+eWvk2F9moe59tYlEvBP6EFaIQAmUV90quDdk7Kjlbl3U5aHZ+", + "hi0Qsbb5TBiLXRqxZLWTIMN1LKt8E5JV/vQ4rq1xf/PKh8J92oLhVuVN9UPgNgnPwKrfQKsj389+Y5sQ", + "uiu4iX56Q0UL3QxY+EMxN0vfIZfrNMPry5T9cHV1wazm06lImJJM2CE75VkWaoWcXJxTiWxh3JS3Tlvd", + "8htgwrIJJLwwwD5IcaP51NKvofN44hs73YBvUrIKRQxCzslPb6KlPuiYl+7kV+rvoFVvl7BG/H5g1cCd", + "knlYpY+CnPMUFrmypDb8zAhXCFCtgWi4jjiQm/H2HoxVGowvm0lTl0cpOxFUa/Sd/FW3aEIgNJubIasB", + "LRqRZkAIpbGlmfPTGyaVLyWClbONt23mkKWMO7RFX9nlw3ED8olQQxNvw4yFDBbO9tlaaKfekKkc1Sy1", + "N2Th42+Pv2ViWvuOqnZXRVKjrWf+Bvaq3M8Ter/KRS4tt1G3+1X8gPe13da7W3XPX1aubIkzrn0TDMp3", + "JYR0IgK1WsItzKgSL9w5YAlHGAbrR9TrqLCJSldYTZaCutMX4SZXn0KD5TRO6JISDHXoN3uhnvm+/mg4", + "TTEnqVrGljzxnGF3f5ZkwLUJxZpqp4x1L3LQaxLRE3TopcCLcpl6oc0/zod7byr+VBnTsZKdmxihiPXN", + "AbuF8gMdfn38rEmHt5wIseZHqWjyhQ+vcuOO3Thh3YDHItUXJHbd/5Uy2quf/UTkRWE/HXV/9tS8b7bQ", + "02zIwKcNJ7rcpGAaSr+W/hE3xs7lP7E7BpfkeWciZIJWC9BDgO/SQR8Zxo0RMwnU5lQqq6Q3gYVMNHBs", + "yRR6uofS41ymbMqlG6UKtOQc06kcZHhsSJSUQH3B48wxyYSpxD+9XzzRIx6thUt8oke86pxyCZnKo0SK", + "G8Sw1Nz6LMictv4QBdBsekfz7UAkbfJbe2hre5xBUvPaJbDmm1M1M5HwkL3kyZxNNV9QIC6Wf1B6wcYi", + "fc5+N/Drx+trmXLLn7PfwQNs4ADu/n59LcdO1jcIsmxRloAxg5KMCYagDbp+Eq2MaQkAnxr3gnH2mhs7", + "QBwMzs/oDorderwOqlG045olz0SKF0QNpliEa2fgsDOtctoUBfVQx8oZz00w6MYiHVOPDOyI4+/QIJaQ", + "0m/CUBUFO+eSPWN8DjwNIceZ26sBkPhpP7y13YJ2jC0wb7bsUz4pplPQQ3aaCfzK99a0mic3kdkcN6dg", + "IbG43yF7hdHXNYamZHSpWiBDl1O1bGV3elQ5ZGBYvwHAAtOBHpw4uhUOVnOeY4g/ttIDCVokbNwUEmPq", + "9xnCvf3JwRvBkxWO/RHbZlBTQnbgPl9h+x5HKdRkjrNUJcUCpBs1tqscxtSAimb8yrAx9dtw9KL0oiw4", + "UTWD8dr3X3FbZ/gx8XufGcgg8fuhyaPd6ZBYmsfbWtXtvSO30MkCTZWWcPadppRmBmTKjilHPIqa0NJt", + "V37qM6OaTLHkWUHx8AtwLKI1JFhHgJbibg2BDavCExI9BlRvSA0a+nR5GjtJ6Nc7SLc/XQpH+wSMG3aJ", + "D4KDS0cknizd6P83AAD//5toftdtsAEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/server/lib/sysmon/README.md b/server/lib/sysmon/README.md new file mode 100644 index 00000000..fece52cf --- /dev/null +++ b/server/lib/sysmon/README.md @@ -0,0 +1,206 @@ +# sysmon + +VM-internal failure telemetry for the kernel-images browser VM. Surfaces two event types onto the existing `EventStream` → SSE / S2 pipeline: + +| Event type | Source | Owned by | +| ------------------ | ------------------------------------------ | --------------------- | +| `system_oom_kill` | Linux kernel OOM-killer via `/dev/kmsg` | `lib/sysmon` (in-process goroutine) | +| `service_crashed` | supervisord eventlistener protocol | `cmd/supervisord-shim` (separate binary, POSTs to `/telemetry/events`) | + +Both paths terminate in the same `events.EventStream` so downstream consumers (SSE clients, the S2 sink) see them like any other browser telemetry event. + +## Why two binaries + +| Concern | sysmon (in-process) | supervisord-shim (separate process) | +| --- | --- | --- | +| Why separate | n/a | supervisord's eventlistener protocol requires a separate process talking over stdin/stdout | +| Triggers | kernel OOM-killer writes to `/dev/kmsg` | supervised service exits unexpectedly or FATALs | +| Transport | direct call to `EventStream.Publish` | `POST /telemetry/events` over localhost HTTP | +| Failure mode | open of `/dev/kmsg` may fail (no CAP_SYSLOG); API logs and continues without OOM telemetry | API may be down during shim's POST; shim logs the failure, always ACKs supervisord, and the event is lost | + +## Event taxonomy + +### `system_oom_kill` + +Parsed from one kernel OOM dump in `/dev/kmsg`. Payload (see `BrowserSystemOomKillEventData` in `openapi.yaml` for the authoritative schema): + +| Field | Meaning | Absent when | +| --- | --- | --- | +| `process_name` | comm of the killed process (max 15 chars, kernel TASK_COMM_LEN limit) | never | +| `pid` | PID of the killed process | never | +| `rss_kb` | sum of anon-rss + file-rss + shmem-rss in KiB | never | +| `constraint` | `none` / `memcg` / `cpuset` / `memory_policy` | pre-Linux-5.0 kernels (no structured `oom-kill:` line) | +| `mem_total_kb` | total RAM from `N pages RAM` × 4 KiB | kernel did not emit Mem-Info (e.g. memcg OOM) | +| `mem_free_kb` | free RAM from `free:N free_pcp:N` × 4 KiB | as above | +| `top_tasks` | up to 5 processes from `Tasks state` table, sorted by RSS desc | kernel did not emit the table | +| `trigger_process_name` | comm of the process whose allocation triggered the OOM-killer | sysrq-triggered OOMs (no opener line) | +| `trigger_pid` | PID of the trigger | as above; pre-CPU/PID header kernels | + +### `service_crashed` + +Mapped from supervisord `PROCESS_STATE_EXITED` (with `expected=0`) or `PROCESS_STATE_FATAL`. Schema in `BrowserServiceCrashedEventData`: + +| Field | Meaning | +| --- | --- | +| `service_name` | supervisord program name (e.g. `chromium`, `mutter`, `kernel-images-api`) | +| `pid` | live PID at exit (omitted for `gave_up` since supervisord no longer tracks one) | +| `phase` | `startup` (died during STARTING) / `running` (crashed after reaching RUNNING) / `gave_up` (FATAL via exhausted startretries) | + +Clean stops (`supervisorctl stop`, exit codes in the configured `exitcodes` list) do **not** produce events — supervisord marks them `expected=1` and the shim skips them. + +## File layout + +| File | Concern | +| --- | --- | +| `sysmon.go` | `Monitor` lifecycle (Start/Wait), goroutine wiring, `publishOomKill` | +| `kmsg.go` | OOM-dump text parser (regex + state machine) — see file header for format compatibility notes | +| `kmsg_linux.go` | Linux-only `/dev/kmsg` open via `euank/go-kmsg-parser`, SeekEnd on start so we don't replay history on API restart | +| `kmsg_other.go` | non-Linux stub so dev machines still compile | +| `kmsg_test.go` | parser fixtures + tests (both pre-5.14 and post-5.14 Tasks-state layouts) | +| `sysmon_test.go` | end-to-end test from stub kmsg source through EventStream | + +The supervisord-shim lives at `cmd/supervisord-shim/`. Its configuration is duplicated as `supervisord-shim.conf` under both `images/chromium-headless/image/supervisor/services/` and `images/chromium-headful/supervisor/services/`. + +## How to verify locally (Docker) + +These steps reproduce the smoke matrix from PR #254. Container image is built with `cd images/chromium-headless && ./build-docker.sh`. + +```bash +# Start the container detached (the script's run-docker.sh hardcodes -it). +docker run -d --rm --name chromium-headless-test \ + --platform linux/amd64 --privileged --tmpfs /dev/shm:size=128m \ + -p 9222:9222 -p 444:10001 \ + onkernel/chromium-headless-test:latest + +# Wait for the API. +sleep 10 && curl -sf http://localhost:444/spec.json >/dev/null && echo "API up" + +# Open the SSE stream in another shell to watch events in real time. +curl -sN http://localhost:444/telemetry/stream +``` + +### service_crashed (phase=running) + +```bash +# Kill the chromium browser process the launcher actually spawned. +docker exec chromium-headless-test supervisorctl signal KILL chromium +# Expect one service_crashed event with phase=running. +``` + +### service_crashed (phase=gave_up) + +```bash +# Install a deliberately failing service. +docker exec chromium-headless-test bash -c 'cat > /etc/supervisor/conf.d/services/flaky.conf <$line" > /dev/kmsg +done' +# Expect one system_oom_kill event with constraint=none, mem_total_kb=2097152, +# top_tasks[0].name="chromium", trigger_process_name="chromium". +``` + +### system_oom_kill (real cgroup OOM) + +```bash +docker rm -f chromium-headless-test +# 512 MB cap keeps the API itself alive while letting Chrome OOM. +docker run -d --rm --name chromium-headless-test \ + --platform linux/amd64 --privileged --tmpfs /dev/shm:size=128m \ + --memory 512m --memory-swap 512m \ + -p 9222:9222 -p 444:10001 \ + onkernel/chromium-headless-test:latest + +# Run a memory hog inside. +docker exec chromium-headless-test python3 -c ' +import sys, time +chunks=[] +while True: + chunks.append(b"x"*(60*1024*1024)); sys.stdout.write(f"{len(chunks)*60}MB\n"); sys.stdout.flush(); time.sleep(0.3) +' +# Expect system_oom_kill events with constraint=memcg, and mem_total_kb / +# mem_free_kb omitted (the kernel skips the global Mem-Info dump on memcg +# OOMs). Sanity-check top_tasks names: they should be single tokens +# (`chromium`, `python3`). If they include numbers or extra columns, the +# Tasks-state regex in kmsg.go needs updating for the current kernel. +``` + +## How to verify in production (real Linux 6.x VM) + +```bash +# Spin up a browser session. +kernel browsers create +# Note the session ID, then exec into the VM. + +# Confirm sysmon is running. +kernel browsers process exec -- /bin/bash -c \ + 'tail -50 /var/log/supervisord/kernel-images-api | grep sysmon' +# Look for: "sysmon: kmsg OOM reader started" + +# Trigger an OOM (kills the highest-oom_score process; expect chromium). +kernel browsers process exec -- /bin/bash -c \ + 'echo 1 > /proc/sys/kernel/sysrq; echo f > /proc/sysrq-trigger' + +# Verify the event hit the API stream. +kernel browsers process exec -- /bin/bash -c \ + 'tail -50 /var/log/supervisord/kernel-images-api | grep "sysmon: oom kill"' + +# Clean up. +kernel browsers delete +``` + +## Known limitations + +1. **API self-crash is invisible to sysmon.** If `kernel-images-api` itself dies, the shim's POST fails (connection refused) and that event is lost. The host platform's process/VM-level monitoring is the layer that catches it. Closing the gap inside this binary would require persistent shim-side buffering and is out of scope. +2. **`process_name` is truncated to 15 chars.** This is the kernel's `TASK_COMM_LEN-1` limit, not a parser bug. `kernel-images-api` shows up as `kernel-images-a`. +3. **Page size is hard-coded to 4 KiB.** Correct on x86_64; would be wrong on ARM 16K/64K page kernels. +4. **`mem_total_kb` / `mem_free_kb` are omitted on memcg OOMs.** The kernel does not emit the global `pages RAM` / `free:N` lines when the OOM is cgroup-scoped. This is correct behavior, documented in the openapi schema. +5. **`trigger_*` fields are absent on sysrq-triggered OOMs.** The `X invoked oom-killer:` opener line is only emitted on allocation-driven kills. Real allocation OOMs always populate these fields; only the synthetic `sysrq f` test path doesn't. +6. **No de-dup between kmsg and supervisord.** If a Chrome OOM both fires kmsg (`system_oom_kill`) and causes supervisord to notice the exit (`service_crashed`), both events fire. The overlap is itself a useful signal (RAM exhaustion vs. process bug). + +## Where to look when things break + +| Symptom | First place to check | +| --- | --- | +| No `system_oom_kill` events in prod | API logs for `sysmon: kmsg OOM monitor disabled` — indicates `/dev/kmsg` open failed | +| `system_oom_kill` events have corrupt `top_tasks` names | `oomTaskEntryRe` in `kmsg.go` — kernel changed the Tasks-state column layout again | +| `system_oom_kill` events missing fields after a kernel upgrade | each `oom*Re` regex in `kmsg.go` — sections may have been renamed in the kernel | +| No `service_crashed` events | `cat /var/log/supervisord/supervisord-shim` inside the container; check for `connection refused` to the API | +| Shim looping (supervisord shows repeated spawn) | the shim should never enter FATAL because `startretries=999999`; if it does, check `/var/log/supervisord.log` for spawn errors | +| Events fire locally but don't reach downstream consumers | check the SSE / S2 pipeline (`POST /telemetry/events` → `EventStream.Publish` → SSE / S2) — that's `lib/events` territory, not sysmon | diff --git a/server/lib/sysmon/kmsg.go b/server/lib/sysmon/kmsg.go new file mode 100644 index 00000000..576b6b71 --- /dev/null +++ b/server/lib/sysmon/kmsg.go @@ -0,0 +1,293 @@ +// kmsg.go parses the kernel OOM-dump text inside /dev/kmsg messages. +// The wire envelope is handled by euank/go-kmsg-parser; body text +// parsing is ours. +// +// Format stability across kernel versions: +// - "Killed process N (name) anon-rss:... file-rss:... shmem-rss:..." +// is unchanged since 2.6.x. +// - "oom-kill:constraint=CONSTRAINT_X" appeared in 5.0 and is +// stable since. Absent on older kernels; Constraint is omitted. +// - "N pages RAM" and "free:N free_pcp:N free_cma:N" are stable. +// - Tasks-state row gained rss_anon/rss_file/rss_shmem in 5.14 +// (9-col → 12-col). We anchor on bracketed pid + rss-as-5th-col +// + trailing-token name so both layouts parse. Production is +// Linux 6.12; the older layout is kept for dev environments. +// +// On format breakage the failure mode is graceful: missing fields +// are omitted from the published event, and oomScannerWatchdog +// abandons a stuck section without leaking memory. +package sysmon + +import ( + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// pageSizeKB is the kernel page size in KiB used to convert page-denominated +// fields from the OOM dump (RSS in `Tasks state`, free/total in Mem-Info) +// into KiB. The kernel-images target is x86_64 with the standard 4 KiB page; +// this constant is wrong on architectures with a different page size. +const pageSizeKB = 4 + +// topTasksN bounds the number of process entries from the kernel's Tasks +// state table that we surface in the OOM event payload. Five is enough to +// answer "what consumed memory" for a typical Chromium-on-VM workload +// without bloating the event. +const topTasksN = 5 + +// oomScannerWatchdog bounds the number of UNRECOGNIZED kmsg messages we +// will tolerate inside a single OOM section before abandoning it. +// Recognized lines (Mem-Info, Tasks state, constraint, killed) don't +// count toward the budget, so the watchdog only trips when the section +// diverges from the expected kernel format. A busy VM can emit several +// hundred Tasks state rows during a single dump; this budget leaves +// headroom for that plus the full Mem-Info block. +const oomScannerWatchdog = 2000 + +// OomInstance is a parsed kernel OOM-killer event extracted from /dev/kmsg. +// +// Fields map to BrowserSystemOomKillEventData. Optional fields use the +// zero value when the kernel did not emit the corresponding kmsg line; +// the publisher decides whether to encode them. +type OomInstance struct { + // ProcessName is the comm of the killed process, bounded to 15 chars + // by the kernel (TASK_COMM_LEN-1). May contain spaces. + ProcessName string + // Pid is the PID of the killed process. + Pid int + // RssKb is the sum of anon-rss, file-rss, and shmem-rss in KiB. + // Zero if the kernel format did not include the per-class breakdown. + RssKb int + // Constraint is one of "none", "memcg", "cpuset", "memory_policy", + // extracted from the structured `oom-kill:` line that kernels >= 5.0 + // emit. Empty on older kernels. + Constraint string + // MemTotalKb is the total system memory at the time of the kill, + // derived from the `N pages RAM` line. Zero if not parseable. + MemTotalKb int + // MemFreeKb is free memory at the time of the kill, derived from the + // `free:N` field in Mem-Info. Zero if not parseable. + MemFreeKb int + // TopTasks is up to topTasksN processes from the Tasks state table, + // sorted by RSS descending. Nil if the kernel did not emit the table. + TopTasks []TaskMemSnapshot + // TriggerProcessName is the comm of the process whose allocation + // failed and caused the OOM-killer to run. Captured from the prefix + // of the "invoked oom-killer:" line. Often equal to ProcessName but + // can differ when the kernel selected a different victim. + TriggerProcessName string + // TriggerPid is the PID of the triggering process, captured from + // the standard "CPU: N PID: N Comm: ..." header line. Zero if the + // kernel did not emit that header. + TriggerPid int + // TimeOfDeath is the timestamp of the closing "Killed process" line + // as reported by the kmsg envelope. + TimeOfDeath time.Time +} + +// TaskMemSnapshot is one row from the kernel's Tasks state dump, +// representing a single process's memory footprint at the moment of the +// OOM kill. +type TaskMemSnapshot struct { + Pid int + Name string + RssKb int +} + +// KmsgMessage is the minimal subset of a /dev/kmsg record that the OOM +// state machine consumes. Decoupling from the underlying kmsg library +// lets the parser run portably under unit tests; the production wiring +// lives in kmsg_linux.go. +type KmsgMessage struct { + Timestamp time.Time + Body string +} + +var ( + // Opening line. Captures the triggering process's comm (the prefix). + // Example: `chromium invoked oom-killer: gfp_mask=0x100cca, order=0, oom_score_adj=0`. + // Comm can contain spaces (e.g. `kworker u4:1`), so the lazy match + // keeps the entire prefix up to the literal `invoked oom-killer:`. + oomStartRe = regexp.MustCompile(`^(.+?)\s+invoked oom-killer:`) + // Standard kernel printk header that immediately follows the opening + // line. Source of the triggering PID. Example: + // `CPU: 2 PID: 1234 Comm: chromium Not tainted 5.15.0-1-amd64 #1` + oomTriggerPidRe = regexp.MustCompile(`^CPU:\s+\d+\s+PID:\s+(\d+)\s+Comm:`) + // Modern (Linux >= 5.0) structured constraint+task line. + // Example: + // oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chromium,pid=1234,uid=0 + oomConstraintRe = regexp.MustCompile(`oom-kill:constraint=CONSTRAINT_(\w+)`) + // Closing line. Example: + // Out of memory: Killed process 1234 (chromium) total-vm:5234572kB, anon-rss:4823900kB, file-rss:100kB, shmem-rss:200kB, UID:0 pgtables:8000kB oom_score_adj:0 + oomKilledRe = regexp.MustCompile(`Killed process (\d+) \(([^)]+)\)`) + // RSS breakdown is matched independently so the killed-line match + // remains useful on kernels that omit per-class RSS fields. + oomAnonRssRe = regexp.MustCompile(`anon-rss:(\d+)kB`) + oomFileRssRe = regexp.MustCompile(`file-rss:(\d+)kB`) + oomShmemRssRe = regexp.MustCompile(`shmem-rss:(\d+)kB`) + // Mem-Info / total RAM. Example: `524288 pages RAM`. + oomTotalPagesRe = regexp.MustCompile(`(\d+)\s+pages\s+RAM`) + // Mem-Info free-pages triple. Example: ` free:4560 free_pcp:0 free_cma:0`. + // The trailing `free_pcp:` anchor distinguishes this from per-zone + // lines like `Node 0 DMA free:11264kB boost:0kB`, which carry kB + // units rather than raw page counts. + oomFreePagesRe = regexp.MustCompile(`(?:^|\s)free:(\d+)\s+free_pcp:`) + // Tasks state row. The column count varies across kernel versions + // (see the file header) so we anchor on the invariants: bracketed + // pid, rss as the 5th numeric column, oom_score_adj as the last + // numeric column (always present, possibly negative), and the + // remainder of the line as the comm. Capturing the trailing comm + // lazily (rather than as a single \S+ token) preserves names with + // internal whitespace; this matches the behavior of the kill-line + // and trigger-line parsers. Example (Linux 5.14+): + // [ 1234] 1000 1234 1308611 1205975 1205675 200 100 9678848 0 0 chromium + oomTaskEntryRe = regexp.MustCompile(`^\[\s*(\d+)\]\s+\d+\s+\d+\s+\d+\s+(\d+)\s+.+\s+-?\d+\s+(.+?)\s*$`) +) + +// oomScanner is a state machine that turns a stream of kmsg message +// bodies into completed OomInstance values. It tolerates the kernel +// emitting many intermediate lines (stack traces, Mem-Info zone +// breakdowns, the Tasks state table) between the opening "invoked +// oom-killer" line and the closing "Killed process" line, and recovers +// if a section never closes. +type oomScanner struct { + pending *OomInstance + noiseBuf int +} + +// feed consumes a single kmsg message body and returns a completed +// OomInstance when the scanner observes the closing "Killed process" +// line. All other inputs return nil; the scanner accumulates partial +// state internally. +func (s *oomScanner) feed(body string, ts time.Time) *OomInstance { + if m := oomStartRe.FindStringSubmatch(body); m != nil { + // New section. If a previous section is still pending, the + // kernel either failed to emit the closing line or the kmsg + // ring buffer dropped it; abandon and start fresh. + s.pending = &OomInstance{ + TimeOfDeath: ts, + TriggerProcessName: m[1], + } + s.noiseBuf = 0 + return nil + } + + if s.pending == nil { + return nil + } + + if m := oomKilledRe.FindStringSubmatch(body); m != nil { + pid, _ := strconv.Atoi(m[1]) + s.pending.Pid = pid + s.pending.ProcessName = m[2] + s.pending.RssKb = sumRss(body) + s.pending.TimeOfDeath = ts + trimTopTasks(s.pending) + out := s.pending + s.pending = nil + s.noiseBuf = 0 + return out + } + + // Only unparseable intermediate lines erode the watchdog budget; + // recognized lines are free. The budget is a per-section total + // (see oomScannerWatchdog) — it does not reset on productive + // matches. + matched := false + if m := oomTriggerPidRe.FindStringSubmatch(body); m != nil { + // Only set on first match — a kernel oops can include multiple + // CPU/PID lines in stack traces and we want the one that opened + // the section, which is always emitted first. + if s.pending.TriggerPid == 0 { + if n, err := strconv.Atoi(m[1]); err == nil { + s.pending.TriggerPid = n + } + } + matched = true + } + if m := oomConstraintRe.FindStringSubmatch(body); m != nil { + s.pending.Constraint = constraintFromKernel(m[1]) + matched = true + } + if m := oomTotalPagesRe.FindStringSubmatch(body); m != nil { + if n, err := strconv.Atoi(m[1]); err == nil { + s.pending.MemTotalKb = n * pageSizeKB + } + matched = true + } + if m := oomFreePagesRe.FindStringSubmatch(body); m != nil { + if n, err := strconv.Atoi(m[1]); err == nil { + s.pending.MemFreeKb = n * pageSizeKB + } + matched = true + } + if m := oomTaskEntryRe.FindStringSubmatch(body); m != nil { + pid, _ := strconv.Atoi(m[1]) + rss, _ := strconv.Atoi(m[2]) + s.pending.TopTasks = append(s.pending.TopTasks, TaskMemSnapshot{ + Pid: pid, + Name: m[3], + RssKb: rss * pageSizeKB, + }) + matched = true + } + + if !matched { + s.noiseBuf++ + if s.noiseBuf > oomScannerWatchdog { + s.pending = nil + s.noiseBuf = 0 + } + } + return nil +} + +// trimTopTasks sorts the accumulated Tasks state entries by RSS +// descending and caps the slice at topTasksN. +func trimTopTasks(o *OomInstance) { + if len(o.TopTasks) == 0 { + return + } + sort.Slice(o.TopTasks, func(i, j int) bool { + return o.TopTasks[i].RssKb > o.TopTasks[j].RssKb + }) + if len(o.TopTasks) > topTasksN { + o.TopTasks = o.TopTasks[:topTasksN] + } +} + +// constraintFromKernel converts the kernel's CONSTRAINT_* identifier +// into the lowercase form used in the public event schema. Unknown +// values are passed through lowercased so the kernel's exact label still +// reaches logs even if it falls outside the schema enum. +func constraintFromKernel(raw string) string { + switch raw { + case "NONE": + return "none" + case "CPUSET": + return "cpuset" + case "MEMCG": + return "memcg" + case "MEMORY_POLICY": + return "memory_policy" + default: + return strings.ToLower(raw) + } +} + +// sumRss extracts anon-rss, file-rss, and shmem-rss from the canonical +// "Killed process" line and returns their sum in KiB. Missing fields +// contribute zero, so older kernels that omit the breakdown yield 0. +func sumRss(body string) int { + var total int + for _, re := range []*regexp.Regexp{oomAnonRssRe, oomFileRssRe, oomShmemRssRe} { + if m := re.FindStringSubmatch(body); m != nil { + n, _ := strconv.Atoi(m[1]) + total += n + } + } + return total +} diff --git a/server/lib/sysmon/kmsg_linux.go b/server/lib/sysmon/kmsg_linux.go new file mode 100644 index 00000000..55e3c2dc --- /dev/null +++ b/server/lib/sysmon/kmsg_linux.go @@ -0,0 +1,69 @@ +//go:build linux + +package sysmon + +import ( + "fmt" + "log/slog" + + "github.com/euank/go-kmsg-parser/v2/kmsgparser" +) + +// openKmsgSource opens /dev/kmsg and seeks past the existing ring buffer +// so the scanner only sees events that occur after this call. Without the +// seek, each process restart would replay the entire historical buffer +// and emit stale events with current timestamps. +func openKmsgSource(logger *slog.Logger) (kmsgSource, error) { + p, err := kmsgparser.NewParser() + if err != nil { + return nil, fmt.Errorf("open /dev/kmsg: %w", err) + } + p.SetLogger(kmsgLogger{logger: logger}) + if err := p.SeekEnd(); err != nil { + p.Close() + return nil, fmt.Errorf("seek to end of /dev/kmsg: %w", err) + } + return &kmsgparserSource{p: p}, nil +} + +// kmsgparserSource adapts the third-party kmsgparser.Parser to the +// internal kmsgSource interface so the rest of sysmon stays decoupled +// from the library type. +type kmsgparserSource struct { + p kmsgparser.Parser +} + +func (s *kmsgparserSource) Messages() <-chan KmsgMessage { + in := s.p.Parse() + out := make(chan KmsgMessage) + go func() { + defer close(out) + for m := range in { + out <- KmsgMessage{Timestamp: m.Timestamp, Body: m.Message} + } + }() + return out +} + +func (s *kmsgparserSource) Close() error { return s.p.Close() } + +// kmsgLogger routes the kmsgparser library's diagnostic output through our +// structured logger. +type kmsgLogger struct { + logger *slog.Logger +} + +func (l kmsgLogger) Infof(format string, args ...any) { + // The library only logs Info on graceful shutdown ("kmsg reader + // closed, shutting down"). Treat it like sysmon.go's own + // start/stop signals: useful for debugging but not Info-worthy. + l.logger.Debug(fmt.Sprintf("sysmon/kmsg: "+format, args...)) +} + +func (l kmsgLogger) Warningf(format string, args ...any) { + l.logger.Warn(fmt.Sprintf("sysmon/kmsg: "+format, args...)) +} + +func (l kmsgLogger) Errorf(format string, args ...any) { + l.logger.Error(fmt.Sprintf("sysmon/kmsg: "+format, args...)) +} diff --git a/server/lib/sysmon/kmsg_other.go b/server/lib/sysmon/kmsg_other.go new file mode 100644 index 00000000..98111a79 --- /dev/null +++ b/server/lib/sysmon/kmsg_other.go @@ -0,0 +1,18 @@ +//go:build !linux + +package sysmon + +import ( + "errors" + "log/slog" +) + +// errKmsgUnsupported indicates that /dev/kmsg-based OOM telemetry is not +// available on this platform. The production target is Linux; non-Linux +// builds compile so developer machines can run unit tests, but the +// monitor itself is inert there. +var errKmsgUnsupported = errors.New("sysmon: /dev/kmsg OOM monitoring is only supported on Linux") + +func openKmsgSource(_ *slog.Logger) (kmsgSource, error) { + return nil, errKmsgUnsupported +} diff --git a/server/lib/sysmon/kmsg_test.go b/server/lib/sysmon/kmsg_test.go new file mode 100644 index 00000000..9f8a1893 --- /dev/null +++ b/server/lib/sysmon/kmsg_test.go @@ -0,0 +1,358 @@ +package sysmon + +import ( + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// canonicalOomDump is a representative slice of the kmsg lines the kernel +// emits during a global OOM kill, using the pre-5.14 Tasks-state layout +// (9 columns, no rss_anon/rss_file/rss_shmem). The Mem-Info and Tasks +// state sections are abbreviated but preserve the field layout the +// parser depends on. See modernKernelOomDump for the post-5.14 shape. +var canonicalOomDump = []string{ + `chromium invoked oom-killer: gfp_mask=0x100cca(GFP_HIGHUSER_MOVABLE), order=0, oom_score_adj=0`, + `CPU: 2 PID: 1234 Comm: chromium Not tainted 5.15.0-1-amd64 #1`, + `Call Trace:`, + ` dump_stack_lvl+0x44/0x57`, + `Mem-Info:`, + `active_anon:123456 inactive_anon:78901 isolated_anon:0`, + ` slab_reclaimable:2340 slab_unreclaimable:5670`, + ` mapped:8901 shmem:120 pagetables:340`, + ` free:4560 free_pcp:0 free_cma:0`, + `Node 0 active_anon:493824kB inactive_anon:315604kB`, + `Node 0 DMA free:11264kB boost:0kB min:64kB`, + `Node 0 DMA32 free:6976kB boost:0kB min:8120kB`, + `524288 pages RAM`, + `0 pages HighMem/MovableOnly`, + `21465 pages reserved`, + `Tasks state (memory values in pages):`, + `[ pid ] uid tgid total_vm rss pgtables_bytes swapents oom_score_adj name`, + `[ 111] 0 111 1234 234 45056 0 -250 systemd`, + `[ 234] 0 234 65432 12345 200704 0 0 sshd`, + `[ 1234] 1000 1234 1308611 1205975 9678848 0 0 chromium`, + `[ 5678] 1000 5678 123456 34567 331776 0 0 mutter`, + `oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=chromium,pid=1234,uid=1000`, + `Out of memory: Killed process 1234 (chromium) total-vm:5234572kB, anon-rss:4823900kB, file-rss:100kB, shmem-rss:200kB, UID:1000 pgtables:9678848kB oom_score_adj:0`, +} + +// feedAll runs every line in dump through the scanner and returns every +// completed OomInstance, in order. Used by tests that need to verify a +// whole dump's parsing. +func feedAll(s *oomScanner, dump []string, base time.Time) []OomInstance { + var out []OomInstance + for i, line := range dump { + ts := base.Add(time.Duration(i) * time.Millisecond) + if oom := s.feed(line, ts); oom != nil { + out = append(out, *oom) + } + } + return out +} + +func TestOomScannerCanonicalDump(t *testing.T) { + var s oomScanner + base := time.Unix(1_700_000_000, 0) + got := feedAll(&s, canonicalOomDump, base) + require.Len(t, got, 1) + + oom := got[0] + assert.Equal(t, "chromium", oom.ProcessName) + assert.Equal(t, 1234, oom.Pid) + assert.Equal(t, 4823900+100+200, oom.RssKb) + assert.Equal(t, "none", oom.Constraint) + + // Trigger comes from the opening line + the CPU/PID header. In this + // canonical case the trigger and the victim are the same process. + assert.Equal(t, "chromium", oom.TriggerProcessName) + assert.Equal(t, 1234, oom.TriggerPid) + + // 524288 pages * 4 KiB = 2 GiB total + assert.Equal(t, 524288*4, oom.MemTotalKb) + // 4560 pages * 4 KiB = ~17.8 MiB free + assert.Equal(t, 4560*4, oom.MemFreeKb) + + // Top tasks sorted by RSS desc; the 4 tasks in the fixture all fit + // under the cap so the slice contains all of them in the right order. + require.Len(t, oom.TopTasks, 4) + assert.Equal(t, "chromium", oom.TopTasks[0].Name) + assert.Equal(t, 1234, oom.TopTasks[0].Pid) + assert.Equal(t, 1205975*4, oom.TopTasks[0].RssKb) + assert.Equal(t, "mutter", oom.TopTasks[1].Name) + assert.Equal(t, "sshd", oom.TopTasks[2].Name) + assert.Equal(t, "systemd", oom.TopTasks[3].Name) +} + +func TestOomScannerLegacyKernelNoMemInfoNoTasks(t *testing.T) { + // Pre-5.0 kernels emit just the opening line, some stack trace, and + // the closing line. The scanner must still produce a baseline event + // with the new fields left at their zero values. + dump := []string{ + `chromium invoked oom-killer: gfp_mask=0x14000c0, order=0, oom_score_adj=0`, + `Mem-Info:`, + `Out of memory: Killed process 9 (mutter) total-vm:200kB, anon-rss:150kB, file-rss:10kB, shmem-rss:5kB, UID:1000 pgtables:1kB oom_score_adj:0`, + } + var s oomScanner + got := feedAll(&s, dump, time.Now()) + require.Len(t, got, 1) + oom := got[0] + assert.Equal(t, "mutter", oom.ProcessName) + assert.Equal(t, 9, oom.Pid) + assert.Equal(t, 165, oom.RssKb) + assert.Empty(t, oom.Constraint) + assert.Zero(t, oom.MemTotalKb) + assert.Zero(t, oom.MemFreeKb) + assert.Empty(t, oom.TopTasks) +} + +func TestOomScannerTasksTableCappedAtTopN(t *testing.T) { + // Build a synthetic Tasks state with more than topTasksN entries to + // verify the scanner sorts by RSS and trims. + dump := []string{`chromium invoked oom-killer: gfp_mask=0, order=0, oom_score_adj=0`} + rssVals := []int{100, 900, 50, 800, 700, 200, 600, 300, 400, 500} // 10 procs + for i, rss := range rssVals { + pid := strconv.Itoa(i + 1) + dump = append(dump, + "[ "+pid+"] 0 "+pid+" 1000 "+strconv.Itoa(rss)+" 1024 0 0 proc"+pid) + } + dump = append(dump, + `Out of memory: Killed process 1 (proc1) total-vm:0kB, anon-rss:0kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:0kB oom_score_adj:0`, + ) + + var s oomScanner + got := feedAll(&s, dump, time.Now()) + require.Len(t, got, 1) + require.Len(t, got[0].TopTasks, topTasksN, "should cap at topTasksN") + // Top 5 by RSS: 900, 800, 700, 600, 500 + wantRss := []int{900 * pageSizeKB, 800 * pageSizeKB, 700 * pageSizeKB, 600 * pageSizeKB, 500 * pageSizeKB} + for i, w := range wantRss { + assert.Equal(t, w, got[0].TopTasks[i].RssKb, "position %d", i) + } +} + +// modernKernelOomDump represents the Tasks-state format emitted by +// any Linux 5.14+ kernel — including Docker Desktop's bundled +// LinuxKit (6.10 at time of writing) and production VMs (6.12). The +// kernel added rss_anon/rss_file/rss_shmem columns between rss and +// pgtables_bytes, giving 12 numeric+name columns vs. 9 in the +// pre-5.14 layout. The parser must handle both shapes. +var modernKernelOomDump = []string{ + `chromium invoked oom-killer: gfp_mask=0x100cca, order=0, oom_score_adj=0`, + `Mem-Info:`, + `Tasks state (memory values in pages):`, + `[ pid ] uid tgid total_vm rss rss_anon rss_file rss_shmem pgtables_bytes swapents oom_score_adj name`, + `[ 34512] 0 34512 379985 4730 3330 1400 0 188416 0 0 wrapper`, + `[ 34556] 0 34556 126162 23649 11063 12586 0 462848 0 0 Xvfb`, + `[ 34560] 103 34560 73393 1872 819 1053 0 110592 0 0 dbus-daemon`, + `[ 34561] 0 34561 12670355 5635 1775 3860 0 208896 0 0 chromedriver`, + `[ 36183] 1000 36183 302080 72705 71925 780 0 1208320 0 0 chromium`, + `oom-kill:constraint=CONSTRAINT_MEMCG,task=chromium,pid=36183,uid=1000`, + `Out of memory: Killed process 36183 (chromium) total-vm:1208320kB, anon-rss:287700kB, file-rss:3120kB, shmem-rss:0kB, UID:1000 pgtables:1208320kB oom_score_adj:0`, +} + +func TestOomScannerModernKernelTaskColumns(t *testing.T) { + // Regression: real Linux 5.14+ kmsg dumps have 3 extra columns + // (rss_anon, rss_file, rss_shmem) compared to the legacy layout. + // An overly-rigid regex captures those trailing numeric columns as + // part of the `name` field, producing top_tasks entries like + // "1208320 0 0 chromium" instead of "chromium". + var s oomScanner + got := feedAll(&s, modernKernelOomDump, time.Now()) + require.Len(t, got, 1) + oom := got[0] + + assert.Equal(t, "chromium", oom.ProcessName) + assert.Equal(t, 36183, oom.Pid) + assert.Equal(t, "memcg", oom.Constraint) + + require.Len(t, oom.TopTasks, 5) + // Top task by RSS is chromium (72705 pages). Name must be just + // "chromium", not the leading-pgtables-padded "1208320 ... chromium". + assert.Equal(t, "chromium", oom.TopTasks[0].Name) + assert.Equal(t, 36183, oom.TopTasks[0].Pid) + assert.Equal(t, 72705*pageSizeKB, oom.TopTasks[0].RssKb) + + for _, task := range oom.TopTasks { + assert.NotContains(t, task.Name, " ", "task name must be a single token, got %q", task.Name) + } +} + +func TestOomScannerCommWithInternalSpace(t *testing.T) { + // Kernel comms with internal spaces (e.g. kworker threads) must + // survive every capture path: the start line, the killed line, and + // the Tasks state row that feeds top_tasks. + var s oomScanner + s.feed(`kworker u4:1 invoked oom-killer: gfp_mask=0, order=0, oom_score_adj=0`, time.Now()) + s.feed(`[ 42] 0 42 100 100 1024 0 0 kworker u4:1`, time.Now()) + got := s.feed(`Out of memory: Killed process 42 (kworker u4:1) total-vm:0kB, anon-rss:0kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:0kB oom_score_adj:0`, time.Now()) + require.NotNil(t, got) + assert.Equal(t, "kworker u4:1", got.ProcessName) + assert.Equal(t, 42, got.Pid) + assert.Equal(t, "kworker u4:1", got.TriggerProcessName) + require.Len(t, got.TopTasks, 1) + assert.Equal(t, "kworker u4:1", got.TopTasks[0].Name) + assert.Equal(t, 42, got.TopTasks[0].Pid) + assert.Equal(t, 100*pageSizeKB, got.TopTasks[0].RssKb) +} + +func TestOomScannerIgnoresPreambleWhenIdle(t *testing.T) { + // A "Killed process" line in isolation must NOT emit. Without the + // section delimiter we cannot attribute it reliably; the kernel + // occasionally surfaces orphaned OOM lines when the ring buffer + // wraps. + var s oomScanner + got := s.feed(`Out of memory: Killed process 1 (init) total-vm:1kB, anon-rss:0kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:0kB oom_score_adj:0`, time.Now()) + assert.Nil(t, got) +} + +func TestOomScannerIgnoresUnrelatedLines(t *testing.T) { + var s oomScanner + assert.Nil(t, s.feed("usb 1-1: new high-speed USB device number 5", time.Now())) + // A bracketed line that LOOKS like a task entry but has the wrong + // column count should not crash or misparse. + assert.Nil(t, s.feed("[1234] only one field", time.Now())) +} + +func TestOomScannerSecondStartAbandonsFirst(t *testing.T) { + // If a section never completes and a new one starts, the new section + // must not inherit state from the abandoned one. + var s oomScanner + s.feed(`stale-proc invoked oom-killer: gfp_mask=0, order=0, oom_score_adj=0`, time.Now()) + s.feed(`CPU: 1 PID: 999 Comm: stale-proc`, time.Now()) + s.feed("oom-kill:constraint=CONSTRAINT_MEMCG,task=stale,pid=1,uid=0", time.Now()) + s.feed("[ 1] 0 1 100 900 1024 0 0 stale", time.Now()) + s.feed(`fresh-proc invoked oom-killer: gfp_mask=0, order=0, oom_score_adj=0`, time.Now()) + got := s.feed(`Out of memory: Killed process 7 (real) total-vm:0kB, anon-rss:10kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:0kB oom_score_adj:0`, time.Now()) + require.NotNil(t, got) + assert.Equal(t, "real", got.ProcessName) + assert.Equal(t, 7, got.Pid) + assert.Empty(t, got.Constraint, "stale section's constraint must not leak") + assert.Empty(t, got.TopTasks, "stale section's tasks must not leak") + assert.Equal(t, "fresh-proc", got.TriggerProcessName, "trigger from abandoned section must not leak") + assert.Zero(t, got.TriggerPid, "trigger PID from abandoned section must not leak") +} + +func TestOomScannerTriggerDiffersFromKilled(t *testing.T) { + // A chrome renderer allocates and trips the OOM-killer; the kernel + // chooses mutter (the largest non-essential victim) to kill. The + // event must surface both the trigger AND the killed process so the + // customer can distinguish "the process that consumed memory" from + // "the process the kernel decided to sacrifice". + dump := []string{ + `chromium-render invoked oom-killer: gfp_mask=0x100cca, order=0, oom_score_adj=0`, + `CPU: 2 PID: 9999 Comm: chromium-render`, + `Mem-Info:`, + `oom-kill:constraint=CONSTRAINT_NONE,task=mutter,pid=5678,uid=0`, + `Out of memory: Killed process 5678 (mutter) total-vm:1234kB, anon-rss:50kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:0kB oom_score_adj:0`, + } + var s oomScanner + got := feedAll(&s, dump, time.Now()) + require.Len(t, got, 1) + assert.Equal(t, "mutter", got[0].ProcessName) + assert.Equal(t, 5678, got[0].Pid) + assert.Equal(t, "chromium-render", got[0].TriggerProcessName) + assert.Equal(t, 9999, got[0].TriggerPid) +} + +func TestOomScannerTriggerHeaderAbsentLeavesPidZero(t *testing.T) { + // On kernels (or kmsg-drop scenarios) where the CPU/PID header is + // missing, the trigger NAME still comes from the opening line, but + // the trigger PID stays zero — the publisher then omits it. + dump := []string{ + `firefox invoked oom-killer: gfp_mask=0, order=0, oom_score_adj=0`, + `Out of memory: Killed process 42 (firefox) total-vm:0kB, anon-rss:10kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:0kB oom_score_adj:0`, + } + var s oomScanner + got := feedAll(&s, dump, time.Now()) + require.Len(t, got, 1) + assert.Equal(t, "firefox", got[0].TriggerProcessName) + assert.Zero(t, got[0].TriggerPid) +} + +func TestOomScannerSequentialKills(t *testing.T) { + // Two complete dumps back-to-back must emit two independent events. + // Real systems do cascade OOM kills when memory pressure is severe. + second := []string{ + `mutter invoked oom-killer: gfp_mask=0, order=0, oom_score_adj=0`, + `Mem-Info:`, + `oom-kill:constraint=CONSTRAINT_MEMCG,task=mutter,pid=99,uid=1000`, + `Out of memory: Killed process 99 (mutter) total-vm:1kB, anon-rss:50kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:0kB oom_score_adj:0`, + } + combined := append([]string{}, canonicalOomDump...) + combined = append(combined, second...) + + var s oomScanner + got := feedAll(&s, combined, time.Now()) + require.Len(t, got, 2) + assert.Equal(t, "chromium", got[0].ProcessName) + assert.Equal(t, "none", got[0].Constraint) + assert.Equal(t, "mutter", got[1].ProcessName) + assert.Equal(t, "memcg", got[1].Constraint) +} + +func TestOomScannerNoiseWatchdogReleasesStuckSection(t *testing.T) { + var s oomScanner + s.feed(`x invoked oom-killer: gfp_mask=0, order=0, oom_score_adj=0`, time.Now()) + for i := 0; i < oomScannerWatchdog+10; i++ { + s.feed("filler line that matches no pattern", time.Now()) + } + // Scanner should have reset; a closing line now lands as orphaned + // preamble and is ignored. + got := s.feed(`Out of memory: Killed process 1 (x) total-vm:0kB, anon-rss:0kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:0kB oom_score_adj:0`, time.Now()) + assert.Nil(t, got) +} + +func TestOomScannerNoiseBudgetIsTotalNotConsecutive(t *testing.T) { + // The watchdog budget is a per-section TOTAL, not "consecutive + // noise since the last recognized line". A section that interleaves + // noise with sporadic productive matches still trips once cumulative + // noise exceeds the budget. + var s oomScanner + s.feed(`chromium invoked oom-killer: gfp_mask=0, order=0, oom_score_adj=0`, time.Now()) + // Alternate: 1 recognized task entry, then 1001 noise lines, repeat. + // Two cycles -> 2002 noise > oomScannerWatchdog -> abandoned. + for cycle := 0; cycle < 2; cycle++ { + s.feed("[ 1] 0 1 100 200 1024 0 0 proc1", time.Now()) + for i := 0; i < oomScannerWatchdog/2+1; i++ { + s.feed("filler line that matches no pattern", time.Now()) + } + } + got := s.feed(`Out of memory: Killed process 1 (x) total-vm:0kB, anon-rss:0kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:0kB oom_score_adj:0`, time.Now()) + assert.Nil(t, got, "interleaved noise must accumulate toward the watchdog total") +} + +func TestOomScannerRecognizedLinesDoNotErodeWatchdog(t *testing.T) { + // A Tasks state table with hundreds of entries should not trip the + // watchdog — recognized lines are productive parsing, not noise. + dump := []string{`chromium invoked oom-killer: gfp_mask=0, order=0, oom_score_adj=0`} + for i := 0; i < oomScannerWatchdog+100; i++ { + pid := strconv.Itoa(i) + dump = append(dump, + "[ "+pid+"] 0 "+pid+" 1234 567 45056 0 0 proc"+pid) + } + dump = append(dump, + `Out of memory: Killed process 1 (x) total-vm:0kB, anon-rss:0kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:0kB oom_score_adj:0`, + ) + + var s oomScanner + got := feedAll(&s, dump, time.Now()) + require.Len(t, got, 1, "watchdog must not abandon a section composed only of recognized lines") +} + +func TestConstraintFromKernel(t *testing.T) { + cases := map[string]string{ + "NONE": "none", + "CPUSET": "cpuset", + "MEMCG": "memcg", + "MEMORY_POLICY": "memory_policy", + "SOMETHING_FUTURE": "something_future", + } + for raw, want := range cases { + assert.Equal(t, want, constraintFromKernel(raw), raw) + } +} + diff --git a/server/lib/sysmon/sysmon.go b/server/lib/sysmon/sysmon.go new file mode 100644 index 00000000..7775385f --- /dev/null +++ b/server/lib/sysmon/sysmon.go @@ -0,0 +1,190 @@ +// Package sysmon emits VM-internal failure telemetry — OOM kills +// surfaced through /dev/kmsg, and (via the supervisord-shim binary +// POSTing to the telemetry HTTP endpoint) supervised-service crashes. +// +// The package only owns the in-process kmsg reader; service crashes are +// delivered as ordinary caller-published events via POST +// /telemetry/events from the shim. Both paths terminate in the same +// EventStream. +package sysmon + +import ( + "context" + "encoding/json" + "log/slog" + "sync" + + "github.com/kernel/kernel-images/server/lib/events" + oapi "github.com/kernel/kernel-images/server/lib/oapi" +) + +// kmsgSource abstracts a /dev/kmsg-shaped stream of kernel ring buffer +// messages. The production implementation lives in kmsg_linux.go; tests +// supply a stub via Monitor.kmsgSource. +type kmsgSource interface { + Messages() <-chan KmsgMessage + Close() error +} + +// Monitor runs the in-process sysmon goroutine and publishes events +// directly to the EventStream. System-category events are always +// captured regardless of any active TelemetrySession config, so we +// deliberately bypass TelemetrySession here. +type Monitor struct { + es *events.EventStream + logger *slog.Logger + + // kmsgSource lets tests inject a stub stream of kmsg messages. + // Production callers leave this nil; Start() then opens /dev/kmsg. + kmsgSource kmsgSource + + wg sync.WaitGroup +} + +type option func(*Monitor) + +// withKmsgSource overrides the kmsg source. Test-only. +func withKmsgSource(src kmsgSource) option { + return func(m *Monitor) { m.kmsgSource = src } +} + +// New constructs a Monitor. The Monitor does nothing until Start is +// called. +func New(es *events.EventStream, logger *slog.Logger, opts ...option) *Monitor { + m := &Monitor{es: es, logger: logger} + for _, opt := range opts { + opt(m) + } + return m +} + +// Start opens the kmsg source (validating that it is usable) and +// launches the background OOM reader goroutine. It returns an error if +// the kmsg source cannot be opened; the goroutine then never starts and +// the caller can decide whether the failure is fatal. +// +// Start must be called at most once per Monitor. Calling it twice would +// spawn two readers racing on the same kmsg channel and corrupt the OOM +// state machine. Callers needing a restart should construct a new +// Monitor. +// +// The goroutine shuts down when ctx is cancelled; Wait blocks until it +// returns. +func (m *Monitor) Start(ctx context.Context) error { + if m.kmsgSource == nil { + src, err := openKmsgSource(m.logger) + if err != nil { + return err + } + m.kmsgSource = src + } + m.wg.Add(1) + go func() { + defer m.wg.Done() + m.runOomLoop(ctx) + }() + return nil +} + +// Wait blocks until all goroutines launched by Start have returned. +func (m *Monitor) Wait() { m.wg.Wait() } + +// runOomLoop consumes the kmsg stream, drives the OOM state machine, and +// publishes a system_oom_kill event for each completed instance. +func (m *Monitor) runOomLoop(ctx context.Context) { + src := m.kmsgSource + // Closing the source unblocks any read in Messages() so the range + // terminates cleanly on shutdown. The done channel lets the + // watcher exit if the source closes on its own (e.g. /dev/kmsg fd + // dropped) so we don't leak the goroutine past loop exit. + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-ctx.Done(): + _ = src.Close() + case <-done: + } + }() + + m.logger.Debug("sysmon: kmsg OOM reader started") + + var s oomScanner + for msg := range src.Messages() { + oom := s.feed(msg.Body, msg.Timestamp) + if oom == nil { + continue + } + m.publishOomKill(*oom) + } + + m.logger.Debug("sysmon: kmsg OOM reader stopped") +} + +func (m *Monitor) publishOomKill(oom OomInstance) { + data := oapi.BrowserSystemOomKillEventData{ + ProcessName: oom.ProcessName, + Pid: oom.Pid, + RssKb: oom.RssKb, + } + if oom.Constraint != "" { + c := oapi.BrowserSystemOomKillEventDataConstraint(oom.Constraint) + // Drop unknown constraint values from the payload rather than + // emitting a non-enum string that SDKs may reject. The raw + // kernel label still reaches structured logs below. + if c.Valid() { + data.Constraint = &c + } else { + m.logger.Warn("sysmon: unknown OOM constraint, omitting from payload", "constraint", oom.Constraint) + } + } + if oom.MemTotalKb > 0 { + v := oom.MemTotalKb + data.MemTotalKb = &v + } + if oom.MemFreeKb > 0 { + v := oom.MemFreeKb + data.MemFreeKb = &v + } + if len(oom.TopTasks) > 0 { + tasks := make([]oapi.BrowserSystemOomKillTask, len(oom.TopTasks)) + for i, t := range oom.TopTasks { + tasks[i] = oapi.BrowserSystemOomKillTask{ + Pid: t.Pid, + Name: t.Name, + RssKb: t.RssKb, + } + } + data.TopTasks = &tasks + } + if oom.TriggerProcessName != "" { + v := oom.TriggerProcessName + data.TriggerProcessName = &v + } + if oom.TriggerPid > 0 { + v := oom.TriggerPid + data.TriggerPid = &v + } + + payload, err := json.Marshal(data) + if err != nil { + m.logger.Warn("sysmon: marshal oom kill payload", "err", err) + return + } + srcEvent := "linux.oom_kill" + ev := events.Event{ + Ts: oom.TimeOfDeath.UnixMicro(), + Type: string(oapi.SystemOomKill), + Category: events.System, + Source: oapi.BrowserEventSource{ + Kind: oapi.LocalProcess, + Event: &srcEvent, + }, + Data: json.RawMessage(payload), + } + m.es.Publish(events.Envelope{Event: ev}) + m.logger.Debug("sysmon: emitted system_oom_kill", + "process", oom.ProcessName, + "pid", oom.Pid, + ) +} diff --git a/server/lib/sysmon/sysmon_test.go b/server/lib/sysmon/sysmon_test.go new file mode 100644 index 00000000..94589beb --- /dev/null +++ b/server/lib/sysmon/sysmon_test.go @@ -0,0 +1,167 @@ +package sysmon + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kernel/kernel-images/server/lib/events" + oapi "github.com/kernel/kernel-images/server/lib/oapi" +) + +// stubKmsgSource pushes synthetic kmsg messages through an in-memory +// channel. Closing the source via Close() (typically triggered by the +// Monitor's ctx-done watcher) terminates the message channel so the +// reader goroutine exits cleanly. +type stubKmsgSource struct { + ch chan KmsgMessage + closed chan struct{} +} + +func newStubKmsgSource() *stubKmsgSource { + return &stubKmsgSource{ + ch: make(chan KmsgMessage, 32), + closed: make(chan struct{}), + } +} + +func (s *stubKmsgSource) Messages() <-chan KmsgMessage { return s.ch } + +func (s *stubKmsgSource) Close() error { + select { + case <-s.closed: + default: + close(s.closed) + close(s.ch) + } + return nil +} + +func (s *stubKmsgSource) send(body string, ts time.Time) { + s.ch <- KmsgMessage{Body: body, Timestamp: ts} +} + +func TestMonitorPublishesOomKillEnd2End(t *testing.T) { + es, err := events.NewEventStream(events.EventStreamConfig{RingCapacity: 16}) + require.NoError(t, err) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + src := newStubKmsgSource() + mon := New(es, logger, withKmsgSource(src)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, mon.Start(ctx)) + + ts := time.Unix(1_700_000_000, 0) + for _, line := range canonicalOomDump { + src.send(line, ts) + } + + reader := es.NewReader(0) + readCtx, readCancel := context.WithTimeout(ctx, 2*time.Second) + defer readCancel() + res, err := reader.Read(readCtx) + require.NoError(t, err) + + ev := res.Envelope.Event + assert.Equal(t, string(oapi.SystemOomKill), ev.Type) + assert.Equal(t, events.System, ev.Category) + assert.Equal(t, oapi.LocalProcess, ev.Source.Kind) + require.NotNil(t, ev.Source.Event) + assert.Equal(t, "linux.oom_kill", *ev.Source.Event) + assert.Equal(t, ts.UnixMicro(), ev.Ts) + + var data oapi.BrowserSystemOomKillEventData + require.NoError(t, json.Unmarshal(ev.Data, &data)) + assert.Equal(t, "chromium", data.ProcessName) + assert.Equal(t, 1234, data.Pid) + assert.Equal(t, 4823900+100+200, data.RssKb) + require.NotNil(t, data.Constraint) + assert.Equal(t, oapi.BrowserSystemOomKillEventDataConstraint("none"), *data.Constraint) + + // Mem-Info and Tasks-state fields round-trip through json correctly. + require.NotNil(t, data.MemTotalKb) + assert.Equal(t, 524288*4, *data.MemTotalKb) + require.NotNil(t, data.MemFreeKb) + assert.Equal(t, 4560*4, *data.MemFreeKb) + require.NotNil(t, data.TopTasks) + require.Len(t, *data.TopTasks, 4) + assert.Equal(t, "chromium", (*data.TopTasks)[0].Name) + + // Trigger fields round-trip too. In the canonical dump the trigger + // and the victim are the same process. + require.NotNil(t, data.TriggerProcessName) + assert.Equal(t, "chromium", *data.TriggerProcessName) + require.NotNil(t, data.TriggerPid) + assert.Equal(t, 1234, *data.TriggerPid) +} + +func TestMonitorOmitsUnknownConstraint(t *testing.T) { + // constraintFromKernel passes through unknown labels lowercased so + // they reach logs, but they would violate the openapi enum if + // emitted on the wire. The publisher must drop them rather than + // produce a non-enum value that SDKs may reject. + dump := []string{ + `x invoked oom-killer: gfp_mask=0, order=0, oom_score_adj=0`, + `oom-kill:constraint=CONSTRAINT_FUTURE_THING,task=x,pid=1,uid=0`, + `Out of memory: Killed process 1 (x) total-vm:0kB, anon-rss:1kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:0kB oom_score_adj:0`, + } + + es, err := events.NewEventStream(events.EventStreamConfig{RingCapacity: 4}) + require.NoError(t, err) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + src := newStubKmsgSource() + mon := New(es, logger, withKmsgSource(src)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, mon.Start(ctx)) + + ts := time.Unix(1_700_000_000, 0) + for _, line := range dump { + src.send(line, ts) + } + + reader := es.NewReader(0) + readCtx, readCancel := context.WithTimeout(ctx, 2*time.Second) + defer readCancel() + res, err := reader.Read(readCtx) + require.NoError(t, err) + + var data oapi.BrowserSystemOomKillEventData + require.NoError(t, json.Unmarshal(res.Envelope.Event.Data, &data)) + assert.Nil(t, data.Constraint, "unknown constraint must be omitted from the payload") +} + +func TestMonitorShutsDownOnContextCancel(t *testing.T) { + es, err := events.NewEventStream(events.EventStreamConfig{RingCapacity: 4}) + require.NoError(t, err) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + src := newStubKmsgSource() + mon := New(es, logger, withKmsgSource(src)) + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, mon.Start(ctx)) + + cancel() + + done := make(chan struct{}) + go func() { + mon.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("monitor did not shut down within 2s of context cancellation") + } +} diff --git a/server/openapi.yaml b/server/openapi.yaml index 7f56f440..fa1200be 100644 --- a/server/openapi.yaml +++ b/server/openapi.yaml @@ -2688,6 +2688,162 @@ components: truncated: type: boolean description: True if the data field was truncated due to size limits. + BrowserSystemOomKillEventData: + type: object + description: Per-kill payload for `system_oom_kill` events. + additionalProperties: false + required: [process_name, pid, rss_kb] + properties: + process_name: + type: string + description: Comm of the killed process as reported by the kernel (max 15 chars, truncated by the kernel). + pid: + type: integer + description: PID of the killed process. + rss_kb: + type: integer + description: Resident set size of the killed process in KiB (sum of anon-rss, file-rss, and shmem-rss). This is the physical memory the process was using at the time of the kill. + trigger_process_name: + type: string + description: > + Comm of the process whose allocation request caused the + kernel to invoke the OOM-killer. Often the same as + `process_name` (the kernel killed the requester) but can + differ when the kernel chose a different victim. Max 15 + chars, truncated by the kernel. + trigger_pid: + type: integer + description: > + PID of the triggering process. Absent if the kernel did + not emit the standard `CPU: N PID: N Comm:` header line. + constraint: + type: string + description: > + Why the kernel decided to OOM-kill. `none` means global memory + exhaustion; `memcg` means a cgroup memory limit was hit; + `cpuset` / `memory_policy` are NUMA/policy-driven kills. + Absent on kernels older than 5.0 which did not emit the + structured `oom-kill:` line. + enum: + - none + - memcg + - cpuset + - memory_policy + mem_total_kb: + type: integer + description: > + Total system memory in KiB at the time of the kill, derived + from the `N pages RAM` line in the kernel's Mem-Info dump. + Assumes a 4 KiB page size. Absent if the kernel did not + emit a parseable Mem-Info section. + mem_free_kb: + type: integer + description: > + Free system memory in KiB at the time of the kill, derived + from the `free:N` field in the kernel's Mem-Info dump. + Assumes a 4 KiB page size. Does not include reclaimable + caches, so a small value with a large `mem_total_kb` may + still mean the system was not under hard pressure. Absent + if the kernel did not emit a parseable Mem-Info section. + top_tasks: + type: array + maxItems: 5 + description: > + Top processes by resident-set-size at the moment of the + kill, sorted descending. Sourced from the kernel's + `Tasks state` table. Empty if the kernel did not emit the + table. Capped at 5 entries to bound payload size. + items: + $ref: "#/components/schemas/BrowserSystemOomKillTask" + BrowserSystemOomKillTask: + type: object + description: A single process entry from the kernel's `Tasks state` dump. + additionalProperties: false + required: [pid, name, rss_kb] + properties: + pid: + type: integer + description: PID of the process. + name: + type: string + description: Comm of the process (max 15 chars, truncated by the kernel). + rss_kb: + type: integer + description: Resident set size in KiB at the moment of the kill. + BrowserSystemOomKillEvent: + type: object + description: > + The Linux kernel OOM-killer terminated a process inside the VM. Sourced from + `/dev/kmsg`. Fires for any process killed by the kernel due to memory + exhaustion, including Chrome renderer subprocesses that are not supervised. + required: [ts, type, source] + properties: + ts: + type: integer + format: int64 + description: Event timestamp in Unix microseconds. + type: + type: string + const: system_oom_kill + source: + $ref: "#/components/schemas/BrowserEventSource" + data: + $ref: "#/components/schemas/BrowserSystemOomKillEventData" + truncated: + type: boolean + description: True if the data field was truncated due to size limits. + BrowserServiceCrashedEventData: + type: object + description: > + Per-crash payload for `service_crashed` events. Exit code and + signal are not exposed by the underlying process manager on this + channel, so only the service identity, the lifecycle phase the + crash occurred in, and (when available) the PID are reported. + additionalProperties: false + required: [service_name, phase] + properties: + service_name: + type: string + description: Program name of the crashed service (e.g. `chromium`, `mutter`, `kernel-images-api`). + pid: + type: integer + description: PID of the crashed process. Absent when the process manager gave up after exhausting restart attempts and is no longer tracking a live PID. + phase: + type: string + description: > + Lifecycle phase the crash occurred in. + `startup` means the process died before it ever reached a + healthy running state. `running` means a previously healthy + process died unexpectedly. `gave_up` means the process + manager exhausted its restart attempts and stopped trying; + no further `service_crashed` events will fire for this + service until something restarts it. + enum: + - startup + - running + - gave_up + BrowserServiceCrashedEvent: + type: object + description: > + A managed service exited unexpectedly. Intentional stops (e.g. + operator-initiated shutdown) do not produce this event — only + unexpected exits and terminal restart-give-up transitions do. + required: [ts, type, source] + properties: + ts: + type: integer + format: int64 + description: Event timestamp in Unix microseconds. + type: + type: string + const: service_crashed + source: + $ref: "#/components/schemas/BrowserEventSource" + data: + $ref: "#/components/schemas/BrowserServiceCrashedEventData" + truncated: + type: boolean + description: True if the data field was truncated due to size limits. KnownBrowserTelemetryEvent: description: > Discriminated union of browser telemetry events emitted by the Kernel @@ -2726,6 +2882,8 @@ components: - $ref: "#/components/schemas/BrowserLiveViewConnectEvent" - $ref: "#/components/schemas/BrowserLiveViewDisconnectEvent" - $ref: "#/components/schemas/BrowserCaptchaSolveResultEvent" + - $ref: "#/components/schemas/BrowserSystemOomKillEvent" + - $ref: "#/components/schemas/BrowserServiceCrashedEvent" discriminator: propertyName: type mapping: @@ -2757,6 +2915,8 @@ components: live_view_connect: "#/components/schemas/BrowserLiveViewConnectEvent" live_view_disconnect: "#/components/schemas/BrowserLiveViewDisconnectEvent" captcha_solve_result: "#/components/schemas/BrowserCaptchaSolveResultEvent" + system_oom_kill: "#/components/schemas/BrowserSystemOomKillEvent" + service_crashed: "#/components/schemas/BrowserServiceCrashedEvent" TelemetryState: type: object description: Current telemetry configuration.