From b0029d311dbe2d74cf362da763f90065e01cee6f Mon Sep 17 00:00:00 2001 From: kahhaw9368 Date: Mon, 22 Jun 2026 11:17:43 +0800 Subject: [PATCH] feat(skills): Add EKS Operation Review skill Integrate the EKS Operation Review skill as a sixth portable skill under skills/, mirroring the existing CloudOps skill conventions. The skill runs a structured 10-section operational excellence assessment against a live EKS cluster and produces a GREEN/AMBER/RED rated report with prioritized recommendations. All operations are read-only. What's included: - SKILL.md with CloudOps-style frontmatter (user-invocable, argument-hint, allowed-tools), a 3-path routing block (EKS MCP tools -> AWS CLI+kubectl -> delegate), and a Constraints section (read-only, no fabrication, reports written inside the workspace only). - 11 steering files (one per assessment section + report-generation) and the report_to_html.py converter, copied from the source skill. - reference/workflow.md (ported from the source .claude/commands entry point, paths adapted to the CloudOps layout) and reference/mcp-setup.md + reference/mcp.json.example for the two pinned MCP servers (awslabs.eks-mcp-server@0.1.28, awslabs.aws-documentation-mcp-server@1.1.21). - README.md skills table + project-structure entries. Skill-only change: no agent hierarchy, Lambda tool, or Terraform changes. Verified end-to-end against a live cluster (Section 06 Networking): steering loads from the new path, MCP checks execute, the rated report writes to reports/, and HTML conversion succeeds. --- README.md | 2 + skills/eks-operation-review/SKILL.md | 75 +++++ .../reference/mcp-setup.md | 57 ++++ .../reference/mcp.json.example | 24 ++ .../reference/workflow.md | 271 ++++++++++++++++ .../steering/access-identity.md | 97 ++++++ .../steering/addon-management.md | 70 +++++ .../steering/autoscaling.md | 78 +++++ .../steering/cluster-lifecycle.md | 132 ++++++++ .../steering/deployment-practices.md | 75 +++++ .../steering/infrastructure-as-code.md | 87 ++++++ .../steering/networking.md | 81 +++++ .../steering/observability.md | 93 ++++++ .../steering/operational-processes.md | 95 ++++++ .../steering/report-generation.md | 164 ++++++++++ .../steering/workload-configuration.md | 122 ++++++++ .../tools/report_to_html.py | 295 ++++++++++++++++++ 17 files changed, 1818 insertions(+) create mode 100644 skills/eks-operation-review/SKILL.md create mode 100644 skills/eks-operation-review/reference/mcp-setup.md create mode 100644 skills/eks-operation-review/reference/mcp.json.example create mode 100644 skills/eks-operation-review/reference/workflow.md create mode 100644 skills/eks-operation-review/steering/access-identity.md create mode 100644 skills/eks-operation-review/steering/addon-management.md create mode 100644 skills/eks-operation-review/steering/autoscaling.md create mode 100644 skills/eks-operation-review/steering/cluster-lifecycle.md create mode 100644 skills/eks-operation-review/steering/deployment-practices.md create mode 100644 skills/eks-operation-review/steering/infrastructure-as-code.md create mode 100644 skills/eks-operation-review/steering/networking.md create mode 100644 skills/eks-operation-review/steering/observability.md create mode 100644 skills/eks-operation-review/steering/operational-processes.md create mode 100644 skills/eks-operation-review/steering/report-generation.md create mode 100644 skills/eks-operation-review/steering/workload-configuration.md create mode 100644 skills/eks-operation-review/tools/report_to_html.py diff --git a/README.md b/README.md index adcc9cb..fdeeae7 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,7 @@ docs/ architecture.md # Platform internals skills/ developer-guide/SKILL.md # Adding agents / tools / collectors (interactive) + eks-operation-review/SKILL.md # EKS operational excellence assessment (10 areas, rated report) agents/ # Per-leaf-agent references (deploy modes, data model, gotchas) README.md # When to add a file + section template cost-operations.md # Cost Explorer / CUR / COH reference @@ -391,6 +392,7 @@ Skills provide the platform's analytical capabilities as portable workflows that | `/finops-analysis` | "what are my top costs?" | Cost breakdown, trends, forecasts, savings | | `/health-events-digest` | "any critical health events?" | Health event digest with risk scoring | | `/tag-governance-assessment` | "how's my tag compliance?" | Tag compliance scoring + remediation links | +| `/eks-operation-review` | "review my EKS cluster" | EKS operational excellence assessment (10 areas, GREEN/AMBER/RED rated report) | ### How to use skills diff --git a/skills/eks-operation-review/SKILL.md b/skills/eks-operation-review/SKILL.md new file mode 100644 index 0000000..1287471 --- /dev/null +++ b/skills/eks-operation-review/SKILL.md @@ -0,0 +1,75 @@ +--- +name: eks-operation-review +description: "Run a structured EKS operational excellence assessment against a live cluster. Covers 10 areas — cluster lifecycle, IaC/GitOps, access & identity, observability, workload config, networking, autoscaling, deployment practices, operational processes, and add-on management — and produces a GREEN/AMBER/RED rated report with prioritized recommendations. Three paths: (1) EKS MCP tools via awslabs.eks-mcp-server (preferred); (2) direct AWS CLI + kubectl; (3) delegation to a coding agent. Activate for any request to audit, review, health-check, or score an EKS cluster's operational posture, including section-scoped reviews of individual areas. Not for upgrade readiness, cluster discovery, or architectural design advice." +argument-hint: "[which cluster / which area? e.g. 'review my EKS cluster', 'check EKS networking', 'score RBAC on my cluster']" +allowed-tools: Bash, Write, Read, Glob, Grep +user-invocable: true +--- + +# EKS Operation Review + +This skill performs a structured 10-section operational assessment of a live EKS cluster, producing a GREEN/AMBER/RED rated report with prioritized recommendations. Checks are informed by the [EKS Best Practices Guide](https://docs.aws.amazon.com/eks/latest/best-practices/) and [EKS User Guide](https://docs.aws.amazon.com/eks/latest/userguide/). All operations are **read-only**. + +## When to use + +Activate for any request to audit, review, health-check, or score an EKS cluster's operational posture — including section-scoped reviews of individual areas (e.g., "check my EKS networking", "review RBAC on my cluster", "is my EKS cluster following best practices", "EKS operational health check"). + +Not for: upgrade readiness assessments, cluster discovery, or architectural design advice. General Kubernetes questions, AWS troubleshooting, cluster creation, and one-off kubectl commands should be handled directly without this skill. + +## Routing — read first, before anything else + +This skill works in three environments. Decide which path you are on **before** you start. The decisive signal at each step is what is reachable from the current environment — not user preference. + +``` +Are EKS MCP tools available (list_k8s_resources, describe_eks_resource, +list_eks_resources, get_eks_insights, ...)? +├── Yes → Path M (use awslabs.eks-mcp-server tools — preferred, richest cluster access) +└── No + ├── Can you run `aws eks list-clusters` AND `kubectl`? + │ ├── Yes → Path A (AWS CLI + kubectl directly) + │ └── No + │ ├── Is a coding agent enabled? → Path B (delegate) + │ └── No → Stop. Tell the user: "I can't reach an EKS cluster from + │ here. Enable the awslabs.eks-mcp-server MCP server + │ (see .kiro/settings/mcp.json) and ensure AWS + │ credentials + Kubernetes RBAC access, or run in a + │ shell-capable agent with kubectl." +``` + +### Path M — EKS MCP tools (preferred) + +The `awslabs.eks-mcp-server` exposes the cluster-state tools the assessment relies on — `list_eks_resources`, `describe_eks_resource`, `list_k8s_resources`, `read_k8s_resource`, `get_eks_insights`, `get_eks_vpc_config`, `get_cloudwatch_logs`/`get_cloudwatch_metrics`, `get_policies_for_role`, etc. Use these directly. AWS documentation lookups (when needed) use `awslabs.aws-documentation-mcp-server`. Both are declared in `.kiro/settings/mcp.json`. + +Follow the workflow in `reference/workflow.md`. Per its tool-usage rules: do NOT call any tool on activation, discover clusters first (`aws eks list-clusters`), never use `manage_eks_stacks` for discovery, and do NOT retry a failed MCP call more than once. + +### Path A — AWS CLI + kubectl + +If the MCP tools aren't available but the AWS CLI and `kubectl` are reachable, run the same checks yourself: `aws eks list-clusters` / `describe-cluster` / `describe-addon` / `list-insights` for the AWS control-plane surface, and `kubectl get/describe` for the Kubernetes-level checks each steering file enumerates. Raw CLI output won't be pre-enriched, so assess each item against the steering rubric manually. + +### Path B — Delegate + +In a sandboxed host that cannot reach AWS or the MCP server but has a coding agent with shell + AWS/kubectl access, hand the coding agent this brief: +> "Run an EKS operational excellence assessment for the user's selected cluster by following the skill specification on disk at `skills/eks-operation-review/SKILL.md` and `skills/eks-operation-review/reference/workflow.md`. Execute Path A (AWS CLI + kubectl). Load each steering file from `skills/eks-operation-review/steering/` before its section. Read-only only. Produce the rated report per `steering/report-generation.md` and save it to `reports/`. Return the report path and the top findings by severity." + +Relay only the coding agent's final summary back to the user. + +## Instructions + +Read and follow `reference/workflow.md` — it contains the full workflow (pre-flight, sections 1–10, report generation), tool-usage rules, the scenario→steering-file map, the rating rubric, and the report format. Load each steering file from `skills/eks-operation-review/steering/` before running its corresponding section. + +## Prerequisites + +- AWS credentials with EKS **read** access (`aws sts get-caller-identity` succeeds) +- Kubernetes RBAC read access to the target cluster (EKS access entry or `aws-auth`) +- Python 3.10+ and uv installed (to run the EKS MCP server) +- EKS MCP servers configured for your IDE — see `reference/mcp-setup.md` (copy `reference/mcp.json.example` to `.kiro/settings/mcp.json`). Pinned: `awslabs.eks-mcp-server@0.1.28`, `awslabs.aws-documentation-mcp-server@1.1.21` + +## Constraints + +- **Read-only.** Use only describe/list/get/read operations. Never call any AWS or Kubernetes API that mutates state (no Create*/Modify*/Delete*/Update*/Put*/apply/patch/scale). +- **Never fabricate findings, ratings, counts, versions, or cluster facts.** Every rating must be grounded in data actually returned by a tool/CLI call this run. If a check returns no data or fails, mark the item **UNKNOWN** — do not guess. +- **Discover, don't assume.** Always list clusters first; never hardcode or guess a cluster name. +- **Do not fabricate AWS documentation URLs.** Use only the pre-verified reference map in `steering/report-generation.md`; fall back to the section-level page when no specific URL matches. +- **Write reports only inside the workspace.** Output to the repo root or a `reports/` subfolder using the filename format `EKS-Operation-Review---.md`. Never write outside the workspace. +- **One cluster at a time.** Re-run the skill for additional clusters. +- **Don't retry a failed MCP/CLI call more than once.** If it fails twice, stop and surface troubleshooting steps. diff --git a/skills/eks-operation-review/reference/mcp-setup.md b/skills/eks-operation-review/reference/mcp-setup.md new file mode 100644 index 0000000..b3358ee --- /dev/null +++ b/skills/eks-operation-review/reference/mcp-setup.md @@ -0,0 +1,57 @@ +# EKS MCP server setup + +The EKS Operation Review skill's preferred execution path (Path M) uses two MCP +servers. This file explains how to make them available to your IDE. + +## Servers used + +| Server | Purpose | Pinned version | +|--------|---------|----------------| +| `awslabs.eks-mcp-server` | Live EKS cluster + Kubernetes resource access | `0.1.28` | +| `awslabs.aws-documentation-mcp-server` | AWS documentation lookups during assessment | `1.1.21` | + +Versions are **pinned** to keep behaviour reproducible and avoid pulling +unreviewed upstream updates. To upgrade, bump the version strings after +reviewing the upstream changelog at . + +## Install (Kiro) + +Kiro merges MCP config from the user level (`~/.kiro/settings/mcp.json`) and the +workspace level (`.kiro/settings/mcp.json`), with workspace taking precedence. + +Copy the bundled example into your workspace settings: + +```bash +mkdir -p .kiro/settings +cp skills/eks-operation-review/reference/mcp.json.example .kiro/settings/mcp.json +``` + +> `.kiro/settings/` holds machine-local settings and is intentionally NOT +> committed to this repository — keep your real `mcp.json` out of version +> control. The committed artifact is the `.example` template only. + +Alternatively, add the two servers via Kiro's **Settings → MCP Servers** UI using +the same `command` / `args` / `env` shown in `mcp.json.example`. + +## Install (Claude Code) + +Add the two servers to `.mcp.json` at the project root (same JSON shape as +`mcp.json.example`). Claude Code will prompt to enable them on next launch. + +## Prerequisites for the servers + +- Python 3.10+ and [uv](https://docs.astral.sh/uv/getting-started/installation/) + installed (`uvx` runs the pinned server packages on demand). +- Working AWS credentials — `aws sts get-caller-identity` must succeed. The + servers read `AWS_PROFILE` / `AWS_REGION` from the environment; set them in the + `env` block of `mcp.json` if you need a specific profile or region. +- Kubernetes RBAC read access to the target cluster (EKS access entry or + `aws-auth` ConfigMap). + +## Verify + +Do NOT treat editing the config as a verification step. Confirm the server works +by making one live tool call — e.g. discover clusters with +`aws eks list-clusters`, then call `list_k8s_resources` against a discovered +cluster. If a call fails, see the troubleshooting steps in +`reference/workflow.md`. diff --git a/skills/eks-operation-review/reference/mcp.json.example b/skills/eks-operation-review/reference/mcp.json.example new file mode 100644 index 0000000..359b229 --- /dev/null +++ b/skills/eks-operation-review/reference/mcp.json.example @@ -0,0 +1,24 @@ +{ + "mcpServers": { + "awslabs.eks-mcp-server": { + "command": "uvx", + "args": [ + "awslabs.eks-mcp-server@0.1.28" + ], + "env": { + "FASTMCP_LOG_LEVEL": "ERROR" + }, + "disabled": false + }, + "awslabs.aws-documentation-mcp-server": { + "command": "uvx", + "args": [ + "awslabs.aws-documentation-mcp-server@1.1.21" + ], + "env": { + "FASTMCP_LOG_LEVEL": "ERROR" + }, + "disabled": false + } + } +} diff --git a/skills/eks-operation-review/reference/workflow.md b/skills/eks-operation-review/reference/workflow.md new file mode 100644 index 0000000..44191b8 --- /dev/null +++ b/skills/eks-operation-review/reference/workflow.md @@ -0,0 +1,271 @@ +# EKS Operation Review — Workflow + +> Loaded by `skills/eks-operation-review/SKILL.md`. This file contains the full +> assessment workflow, tool-usage rules, steering-file map, rating rubric, and +> report format. All paths below are relative to the repository root. + +## Tool Usage Rules + +1. **Do NOT call any tools when this skill is first activated.** Wait for the user to explicitly ask for a review. +2. **Do NOT read mcp.json or config files as a "check".** The only way to verify the MCP server works is to call an actual tool. +3. **Do NOT hardcode or guess cluster names.** Always discover clusters by listing them first. +4. **Do NOT retry a failed MCP tool call more than once.** If it fails twice, stop and show troubleshooting steps. +5. **Always load the relevant steering file before executing checks for that section.** +6. **Do NOT use the `manage_eks_stacks` MCP tool for cluster discovery or description.** That tool manages CloudFormation stacks, not EKS clusters. Use `aws eks list-clusters` and `aws eks describe-cluster` via Bash for pre-flight, and `list_k8s_resources` for Kubernetes checks. + +## Steering File Loading + +Before executing checks for any section, read the corresponding steering file from +`skills/eks-operation-review/steering/` using the Read tool. + +### Scenario to Steering File Map + +| User Request | Steering File(s) to Load | +|---|---| +| Full review / assess / audit / health check | ALL files in order: cluster-lifecycle -> addon-management, then report-generation | +| Upgrade / version / deprecated API | `skills/eks-operation-review/steering/cluster-lifecycle.md` | +| IRSA / RBAC / access / pod identity / endpoint | `skills/eks-operation-review/steering/access-identity.md` | +| Logging / metrics / alerting / observability | `skills/eks-operation-review/steering/observability.md` | +| Resource requests / probes / PDB / image tags / storage | `skills/eks-operation-review/steering/workload-configuration.md` | +| IP / subnet / DNS / CoreDNS / network policy | `skills/eks-operation-review/steering/networking.md` | +| Autoscaling / Karpenter / HPA / topology spread | `skills/eks-operation-review/steering/autoscaling.md` | +| Deployment / rollout / CI/CD / graceful shutdown | `skills/eks-operation-review/steering/deployment-practices.md` | +| Runbook / on-call / backup / DR / Velero | `skills/eks-operation-review/steering/operational-processes.md` | +| Add-on / node monitoring / cluster insights | `skills/eks-operation-review/steering/addon-management.md` | +| Generate / write report | `skills/eks-operation-review/steering/report-generation.md` | +| IaC / GitOps / ArgoCD / Flux / drift | `skills/eks-operation-review/steering/infrastructure-as-code.md` | + +--- + +## Overview + +This skill assesses your live EKS cluster against 10 areas of operational best practice. It connects to your cluster via the EKS MCP server, runs automated checks, rates each item as GREEN/AMBER/RED, and produces a report with prioritized recommendations and AWS documentation links. + +## What Gets Assessed + +| # | Section | Key Checks | +|---|---------|------------| +| 01 | Cluster Lifecycle & Upgrades | Version currency, data plane alignment, deprecated APIs, add-on compatibility, upgrade process | +| 02 | Infrastructure as Code & GitOps | IaC provenance, GitOps tools, drift detection, RBAC in code | +| 03 | Access & Identity | IRSA/Pod Identity, least privilege RBAC, API server endpoint security, Pod Security Admission | +| 04 | Observability | Control plane logging, metrics stack, log aggregation, alerting | +| 05 | Workload Configuration | Resource requests/limits, health probes, PDBs, image tags, storage | +| 06 | Networking | IP capacity, CoreDNS health, network policies | +| 07 | Autoscaling | Cluster autoscaler/Karpenter, HPA, topology spread | +| 08 | Deployment Practices | Rollout strategy, CI/CD, graceful shutdown | +| 09 | Operational Processes | Backup/DR, tool presence (Velero, AWS Backup) | +| 10 | Add-on Management | Managed add-ons, node health monitoring, cluster insights | + +~70-75% of items are fully automatable. Items that require human knowledge (runbooks, on-call processes) are marked UNKNOWN with suggestions for what to investigate. + +## Prerequisites + +1. **AWS credentials configured** -- `aws configure` or `~/.aws/credentials` with a profile that has EKS access +2. **Python 3.10+** and **uv** installed ([Install uv](https://docs.astral.sh/uv/getting-started/installation/)) -- required to run the EKS MCP server +3. **Required AWS Permissions**: + - `eks:Describe*`, `eks:List*` + - `ec2:DescribeSubnets`, `ec2:DescribeSecurityGroupRules` + - `ecr:DescribeRepositories` + - `iam:ListAttachedRolePolicies`, `iam:ListRolePolicies` + - `logs:DescribeLogGroups` + - `cloudwatch:DescribeAlarms` + - `backup:ListBackupPlans` (optional) + +## MCP Server Configuration + +The MCP servers use your existing AWS credentials from the environment (`~/.aws/credentials`, `AWS_PROFILE`, `AWS_REGION`). No additional configuration is needed if your AWS CLI already works. + +The EKS MCP servers for this workspace are declared in `.kiro/settings/mcp.json`. If you +need to use a specific profile or region, update the EKS MCP server's `env` block there: + +```json +"env": { + "AWS_PROFILE": "your-profile-name", + "AWS_REGION": "your-region", + "FASTMCP_LOG_LEVEL": "ERROR" +} +``` + +## Getting Started + +Run: `/eks-operation-review` + +The skill will automatically discover your clusters and walk you through the assessment. + +--- + +## Assessment Workflow + +### Step 0: Pre-flight + +This step verifies everything works before starting the assessment. Follow this exact sequence: + +**Action 1 -- List clusters (discovers clusters)** + +Use the AWS CLI to list clusters. Do NOT use the `manage_eks_stacks` MCP tool — it manages CloudFormation stacks, not cluster discovery. + +``` +aws eks list-clusters --output json +``` + +- Success -> Show the cluster list. Ask the user which cluster to assess. If only one cluster, confirm: "I found one cluster: [name]. Shall I assess this one?" +- Failure -> STOP. Do NOT retry more than once. Do NOT read config files. Show: + +> **Cannot list EKS clusters.** Try these steps: +> 1. Check that AWS credentials work: `aws sts get-caller-identity` +> 2. Check the region: `aws configure get region` +> 3. Verify AWS_PROFILE and AWS_REGION in `.kiro/settings/mcp.json` + +Wait for the user to resolve the issue. + +**Action 2 -- Describe the selected cluster** + +Use the AWS CLI to describe the cluster. Do NOT use `manage_eks_stacks` — it looks for CloudFormation stacks, not EKS clusters. + +``` +aws eks describe-cluster --name --output json +``` + +From the response, show: +- Cluster name, Kubernetes version, platform version, region, status +- AWS account ID +- Authentication mode + +**Action 3 -- Verify MCP connectivity** + +Call one MCP tool to confirm the EKS MCP server works (e.g., list Nodes): + +``` +list_k8s_resources(cluster_name="", kind="Node", api_version="v1") +``` + +- Success -> MCP server is working. Proceed. +- Failure -> STOP. Show: + +> **The EKS MCP server isn't responding.** Try these steps: +> 1. Check that Python 3.10+ and uv are installed: `uv --version` +> 2. Test the MCP server: `uvx awslabs.eks-mcp-server@0.1.28` +> 3. Verify AWS_PROFILE and AWS_REGION in `.kiro/settings/mcp.json` + +Wait for the user to resolve the issue. + +**Action 4 -- Confirm** + +Ask: *"Ready to start the assessment on [cluster-name] (v[version])?"* + +Proceed only after the user confirms. + +### Steps 1-10: Run Assessment + +Read each steering file in section order using the Read tool. For each section: +1. Read the steering file from `skills/eks-operation-review/steering/` +2. Execute the checks described in it +3. Rate each item using the rubric below + +**Error recovery:** If a section fails entirely (MCP server unreachable, permissions denied for all checks in that section, or repeated timeouts), mark all items in that section as UNKNOWN with a note explaining the failure reason, then proceed to the next section. Do not let one failed section block the rest of the assessment. + +### Step 11: Generate Report + +Read `skills/eks-operation-review/steering/report-generation.md` and produce the report. + +--- + +## Rating Rubric + +| Rating | Meaning | +|--------|---------| +| GREEN | Fully implemented -- matches EKS best practices | +| AMBER | Partial or inconsistent -- improvement opportunity | +| RED | Not implemented or significant gap -- action needed | +| UNKNOWN | Cannot be determined from cluster data -- investigate manually | + +### Rules + +- Only rate based on what was actually observed -- never assume +- If a check fails or returns no data, mark UNKNOWN +- Prioritize by blast radius: security > availability > cost +- Every RED finding must have a specific, actionable recommendation + +--- + +## Report Format + +### Consistency Rules (MANDATORY) + +1. **Ratings must be consistent across the entire report.** If 4.1 is RED in the findings table, it must appear as RED everywhere -- executive summary, prioritized actions, quick wins. +2. **Prioritized Actions must reference the finding ID.** Write "4.1 -- Control Plane Logging RED" not just "Enable logging". +3. **Every RED must appear in Critical or Important.** Every AMBER must appear in Important or Quick Wins. Nothing rated RED/AMBER can be missing from Prioritized Actions. +4. **Executive Summary must match the findings.** Do not call something a "critical gap" if it's AMBER, or skip a RED item. + +### File Output + +- **Location:** Workspace root or `reports/` subfolder. Do NOT write outside the workspace. +- **Filename:** `EKS-Operation-Review---.md` + +### Template + +```markdown +# EKS Operation Review Report +Cluster: [name] | Region: [region] | Version: [version] +Date: [YYYY-MM-DD HH:MM] + +## Executive Summary +[2-3 paragraphs. Strengths first, then gaps. Every rating mentioned must match the findings tables.] + +## Maturity Score +| Rating | Count | Percentage | +|--------|-------|------------| +| GREEN | X | X% | +| AMBER | X | X% | +| RED | X | X% | +| UNKNOWN | X | -- | + +## Findings + +### Section 01 -- Cluster Lifecycle & Upgrades +| Item | Status | Current State | Recommendation | References | +|------|--------|---------------|----------------|------------| + +[Repeat for all 10 sections] + +## Prioritized Actions + +### Critical (Address within 30 days) +| # | Finding | Action | References | +|---|---------|--------|------------| +| 1 | [X.X -- Item Name] RED | [specific action] | [links] | + +### Important (Address within 90 days) +| # | Finding | Action | References | +|---|---------|--------|------------| +| 1 | [X.X -- Item Name] AMBER | [specific action] | [links] | + +### Quick Wins +| # | Finding | Action | Effort | Impact | References | +|---|---------|--------|--------|--------|------------| +| 1 | [X.X -- Item Name] | [action] | [estimate] | [what improves] | [links] | + +## Items to Investigate Manually +[UNKNOWN items with specific questions to answer] + +## AWS Reference Links +[All links grouped by topic] + +--- + +*This report was generated by a Claude Code / Kiro skill provided as sample code for educational and demonstration purposes only. Findings should be reviewed and validated before acting on them. See the project's README and LICENSE for full terms.* +``` + +### AWS References + +Use the pre-verified reference map in `skills/eks-operation-review/steering/report-generation.md` Step 7. Do NOT call the AWS Documentation MCP server during report generation — it adds latency and token cost. All URLs are pre-verified and mapped by section. + +Do NOT fabricate URLs beyond the reference map. If a finding doesn't match a specific URL, use the fallback section-level page. + +### HTML Conversion + +If the user wants HTML output, run the bundled converter — do NOT generate HTML by hand: + +```bash +python3 skills/eks-operation-review/tools/report_to_html.py .md +``` diff --git a/skills/eks-operation-review/steering/access-identity.md b/skills/eks-operation-review/steering/access-identity.md new file mode 100644 index 0000000..9e147d5 --- /dev/null +++ b/skills/eks-operation-review/steering/access-identity.md @@ -0,0 +1,97 @@ +# Access & Identity + +## Purpose +Assess IAM and RBAC configuration for security and operational excellence — pod-level permissions, least privilege, and API server access controls. + +## Checks to Execute + +### 3.1 — Pod-Level AWS Permissions (IRSA / EKS Pod Identity) + +**What to check:** +- OIDC provider configured (prerequisite for IRSA) +- EKS Pod Identity associations +- Service accounts with IRSA annotations (`eks.amazonaws.com/role-arn`) +- Node IAM role policies (are they overly broad?) +- AWS credentials in Kubernetes Secrets or pod env vars + +**How to check:** +1. Describe cluster → `identity.oidc.issuer` (OIDC configured?) +2. List Pod Identity associations +3. List ServiceAccounts across all namespaces → filter for IRSA annotation +4. List node groups → describe first one → get `nodeRole` → get policies for that role +5. List Secrets across namespaces → check for `AWS_ACCESS_KEY_ID` keys (⚠️ avoid reading Secret values) +6. List pods → check container env vars for `AWS_ACCESS_KEY_ID` + +**Rating:** +- 🟢 GREEN: All AWS-accessing pods use IRSA or Pod Identity, node role is minimal, no hardcoded credentials +- 🟡 AMBER: IRSA partially adopted, or node role has some extra permissions +- 🔴 RED: No IRSA/Pod Identity, node role has broad permissions (S3FullAccess, DynamoDBFullAccess), or hardcoded AWS credentials found +- ⬜ UNKNOWN: Cannot determine which pods need AWS access vs which don't + +**Key talking point:** Node-level IAM = every pod on that node inherits the same permissions. One compromised pod gets access to everything. + +--- + +### 3.2 — Least Privilege RBAC + +**What to check:** +- ClusterRoleBindings to cluster-admin (count and subjects) +- ClusterRoles with wildcard permissions (`*` verbs on `*` resources) +- Ratio of namespace-scoped RoleBindings vs cluster-scoped ClusterRoleBindings +- Service accounts with cluster-admin + +**How to check:** +1. List ClusterRoleBindings → filter `roleRef.name == "cluster-admin"` → count and list subjects +2. List ClusterRoles → check rules for `verbs: ["*"]` and `resources: ["*"]` +3. Count RoleBindings across all namespaces vs ClusterRoleBindings +4. List application namespaces (exclude kube-system, kube-public, kube-node-lease, default) + +**Rating:** +- 🟢 GREEN: Namespace-scoped RBAC, cluster-admin limited to 1-2 break-glass bindings, periodic review +- 🟡 AMBER: Some namespace isolation but cluster-admin overused (>3 bindings) +- 🔴 RED: Developers have cluster-admin in production, wildcard service accounts, no namespace isolation +- ⬜ UNKNOWN: Cannot determine if RBAC is reviewed periodically — suggest user investigate + +--- + +### 3.3 — EKS API Server Endpoint & Network Access + +**What to check:** +- Public/private endpoint configuration +- Public access CIDR restrictions +- Cluster security group inbound rules +- Whether audit logging is enabled + +**How to check:** +1. Describe cluster → `resourcesVpcConfig.endpointPublicAccess`, `endpointPrivateAccess`, `publicAccessCidrs` +2. Describe cluster → `logging.clusterLogging` (check if audit log type is enabled) + +**Rating:** +- 🟢 GREEN: Private endpoint enabled, public either disabled or CIDR-restricted, audit logging on +- 🟡 AMBER: Public endpoint with CIDR restrictions, or private enabled but audit logging off +- 🔴 RED: Public endpoint open to `0.0.0.0/0`, or no audit logging +- ⬜ UNKNOWN: Cannot determine MFA/SSO requirements — suggest user investigate + +**Key talking point:** An API server open to 0.0.0.0/0 is exposed to the internet. You're relying entirely on authentication. + +--- + +### 3.4 — Pod Security Admission (PSA) + +**What to check:** +- Pod Security Standards enforcement via namespace labels (`pod-security.kubernetes.io/enforce`) +- Which namespaces have PSA labels and at what level (privileged, baseline, restricted) +- Production namespaces without PSA enforcement + +**How to check:** +1. List namespaces → inspect labels for `pod-security.kubernetes.io/enforce`, `pod-security.kubernetes.io/warn`, `pod-security.kubernetes.io/audit` +2. Count namespaces with enforcement vs without (exclude kube-system, kube-public, kube-node-lease) +3. Check if any application namespaces use `privileged` enforce level + +**Rating:** +- 🟢 GREEN: PSA labels on all application namespaces, `baseline` or `restricted` enforcement +- 🟡 AMBER: PSA labels on some namespaces but not all, or only `warn`/`audit` mode (no enforcement) +- 🔴 RED: No PSA labels on any namespace, or application namespaces set to `privileged` +- ⬜ UNKNOWN: Cannot determine if third-party admission controller (OPA/Gatekeeper, Kyverno) handles pod security instead + +**Key talking point:** PodSecurityPolicy was removed in Kubernetes 1.25. Pod Security Admission is the built-in replacement. Without it (or a third-party equivalent), any pod spec is accepted — including privileged containers. diff --git a/skills/eks-operation-review/steering/addon-management.md b/skills/eks-operation-review/steering/addon-management.md new file mode 100644 index 0000000..f14281d --- /dev/null +++ b/skills/eks-operation-review/steering/addon-management.md @@ -0,0 +1,70 @@ +# Add-on & Component Management + +## Purpose +Assess add-on management maturity, node health monitoring, and cluster insights usage. + +## Checks to Execute + +### 10.1 — Core Add-ons Managed via EKS Managed Add-ons + +**What to check:** +- All EKS managed add-ons: name, version, status, health +- Compare installed versions against latest compatible for the cluster version +- Self-managed add-ons in kube-system (Helm releases) +- Deprecated in-tree EBS plugin usage + +**How to check:** +1. List addons → describe each for version, status, health issues +2. For each of the 4 core add-ons (vpc-cni, coredns, kube-proxy, aws-ebs-csi-driver): + - Check if installed as managed add-on + - Compare installed version vs latest compatible +3. List PersistentVolumes → check for `spec.awsElasticBlockStore` (deprecated in-tree) + +**Rating:** +- 🟢 GREEN: All core add-ons are EKS Managed, on latest or N-1 version, healthy +- 🟡 AMBER: Managed but behind (>1 minor version), or mix of managed and self-managed +- 🔴 RED: Core add-ons self-managed with no version tracking, health issues, or deprecated in-tree plugin +- ⬜ UNKNOWN: Cannot list add-ons + +**Key talking point:** EKS does NOT auto-update add-ons when you upgrade the control plane. Clusters upgraded to 1.31 still running vpc-cni from 1.27 is a ticking time bomb. + +--- + +### 10.2 — Node Health Monitoring & Auto-Repair + +**What to check:** +- EKS Node Monitoring Agent add-on (`eks-node-monitoring-agent`) +- Node auto-repair configuration on managed node groups +- GPU nodes (need NMA for GPU failure detection) +- Current node conditions + +**How to check:** +1. Describe addon `eks-node-monitoring-agent` +2. List node groups → describe each → check `nodeRepairConfig` +3. List nodes → check for `nvidia.com/gpu` in capacity (GPU nodes) +4. List nodes → inspect conditions for MemoryPressure, DiskPressure, etc. + +**Rating:** +- 🟢 GREEN: NMA installed and node auto-repair enabled +- 🟡 AMBER: No NMA but node conditions monitored, or NMA without auto-repair +- 🔴 RED: No node health monitoring beyond basic Kubernetes conditions, especially with GPU workloads +- ⬜ UNKNOWN: Should not happen with live access + +--- + +### 10.3 — EKS Cluster Insights Reviewed + +**What to check:** +- All cluster insights with status +- Count by status (PASSING, WARNING, ERROR) +- Details on any ERROR or WARNING insights + +**How to check:** +1. Get EKS Insights for the cluster +2. For any non-PASSING insights → get detailed description and recommendation + +**Rating:** +- 🟢 GREEN: Insights reviewed, no ERROR/WARNING, or all addressed +- 🟡 AMBER: WARNING insights unaddressed +- 🔴 RED: ERROR insights ignored +- ⬜ UNKNOWN: Insights API not accessible diff --git a/skills/eks-operation-review/steering/autoscaling.md b/skills/eks-operation-review/steering/autoscaling.md new file mode 100644 index 0000000..2331086 --- /dev/null +++ b/skills/eks-operation-review/steering/autoscaling.md @@ -0,0 +1,78 @@ +# Autoscaling + +## Purpose +Assess cluster-level autoscaling (nodes) and workload-level autoscaling (pods), plus AZ resilience. + +## Checks to Execute + +### 7.1 — Cluster Autoscaling Strategy + +**What to check:** +- Karpenter NodePools and EC2NodeClasses +- Cluster Autoscaler deployment in kube-system +- EKS Auto Mode (cluster computeConfig) +- Node group scaling config (min/max/desired) +- Spot vs On-Demand nodes +- Currently pending pods + +**How to check:** +1. List resources `nodepools.karpenter.sh` → check limits and consolidation policy. If 404/NotFound (CRD not installed) → Karpenter is not deployed, proceed to check for Cluster Autoscaler or Auto Mode. If 403/Forbidden → mark Karpenter status UNKNOWN. +2. List Deployments in kube-system → check for `cluster-autoscaler` +3. Describe cluster → `computeConfig` for Auto Mode +4. List node groups → describe each for scalingConfig and capacityType +5. List nodes → check labels for capacity type (`karpenter.sh/capacity-type` or `eks.amazonaws.com/capacityType`) +6. List pods with field selector `status.phase=Pending` + +**Rating:** +- 🟢 GREEN: Karpenter or EKS Auto Mode with consolidation enabled (AWS-preferred path) +- 🟡 AMBER: Cluster Autoscaler present (legacy — consider migration to Karpenter), or Karpenter without consolidation +- 🔴 RED: No cluster autoscaling — manual node management +- ⬜ UNKNOWN: Cannot determine scale-up latency without testing + +--- + +### 7.2 — Horizontal Pod Autoscaler (HPA) + +**What to check:** +- HPAs across all namespaces (targets, min/max, current replicas) +- Multi-replica deployments without HPA +- HPAs with minReplicas=1 (single point of failure) +- KEDA ScaledObjects +- VPA resources + +**How to check:** +1. List HPAs across all namespaces → check minReplicas, maxReplicas, current metrics +2. List Deployments with replicas > 1 → cross-reference with HPA targets +3. List HPAs where minReplicas == 1 (single point of failure for production workloads; acceptable for dev/staging) +4. List ScaledObjects (KEDA CRD). If 404/NotFound → KEDA not installed, skip. If 403/Forbidden → mark UNKNOWN. +5. List VPA resources. If 404/NotFound → VPA not installed, skip. If 403/Forbidden → mark UNKNOWN. + +**Rating:** +- 🟢 GREEN: HPAs on stateless production workloads, min >= 2, tested under load +- 🟡 AMBER: HPAs exist but min=1, or some workloads missing HPA +- 🔴 RED: No HPAs — all workloads at fixed replica count +- ⬜ UNKNOWN: Cannot determine if HPAs have been load-tested + +--- + +### 7.3 — Pod Topology Spread & AZ Resilience + +**What to check:** +- Node distribution across AZs +- Deployments with topology spread constraints +- Deployments with pod anti-affinity +- Multi-replica deployments with neither (vulnerable to AZ failure) + +**How to check:** +1. List nodes → group by label `topology.kubernetes.io/zone` +2. List Deployments → check `spec.template.spec.topologySpreadConstraints` +3. List Deployments → check `spec.template.spec.affinity.podAntiAffinity` +4. List multi-replica Deployments with neither topology spread nor anti-affinity + +**Rating:** +- 🟢 GREEN: Nodes in 3 AZs, topology spread on production deployments +- 🟡 AMBER: Nodes in multiple AZs but no topology spread constraints +- 🔴 RED: Single-AZ deployment, or multi-replica services with no AZ spread +- ⬜ UNKNOWN: Cannot verify actual pod distribution without checking pod-to-node mapping + +**Key talking point:** Having nodes in 3 AZs doesn't mean pods are spread. Without topology spread constraints, the scheduler may pack all pods into one AZ. diff --git a/skills/eks-operation-review/steering/cluster-lifecycle.md b/skills/eks-operation-review/steering/cluster-lifecycle.md new file mode 100644 index 0000000..5874657 --- /dev/null +++ b/skills/eks-operation-review/steering/cluster-lifecycle.md @@ -0,0 +1,132 @@ +# Cluster Lifecycle & Upgrades + +## Purpose +Assess EKS cluster version currency, data plane alignment, upgrade readiness, add-on compatibility, and upgrade process maturity. + +## EKS Version Support Calendar + +> **Last verified:** 2026-04-24. This table can become stale. Before rating version currency, cross-check against the [official EKS version calendar](https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html). + +Use this table to determine the ACTUAL support status. Do NOT guess or use training data. + +| Version | Standard Support Until | Extended Support Until | Current Status | +|---------|----------------------|----------------------|----------------| +| 1.35 | March 27, 2027 | March 27, 2028 | ✅ STANDARD (latest) | +| 1.34 | December 2, 2026 | December 2, 2027 | ✅ STANDARD | +| 1.33 | July 29, 2026 | July 29, 2027 | ✅ STANDARD | +| 1.32 | March 23, 2026 | March 23, 2027 | ✅ STANDARD | +| 1.31 | November 26, 2025 | November 26, 2026 | ⚠️ EXTENDED (standard ended) | +| 1.30 | July 23, 2025 | July 23, 2026 | ⚠️ EXTENDED (standard ended) | +| 1.29 | March 23, 2025 | March 23, 2026 | 🔴 EXTENDED (ending soon) | + +**CRITICAL:** The `upgradePolicy.supportType` field from the API is a CONFIGURATION PREFERENCE, not the current billing status. A cluster on v1.35 with `supportType: EXTENDED` is still on standard support and NOT paying the extended premium. Always determine actual support status from the calendar above based on the cluster version. + +## Checks to Execute + +### 1.1 — Control Plane Version Currency + +**What to check:** +- Describe the cluster to get the current Kubernetes version and platform version +- Look up the version in the calendar above to determine actual support status +- Report the actual support status, NOT the `supportType` API field + +**How to check:** +1. Describe the cluster → get `version` and `platformVersion` +2. Match the version against the calendar table above +3. Report: version, standard/extended status, and when current support period ends + +**Version out-of-range guard:** If the cluster version is higher than the highest version in this table (currently 1.35), rate GREEN — a newer version is by definition on standard support. Add a note: "Version 1.X is newer than the reference table (last verified YYYY-MM-DD). Rated GREEN as latest. Update the table when convenient." + +**Rating:** +- 🟢 GREEN: On v1.35 or v1.34 (latest or N-1, standard support), OR version newer than table +- 🟡 AMBER: On v1.33 or v1.32 (standard support but older) +- 🔴 RED: On v1.31 or below (extended support — paying $0.60/hr vs $0.10/hr, 6x premium) +- ⬜ UNKNOWN: Cannot determine (should not happen with live access) + +**Key talking point:** Extended support costs $0.60/hr vs $0.10/hr — that's ~$4,300/month extra per cluster. + +--- + +### 1.2 — Data Plane Version Alignment + +**What to check:** +- List all node groups and their Kubernetes versions +- Compare each node group version against the control plane version +- Check AMI type (AL2 vs AL2023 vs Bottlerocket) +- Check for Karpenter NodePools or EKS Auto Mode + +**How to check:** +1. List node groups → describe each for version, AMI type, capacity type +2. List nodes via Kubernetes API → get kubelet versions +3. Check for Karpenter NodePools (`nodepools.karpenter.sh`). If 404/NotFound (CRD not installed) → Karpenter is not deployed, rate based on other node management. If 403/Forbidden → mark Karpenter status UNKNOWN. +4. Describe cluster → check `computeConfig` for Auto Mode + +**Rating:** +- 🟢 GREEN: All nodes within N-1 of control plane, using managed node groups/Karpenter/Auto Mode +- 🟡 AMBER: Within skew policy but mixed versions, or self-managed nodes +- 🔴 RED: Any node beyond N-2 skew, or no visibility into node versions +- ⬜ UNKNOWN: No nodes found (possible if cluster is new or uses Fargate only) + +**Red flags:** AL2 AMIs (approaching EOL), self-managed nodes with no automated upgrade path. + +--- + +### 1.3 — Upgrade Readiness & Deprecated API Detection + +**What to check:** +- EKS Cluster Insights for upgrade blockers +- Presence of deprecated API usage +- PodSecurityPolicy resources (removed in K8s 1.25) + +**How to check:** +1. Get EKS Insights → filter for UPGRADE_READINESS category +2. List any PodSecurityPolicy resources via Kubernetes API +3. Check for Helm releases in kube-system (may use deprecated APIs) + +**Rating:** +- 🟢 GREEN: No critical insights, no deprecated API usage detected +- 🟡 AMBER: WARNING-level insights, or no automated detection tooling +- �� RED: CRITICAL/ERROR insights, deprecated APIs actively in use +- ⬜ UNKNOWN: Insights API not accessible + +--- + +### 1.4 — Add-on Version Compatibility + +**What to check:** +- List all EKS managed add-ons with versions and health +- Compare installed versions against latest compatible for the cluster version +- Check for self-managed add-ons in kube-system (Helm releases) + +**How to check:** +1. List addons → describe each for version, status, health +2. For each core add-on (vpc-cni, coredns, kube-proxy, aws-ebs-csi-driver), compare installed vs latest compatible + +**Rating:** +- 🟢 GREEN: All core add-ons are EKS Managed and on latest or N-1 compatible version +- 🟡 AMBER: Managed but behind, or mix of managed and self-managed +- 🔴 RED: Core add-ons self-managed with no version tracking, or health issues present +- ⬜ UNKNOWN: Cannot list add-ons + +**Key talking point:** EKS does NOT auto-update add-ons when you upgrade the control plane. This is the #1 thing customers forget. + +--- + +### 1.5 — Upgrade Process Maturity + +**What to check (target cluster only):** +- Cluster tags for environment classification (dev, staging, prod) +- Evidence of IaC-managed upgrades (eksctl, CloudFormation, Terraform tags) + +**How to check:** +1. Describe the target cluster → check tags for environment indicators + +**Do NOT** list or describe other clusters in the account. Stay within the scope of the target cluster. + +**Rating:** +- 🟢 GREEN: Cluster has environment tags, evidence of IaC management +- 🟡 AMBER: No environment tags, or unclear upgrade process +- 🔴 RED: No evidence of structured upgrade process +- ⬜ UNKNOWN: Cannot determine upgrade history from API alone — suggest user investigate + +**Investigate manually:** Do you have a documented upgrade runbook? Do you test upgrades on a non-prod cluster first? Can more than one person execute an upgrade? diff --git a/skills/eks-operation-review/steering/deployment-practices.md b/skills/eks-operation-review/steering/deployment-practices.md new file mode 100644 index 0000000..02176ea --- /dev/null +++ b/skills/eks-operation-review/steering/deployment-practices.md @@ -0,0 +1,75 @@ +# Deployment Practices + +## Purpose +Assess deployment strategies, CI/CD integration, and graceful shutdown configuration. + +## Automation Note +CI/CD pipeline details (approval gates, post-deployment tests) are not fully detectable from cluster state. The skill checks for tool presence and configuration; process maturity items are marked UNKNOWN. + +## Checks to Execute + +### 8.1 — Deployment Strategy & Rollback + +**What to check:** +- Deployment strategies in use (RollingUpdate vs Recreate) +- maxUnavailable and maxSurge settings (defaults of 25% are risky for replicas <= 4 — e.g., 25% of 2 = 0, meaning no controlled rollout. Recommend maxUnavailable: 0, maxSurge: 1 for small deployments) +- Argo Rollouts resources +- Flagger Canary resources +- terminationGracePeriodSeconds and preStop hooks + +**How to check:** +1. List Deployments → inspect `spec.strategy.type`, `rollingUpdate.maxUnavailable`, `rollingUpdate.maxSurge` +2. Flag deployments with replicas <= 4 and default maxUnavailable: 25% +3. List Rollouts (Argo Rollouts CRD, if exists) +4. List Canaries (Flagger CRD, if exists) +5. Inspect Deployments for `terminationGracePeriodSeconds` and `lifecycle.preStop` + +**Rating:** +- 🟢 GREEN: Zero-downtime strategy (maxUnavailable: 0), graceful shutdown configured +- 🟡 AMBER: Rolling update but default settings, or no progressive delivery +- 🔴 RED: Deployments cause downtime, no graceful shutdown +- ⬜ UNKNOWN: Cannot determine rollback speed or process — suggest user investigate + +--- + +### 8.2 — CI/CD Pipeline Integration + +**What to check:** +- ECR repositories: scan-on-push, tag immutability +- Admission webhooks enforcing image policies +- Image registries in use (ECR vs public) + +**How to check:** +1. Describe ECR repositories → scanOnPush, imageTagMutability +2. List ValidatingWebhookConfigurations → filter for image/policy-related names +3. List running pods → aggregate image registries + +**Rating:** +- 🟢 GREEN: Images scanned in CI, private registry, admission enforcement +- 🟡 AMBER: Pipeline exists but no scanning, or no admission enforcement +- 🔴 RED: No CI/CD evidence, images from untrusted public registries +- ⬜ UNKNOWN: Cannot determine full pipeline from cluster state — suggest user investigate + +--- + +### 8.3 — Graceful Shutdown & Connection Draining + +**What to check:** +- Deployments with preStop hooks vs without +- terminationGracePeriodSeconds (default 30s vs customized) +- Services with AWS Load Balancer annotations (deregistration delay matters) +- Ingress resources with target group attributes + +**How to check:** +1. List Deployments → count those with `lifecycle.preStop` vs without +2. List Deployments → check `terminationGracePeriodSeconds` (null = default 30s) +3. List Services → check for `service.beta.kubernetes.io/aws-load-balancer-type` annotation +4. List Ingresses → check for `alb.ingress.kubernetes.io/target-group-attributes` annotation (deregistration delay should match or exceed `terminationGracePeriodSeconds`) + +**Rating:** +- 🟢 GREEN: preStop hooks on all externally-facing deployments, grace period tuned, LB drain aligned +- 🟡 AMBER: Some deployments have preStop but not all +- 🔴 RED: No preStop hooks and experiencing 502s, or grace period too short +- ⬜ UNKNOWN: Cannot determine if 502s occur during deployments — suggest user investigate + +**Key talking point:** There's a race condition during pod termination. The LB still sends traffic for a few seconds after SIGTERM. A preStop sleep of 5-10s fixes it. diff --git a/skills/eks-operation-review/steering/infrastructure-as-code.md b/skills/eks-operation-review/steering/infrastructure-as-code.md new file mode 100644 index 0000000..8fc2796 --- /dev/null +++ b/skills/eks-operation-review/steering/infrastructure-as-code.md @@ -0,0 +1,87 @@ +# Infrastructure as Code & GitOps + +## Purpose +Assess whether cluster infrastructure and workload deployments are reproducible, auditable, and version-controlled. + +## Automation Note +This section is only partially automatable. The skill can detect tool presence (ArgoCD, Flux, CloudFormation stacks, cluster tags) but cannot assess process maturity (PR reviews, pipeline enforcement). Process-dependent items are marked UNKNOWN. + +## Checks to Execute + +### 2.1 — Cluster Provisioned via IaC + +**What to check:** +- Cluster tags for IaC provenance (terraform, eksctl, cdk, aws:cloudformation:stack-name) +- CloudFormation stacks with "eks" or "EKS" in the name + +**How to check:** +1. Describe cluster → inspect `tags` for IaC indicators (tags were already retrieved in Step 0 pre-flight — reuse that data, do NOT call `manage_eks_stacks`) +2. Look for tags: `terraform`, `managed-by`, `aws:cloudformation:stack-name`, `eksctl.cluster.k8s.io/*`, `aws:cdk:*` + +**Rating:** +- 🟢 GREEN: Clear IaC provenance in tags (CloudFormation stack, Terraform tags, eksctl tags) +- 🟡 AMBER: IaC tags present but unclear if current, or eksctl-created (basic IaC) +- 🔴 RED: No IaC tags — cluster appears console/CLI-created +- ⬜ UNKNOWN: Tags alone cannot confirm if IaC is pipeline-driven or manually applied — suggest user verify + +**Investigate manually:** Is IaC applied via CI/CD pipeline or manually? Could you recreate this cluster from code? + +--- + +### 2.2 — Workload Deployment via GitOps or CI/CD + +**What to check:** +- ArgoCD namespace and Application resources +- Flux namespace and Kustomization resources +- Other CD tools (Spinnaker, Tekton namespaces) + +**How to check:** +1. List namespaces → check for `argocd`, `flux-system`, `spinnaker`, `tekton-pipelines` +2. If argocd namespace exists → list `applications.argoproj.io` resources, check sync status +3. If flux-system exists → list `kustomizations.kustomize.toolkit.fluxcd.io` resources + +**Rating:** +- 🟢 GREEN: GitOps tool active with apps in-sync +- 🟡 AMBER: GitOps tool installed but apps out-of-sync, or CI/CD present but no GitOps +- 🔴 RED: No GitOps or CI/CD tools detected +- ⬜ UNKNOWN: No GitOps tools found — could still have external CI/CD. Suggest user verify: how do teams deploy workloads? + +--- + +### 2.3 — Configuration Drift Detection & Remediation + +**What to check:** +- ArgoCD auto-sync and self-heal settings +- Flux reconciliation status + +**How to check:** +1. If ArgoCD present → read Application resources, check `spec.syncPolicy.automated` for `selfHeal: true` +2. If Flux present → check kustomization ready status + +**Rating:** +- 🟢 GREEN: GitOps with self-heal enabled, all apps in-sync +- 🟡 AMBER: GitOps present but no self-heal, or some apps out-of-sync +- 🔴 RED: No drift detection mechanism +- ⬜ UNKNOWN: No GitOps tools found + +--- + +### 2.4 — Access Control & RBAC Defined in Code + +**What to check:** +- Authentication mode (API, CONFIG_MAP, API_AND_CONFIG_MAP) +- EKS Access Entries +- ClusterRoleBindings to cluster-admin +- Whether RBAC resources have GitOps labels + +**How to check:** +1. Describe cluster → `accessConfig.authenticationMode` +2. List access entries +3. List ClusterRoleBindings → filter for `roleRef.name == "cluster-admin"` +4. Check ClusterRoles/ClusterRoleBindings for labels indicating Helm/ArgoCD management + +**Rating:** +- 🟢 GREEN: API mode with Access Entries, RBAC managed by GitOps, cluster-admin limited +- 🟡 AMBER: API_AND_CONFIG_MAP (transitional), or RBAC partially in code +- 🔴 RED: CONFIG_MAP only with manual edits, broad cluster-admin access +- ⬜ UNKNOWN: Cannot determine if RBAC changes go through PR review — suggest user verify diff --git a/skills/eks-operation-review/steering/networking.md b/skills/eks-operation-review/steering/networking.md new file mode 100644 index 0000000..75ec410 --- /dev/null +++ b/skills/eks-operation-review/steering/networking.md @@ -0,0 +1,81 @@ +# Networking + +## Purpose +Assess VPC CNI configuration, IP capacity, DNS health, and network segmentation. + +## Checks to Execute + +### 6.1 — VPC and Subnet IP Capacity + +**What to check:** +- Subnets used by the cluster and available IP count +- VPC CNI configuration: prefix delegation, custom networking, WARM_IP_TARGET +- Current pod count vs IP capacity +- VPC CNI add-on version + +**How to check:** +1. Describe cluster → get subnet IDs from `resourcesVpcConfig.subnetIds` +2. Get VPC config for the cluster (available IPs per subnet) +3. List pods (Running) → count total +4. List nodes → count total +5. Describe addon `vpc-cni` → check version and configuration +6. Check DaemonSet `aws-node` in kube-system → inspect env vars for `ENABLE_PREFIX_DELEGATION`, `AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG` +7. List ENIConfig resources (custom networking indicator) + +**Rating:** +- 🟢 GREEN: >30% IP headroom, prefix delegation or custom networking enabled +- 🟡 AMBER: Adequate IPs now but no prefix delegation and cluster is growing +- 🔴 RED: <15% IPs available, or past IP exhaustion incidents +- ⬜ UNKNOWN: Cannot determine subnet sharing with other workloads + +**Key talking point:** Prefix delegation assigns a /28 (16 IPs) per ENI slot instead of 1 IP — dramatically increases pod density. + +--- + +### 6.2 — CoreDNS Health and Scaling + +**What to check:** +- CoreDNS deployment: replica count, resource requests, pod placement +- Node count (to assess CoreDNS ratio — ~1 replica per 8 nodes, minimum 2) +- NodeLocal DNSCache DaemonSet +- CoreDNS HPA +- CoreDNS topology spread constraints + +**How to check:** +1. Read Deployment `coredns` in kube-system → replicas, resources, topologySpreadConstraints +2. List pods with label `k8s-app=kube-dns` → check node placement +3. Count nodes +4. List DaemonSets → check for `node-local-dns` or `nodelocaldns` (recommended for clusters with 50+ nodes) +5. List HPAs in kube-system with label `k8s-app=kube-dns` (if HPA present, CoreDNS can auto-scale so fewer static replicas is acceptable) + +**Rating:** +- 🟢 GREEN: CoreDNS scaled to cluster size (~1 replica per 8 nodes, min 2), spread across AZs, NodeLocal DNSCache on clusters with 50+ nodes +- 🟡 AMBER: Adequate replicas but no topology spread, or no NodeLocal DNSCache on 50+ node clusters, or no HPA +- 🔴 RED: CoreDNS under-provisioned (2 replicas for 50+ nodes with no HPA), or past DNS incidents +- ⬜ UNKNOWN: Cannot determine if DNS issues have occurred historically + +--- + +### 6.3 — Network Policies & Segmentation + +**What to check:** +- VPC CNI Network Policy Controller enabled (`ENABLE_NETWORK_POLICY` env var on aws-node) +- Calico pods (alternative enforcement engine) +- NetworkPolicy resources across namespaces +- Default-deny policies (podSelector: {}) +- Namespaces without any NetworkPolicy + +**How to check:** +1. Read DaemonSet `aws-node` in kube-system → check env var `ENABLE_NETWORK_POLICY` +2. List pods with label `k8s-app=calico-node` +3. List NetworkPolicies across all namespaces +4. Inspect NetworkPolicies for default-deny (empty podSelector) +5. Compare namespaces with policies vs namespaces without + +**Rating:** +- 🟢 GREEN: Enforcement enabled (VPC CNI controller or Calico), default-deny in production namespaces +- 🟡 AMBER: Policies defined but enforcement not verified, or policies in some namespaces only +- 🔴 RED: No network policies, or policies defined but enforcement not enabled (false security) +- ⬜ UNKNOWN: Cannot determine if policies have been tested + +**Critical gotcha:** VPC CNI requires explicitly enabling the Network Policy Controller. Without it, NetworkPolicy resources are just YAML that does nothing. diff --git a/skills/eks-operation-review/steering/observability.md b/skills/eks-operation-review/steering/observability.md new file mode 100644 index 0000000..385c973 --- /dev/null +++ b/skills/eks-operation-review/steering/observability.md @@ -0,0 +1,93 @@ +# Observability + +## Purpose +Assess observability across three layers: control plane, data plane (nodes), and workloads — covering metrics, logs, and alerting. + +## Checks to Execute + +### 4.1 — EKS Control Plane Logging + +**What to check:** +- Which of the 5 log types are enabled (api, audit, authenticator, controllerManager, scheduler) +- CloudWatch log group existence and retention policy + +**How to check:** +1. Describe cluster → `logging.clusterLogging` → check each entry for `enabled: true` and which `types` +2. Use CloudWatch tools to check log group `/aws/eks/{cluster-name}/cluster` retention (recommend >= 30 days for audit logs; no retention policy = logs kept forever at cost) + +**Rating:** +- 🟢 GREEN: All 5 log types enabled with defined retention policy +- 🟡 AMBER: Some log types enabled (especially if audit is on), or no retention policy +- 🔴 RED: Control plane logging completely disabled, or audit logs specifically disabled +- ⬜ UNKNOWN: Should not happen with live access + +**Key talking point:** EKS control plane logging is OFF by default. The audit log is your security camera for every API call. + +--- + +### 4.2 — Metrics Collection & Dashboards + +**What to check:** +- CloudWatch Container Insights add-on (`amazon-cloudwatch-observability`) +- Prometheus pods (labels: `app.kubernetes.io/name=prometheus` or `app=prometheus`) +- Grafana pods +- kube-state-metrics deployment (critical for cluster state visibility) +- node-exporter DaemonSet +- ADOT add-on +- Third-party monitoring DaemonSets (Datadog, New Relic, Dynatrace) + +**How to check:** +1. Describe addon `amazon-cloudwatch-observability` +2. List pods with label `app.kubernetes.io/name=prometheus` across all namespaces +3. List pods with label `app.kubernetes.io/name=grafana` +4. List pods with label `app.kubernetes.io/name=kube-state-metrics` +5. List DaemonSets across all namespaces (catches node-exporter and third-party agents) + +**Rating:** +- 🟢 GREEN: Metrics collection + kube-state-metrics + dashboards (Container Insights or Prometheus+Grafana or third-party) +- 🟡 AMBER: Partial stack (e.g., Container Insights but no kube-state-metrics, or Prometheus without Grafana) +- 🔴 RED: No metrics collection at all +- ⬜ UNKNOWN: Cannot determine if dashboards are actively used + +--- + +### 4.3 — Centralized Log Aggregation for Workloads + +**What to check:** +- Fluent Bit DaemonSet (labels: `app.kubernetes.io/name=fluent-bit` or `k8s-app=fluent-bit`) +- Fluentd DaemonSet +- CloudWatch agent DaemonSet in `amazon-cloudwatch` namespace +- Application log groups in CloudWatch + +**How to check:** +1. List DaemonSets with Fluent Bit labels across all namespaces +2. List DaemonSets in `amazon-cloudwatch` namespace +3. Use CloudWatch tools to check for log groups with prefix `/aws/eks/{cluster-name}` + +**Rating:** +- 🟢 GREEN: Log shipper deployed, logs centralized with retention policy, structured logging +- 🟡 AMBER: Log shipper exists but no retention policy, or unstructured logging +- 🔴 RED: No centralized log collection — teams rely on kubectl logs +- ⬜ UNKNOWN: Cannot determine log format (structured vs unstructured) without sampling + +--- + +### 4.4 — Alerting Defined and Actionable + +**What to check:** +- CloudWatch Alarms related to EKS/ContainerInsights +- Prometheus Alertmanager pods +- PrometheusRule resources (alert definitions) + +**How to check:** +1. List pods with label `app.kubernetes.io/name=alertmanager` +2. List PrometheusRule resources. If 404/NotFound (CRD not installed) → Prometheus Operator not deployed, rate alerting based on CloudWatch only. If 403/Forbidden → mark UNKNOWN. +3. Use CloudWatch tools to list alarms with ContainerInsights namespace + +**Rating:** +- 🟢 GREEN: Alerts cover critical scenarios (node, pod, capacity), routed to on-call +- 🟡 AMBER: Some alerts exist but incomplete coverage, or no runbooks linked +- 🔴 RED: No alerting configured +- ⬜ UNKNOWN: Cannot determine if alerts have runbooks or if on-call monitors them — suggest user investigate + +**Minimum viable alert set:** NodeNotReady, PodCrashLooping, PodPendingTooLong, HighAPIServerLatency, IPExhaustion. diff --git a/skills/eks-operation-review/steering/operational-processes.md b/skills/eks-operation-review/steering/operational-processes.md new file mode 100644 index 0000000..eba2711 --- /dev/null +++ b/skills/eks-operation-review/steering/operational-processes.md @@ -0,0 +1,95 @@ +# Operational Processes + +## Purpose +Assess operational process maturity: runbooks, on-call, incident response, and disaster recovery. + +## Automation Note +This section is mostly NOT automatable from cluster state. The skill checks for tool presence (Velero, AWS Backup, AWS Support tier) and current cluster health indicators. Process maturity (runbooks, on-call rotation, PIR process) cannot be detected — these items are marked UNKNOWN with suggestions for what to investigate on your own. + +## Checks to Execute + +### 9.1 — Runbooks for Common Failure Scenarios + +**What to check (cluster health indicators that suggest which runbooks should exist):** +- Nodes not in Ready state +- Pods not Running (excluding Completed jobs) +- Recent Warning events +- CrashLoopBackOff pods, Pending pods, OOMKilled events, FailedScheduling events + +**How to check:** +1. List nodes → check for any not Ready +2. List pods with field selector `status.phase=Pending` +3. Get events with type=Warning (recent) +4. Get events with reason=BackOff, OOMKilling, FailedScheduling + +**Rating:** +- ⬜ UNKNOWN: Cannot determine if runbooks exist from cluster state. + +**Investigate manually:** +- Do you have runbooks for node NotReady, CrashLoopBackOff, IP exhaustion, DNS failures? +- Are alerts linked directly to runbooks? +- When was the last time a runbook was updated? + +**If active issues found:** Note them as evidence that runbooks for those scenarios should exist and be tested. + +--- + +### 9.2 — On-Call Rotation & Escalation + +**What to check:** +- AWS Support plan tier (Business/Enterprise = Support API accessible) + +**How to check:** +1. This check is limited — AWS Support API access indicates Business or Enterprise plan + +**Rating:** +- ⬜ UNKNOWN: Primarily a process question. + +**Investigate manually:** +- Do you have a formal on-call rotation? +- What's the escalation path when on-call can't resolve within 30 minutes? +- What AWS Support plan are you on? +- How many people can handle a critical EKS incident independently? + +--- + +### 9.3 — Post-Incident Review Process + +**What to check:** +- Recent significant events (NodeNotReady, BackOff, rollbacks) that would warrant a PIR + +**How to check:** +1. Get events with reason=NodeNotReady +2. Get events with reason=DeploymentRollback + +**Rating:** +- ⬜ UNKNOWN: Cannot determine PIR process from cluster state. + +**Investigate manually:** +- Do you conduct blameless post-mortems after incidents? +- Are action items tracked to completion? +- Can you point to a change made as a result of a post-incident review? + +--- + +### 9.4 — Disaster Recovery & Backup Strategy + +**What to check:** +- Velero pods and backup schedules +- AWS Backup plans +- VolumeSnapshot resources +- StatefulSets and PVCs (data at risk if no backup) + +**How to check:** +1. List pods in `velero` namespace +2. List Backup resources (`backups.velero.io`) and Schedule resources (`schedules.velero.io`) +3. List VolumeSnapshots across all namespaces +4. List StatefulSets across all namespaces +5. List PVCs across all namespaces → count + +**Rating:** +- 🟢 GREEN: Backup tool in place, scheduled backups running, restore tested +- 🟡 AMBER: Backups exist but never tested, or only PV data backed up +- 🔴 RED: Stateful workloads with no backup strategy +- N/A: No stateful workloads and all config is in Git/IaC +- ⬜ UNKNOWN: Cannot determine if restore has been tested — suggest user investigate diff --git a/skills/eks-operation-review/steering/report-generation.md b/skills/eks-operation-review/steering/report-generation.md new file mode 100644 index 0000000..96ee423 --- /dev/null +++ b/skills/eks-operation-review/steering/report-generation.md @@ -0,0 +1,164 @@ +# Report Generation + +## Purpose +After all section checks are complete, generate the EKS Operation Review report. + +## Consistency Checks (MANDATORY before writing) + +Before writing the report, validate consistency: + +1. **Build a master list** of all findings with their ratings from sections 01-10 +2. **For each RED item:** confirm it appears in "Critical" or "Important" prioritized actions +3. **For each AMBER item:** confirm it appears in "Important" or "Quick Wins" +4. **For the Executive Summary:** only mention ratings that match the master list — do not call something a "critical gap" if it's AMBER, or omit a RED from the summary +5. **For Prioritized Actions:** every entry must reference the finding ID (e.g., "4.1 — Control Plane Logging") + +## Workflow + +### Step 1: Build Master Finding List + +``` +| Section | Item ID | Item Name | Rating | +``` + +### Step 2: Calculate Maturity Score + +- Count GREEN, AMBER, RED, UNKNOWN +- Calculate percentages (exclude UNKNOWN from denominator) + +### Step 3: Write Executive Summary + +From the master list, identify: +- **Top strengths** (GREEN items with highest operational impact) +- **Top gaps** (RED items, ordered by blast radius: security > availability > cost) +- Write 2-3 paragraphs. Every rating mentioned must match the master list. + +### Step 4: Write Findings Tables + +One table per section. Every item from the master list must appear. + +### Step 5: Write Prioritized Actions + +Cross-reference against the master list: +- **Critical (30 days):** All RED items. Column: `Finding | Action | References` +- **Important (90 days):** All AMBER items. Column: `Finding | Action | References` +- **Quick Wins:** Items (RED or AMBER) fixable in < 1 hour. Column: `Finding | Action | Effort | Impact | References` + +Every entry must include the finding ID and name (e.g., "4.1 — Control Plane Logging 🔴"). + +**One row per finding.** Never bundle multiple findings into a single row (e.g., "2.2/2.3 — GitOps & Drift Detection"). Each finding has its own context, action, and references — collapsing them hides information and breaks the "every RED must appear in Critical" consistency rule. If two findings genuinely share an action, list them on separate rows that point to the same action. + +**Ordering within Critical:** List RED items by blast radius category: +1. Security first — public API endpoint, hardcoded credentials, no PSA/network policies, overly broad RBAC +2. Availability next — no PDBs, single-replica critical workloads, missing health probes, no alerting +3. Cost last — extended support billing, deprecated storage classes + +Within each category, order by scope (cluster-wide before namespace-scoped). + +### Step 6: Write Investigate Manually + +All UNKNOWN items with specific questions the user should answer. + +### Step 7: Apply AWS Reference Links + +Use the pre-verified reference map below. Do NOT call the AWS Documentation MCP server — it adds latency and token cost with minimal benefit. + +**Section 01 — Cluster Lifecycle & Upgrades** +- Version calendar: `https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html` +- Upgrade cluster: `https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html` +- Best practices for upgrades: `https://docs.aws.amazon.com/eks/latest/best-practices/cluster-upgrades.html` +- Platform versions: `https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html` +- Managed node groups: `https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html` +- EKS Auto Mode: `https://docs.aws.amazon.com/eks/latest/userguide/automode.html` + +**Section 02 — Infrastructure as Code & GitOps** +- EKS User Guide (general): `https://docs.aws.amazon.com/eks/latest/userguide/` +- Best practices (general): `https://docs.aws.amazon.com/eks/latest/best-practices/` + +**Section 03 — Access & Identity** +- IRSA: `https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html` +- EKS Pod Identity: `https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html` +- Access entries: `https://docs.aws.amazon.com/eks/latest/userguide/access-entries.html` +- Grant K8s access: `https://docs.aws.amazon.com/eks/latest/userguide/grant-k8s-access.html` +- RBAC hardening: `https://docs.aws.amazon.com/eks/latest/userguide/rbac-hardening.html` +- API server endpoint: `https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html` +- Security best practices: `https://docs.aws.amazon.com/eks/latest/best-practices/security.html` +- Pod Security Standards: `https://docs.aws.amazon.com/eks/latest/best-practices/pod-security.html` + +**Section 04 — Observability** +- Control plane logging: `https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html` +- Observability overview: `https://docs.aws.amazon.com/eks/latest/userguide/eks-observe.html` + +**Section 05 — Workload Configuration** +- EBS CSI driver: `https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html` +- Reliability best practices: `https://docs.aws.amazon.com/eks/latest/best-practices/reliability.html` + +**Section 06 — Networking** +- VPC CNI: `https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html` +- Prefix delegation: `https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html` +- Custom networking: `https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html` +- CoreDNS: `https://docs.aws.amazon.com/eks/latest/userguide/managing-coredns.html` +- Networking best practices: `https://docs.aws.amazon.com/eks/latest/best-practices/networking.html` + +**Section 07 — Autoscaling** +- Karpenter best practices: `https://docs.aws.amazon.com/eks/latest/best-practices/karpenter.html` +- Scalability best practices: `https://docs.aws.amazon.com/eks/latest/best-practices/scalability.html` +- Cost optimization: `https://docs.aws.amazon.com/eks/latest/best-practices/cost-opt.html` + +**Section 08 — Deployment Practices** +- Reliability best practices: `https://docs.aws.amazon.com/eks/latest/best-practices/reliability.html` + +**Section 09 — Operational Processes** +- Reliability best practices: `https://docs.aws.amazon.com/eks/latest/best-practices/reliability.html` + +**Section 10 — Add-on Management** +- Managed add-ons: `https://docs.aws.amazon.com/eks/latest/userguide/managing-add-ons.html` +- Node health & auto-repair: `https://docs.aws.amazon.com/eks/latest/userguide/node-health.html` + +**Fallback (any topic):** +- EKS Best Practices Guide: `https://docs.aws.amazon.com/eks/latest/best-practices/` +- EKS User Guide: `https://docs.aws.amazon.com/eks/latest/userguide/` + +Do NOT fabricate URLs beyond this list. If a finding doesn't match a specific URL above, use the fallback section-level page. + +### Step 8: Final Consistency Validation + +Before outputting, scan the report for: +- Any RED item missing from Prioritized Actions → add it +- Any item mentioned in Executive Summary with wrong rating → fix it +- Any Prioritized Action without a finding ID → add the ID + +### Step 8b: Append Sample-Code Disclaimer + +Add the following footer at the very end of the report, after the AWS Reference Links section, separated by a horizontal rule: + +```markdown +--- + +*This report was generated by a Claude Code skill provided as sample code for educational and demonstration purposes only. Findings should be reviewed and validated before acting on them. See the project's README and LICENSE for full terms.* +``` + +### Step 9: Write the Report File + +Write the report to the **workspace directory**. The file must be created inside the current workspace. + +**Filename format:** `EKS-Operation-Review---.md` + +**Example:** `EKS-Operation-Review-demo-cluster-2026-03-22-1830.md` + +The file should be written to the workspace root or a `reports/` subfolder within the workspace. Do NOT use absolute paths outside the workspace. + +### Step 10: Offer HTML Conversion + +Ask: "Would you like me to convert the report to HTML?" + +If yes, run the conversion script — do NOT generate HTML manually. Execute this command: + +```bash +python3 skills/eks-operation-review/tools/report_to_html.py .md +``` + +Run this from the repository root. The converter lives at +`skills/eks-operation-review/tools/report_to_html.py`. + +Do NOT create HTML by hand. Always use the script. diff --git a/skills/eks-operation-review/steering/workload-configuration.md b/skills/eks-operation-review/steering/workload-configuration.md new file mode 100644 index 0000000..a1303eb --- /dev/null +++ b/skills/eks-operation-review/steering/workload-configuration.md @@ -0,0 +1,122 @@ +# Workload Configuration + +## Purpose +Assess workload resilience: resource requests/limits, health probes, disruption budgets, image hygiene, and storage configuration. + +## Checks to Execute + +### 5.1 — Resource Requests and Limits + +**What to check:** +- Running pods missing resource requests or limits +- LimitRange resources in namespaces +- ResourceQuota resources in namespaces +- Recent OOMKilled events +- Admission webhooks enforcing resources (Kyverno, Gatekeeper) + +**How to check:** +1. List pods (Running) across all namespaces → inspect `spec.containers[].resources.requests` and `.limits` +2. Count pods with no requests vs total running pods → calculate percentage +3. List LimitRange resources across all namespaces +4. List ResourceQuota resources across all namespaces +5. Get events with reason=OOMKilling (count occurrences — >5 in recent events = AMBER, >20 = RED) +6. List ValidatingWebhookConfigurations and MutatingWebhookConfigurations + +**Rating:** +- 🟢 GREEN: >90% of pods have requests, LimitRange/ResourceQuota in place, admission enforcement +- 🟡 AMBER: Most pods have requests but no enforcement mechanism, or frequent OOMKills +- 🔴 RED: Majority of pods missing requests, no LimitRange, no enforcement +- ⬜ UNKNOWN: Should not happen with live access + +**Key talking point:** Without resource requests, the scheduler is flying blind. Don't set CPU limits equal to requests — causes unnecessary throttling. + +--- + +### 5.2 — Health Probes Configured + +**What to check:** +- Deployments missing readiness probes +- Deployments missing liveness probes +- Deployments missing startup probes (important for slow-starting apps: JVM/Java, Kotlin, Scala, or apps with long initialization >10s) +- Pods in CrashLoopBackOff (may indicate bad liveness probes) + +**How to check:** +1. List Deployments across all namespaces → inspect containers for readinessProbe, livenessProbe, startupProbe +2. Count deployments missing each probe type +3. List pods not in Running/Succeeded phase → check for CrashLoopBackOff + +**Rating:** +- 🟢 GREEN: >90% of deployments have readiness probes, startup probes on slow-starting apps +- 🟡 AMBER: Readiness probes on most but not all, or no startup probes for JVM apps +- 🔴 RED: Majority of deployments missing readiness probes +- ⬜ UNKNOWN: Cannot determine if apps are slow-starting without more context + +--- + +### 5.3 — Pod Disruption Budgets (PDBs) + +**What to check:** +- PDB resources and their settings +- Multi-replica deployments without PDBs +- PDBs with disruptionsAllowed=0 (blocks upgrades). If disruptionsAllowed=0 AND replicas=1, mark RED (single point of failure that also blocks node drains) +- Single-replica deployments (inherently not disruption-safe) + +**How to check:** +1. List PodDisruptionBudgets across all namespaces → check minAvailable, maxUnavailable, disruptionsAllowed +2. List Deployments with replicas > 1 → compare against PDB coverage +3. List Deployments with replicas == 1 + +**Rating:** +- 🟢 GREEN: PDBs on all multi-replica production deployments with reasonable settings +- 🟡 AMBER: PDBs on some but not all, or some PDBs blocking disruptions +- 🔴 RED: No PDBs at all, or critical workloads running single-replica +- ⬜ UNKNOWN: Cannot determine which deployments are "production" vs "dev" + +--- + +### 5.4 — Image Tag Hygiene + +**What to check:** +- Running pods using `:latest` tag or no tag +- ECR repositories: tag immutability and scan-on-push settings +- Image registries in use (ECR vs Docker Hub vs other) + +**How to check:** +1. List running pods → inspect container images for `:latest` or missing tag +2. Use AWS API to describe ECR repositories → check `imageTagMutability` and `imageScanningConfiguration` +3. Aggregate image registries from pod specs + +**Rating:** +- 🟢 GREEN: No `:latest` in production, ECR with tag immutability, scan-on-push enabled +- 🟡 AMBER: Mostly versioned tags but some `:latest`, or ECR without immutability +- 🔴 RED: `:latest` widely used, or images from untrusted public registries +- ⬜ UNKNOWN: Cannot determine if tags are mutable without ECR access + +--- + +### 5.5 — Persistent Volume & Stateful Workload Configuration + +**What to check:** +- StorageClasses: provisioner, reclaimPolicy, volumeBindingMode, gp2 vs gp3 +- PVCs and their status +- CSI drivers installed +- EBS CSI driver add-on status +- VolumeSnapshotClasses (backup support) +- StatefulSets +- Deprecated in-tree volume plugin usage + +**How to check:** +1. List StorageClasses → check for gp3 default, Retain policy, WaitForFirstConsumer +2. List PVCs across all namespaces +3. List CSIDrivers +4. Describe addon `aws-ebs-csi-driver` +5. List VolumeSnapshotClasses. If 404/NotFound (CRD not installed) → no snapshot support configured, factor into rating. If 403/Forbidden → mark snapshot capability UNKNOWN. +6. List StatefulSets across all namespaces +7. List PersistentVolumes → check for `spec.awsElasticBlockStore` (deprecated in-tree) + +**Rating:** +- 🟢 GREEN: gp3, Retain policy, WaitForFirstConsumer, CSI driver managed, snapshots configured +- 🟡 AMBER: gp2 still in use, or Delete policy on production volumes, or no snapshots +- 🔴 RED: Deprecated in-tree plugin, Delete policy on databases, or no backup strategy +- N/A: No stateful workloads on EKS +- ⬜ UNKNOWN: Cannot determine if Delete policy is intentional (dev) vs accidental (prod) diff --git a/skills/eks-operation-review/tools/report_to_html.py b/skills/eks-operation-review/tools/report_to_html.py new file mode 100644 index 0000000..87af7fc --- /dev/null +++ b/skills/eks-operation-review/tools/report_to_html.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +""" +EKS Operation Review — Markdown to HTML Converter + +Converts assessment report markdown files to styled HTML. +No external dependencies required. + +Usage: + python3 report_to_html.py # outputs .html + python3 report_to_html.py -o # custom output path + python3 report_to_html.py *.md # batch convert +""" + +import re +import sys +import html +import urllib.parse +from pathlib import Path + +_ALLOWED_URL_SCHEMES = {'http', 'https', 'mailto'} + +CSS = """ +*{box-sizing:border-box;margin:0;padding:0} +body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;line-height:1.6;color:#1a1a2e;background:#f0f2f5;padding:2rem} +main{max-width:1100px;margin:0 auto;background:#fff;border-radius:12px;box-shadow:0 4px 24px rgba(0,0,0,.08);padding:3rem} +h1{color:#0f1b61;border-bottom:3px solid #ff9900;padding-bottom:.5rem;margin-bottom:1.5rem;font-size:1.8rem} +h2{color:#232f3e;margin:2rem 0 1rem;padding:.5rem 0;border-bottom:2px solid #e8e8e8;font-size:1.4rem} +h3{color:#37475a;margin:1.5rem 0 .75rem;font-size:1.15rem} +h4{color:#37475a;margin:1.2rem 0 .5rem;font-size:1.05rem} +table{width:100%;border-collapse:collapse;margin:1rem 0 1.5rem;font-size:.9rem} +th{background:#232f3e;color:#fff;padding:10px 14px;text-align:left;font-weight:600} +td{padding:8px 14px;border-bottom:1px solid #e8e8e8;vertical-align:top} +tr:nth-child(even){background:#f8f9fa} +tr:hover{background:#fff3e0} +code{background:#f1f3f5;padding:2px 6px;border-radius:4px;font-size:.85em;color:#c7254e} +pre{background:#1a1a2e;color:#e8e8e8;padding:1rem;border-radius:8px;overflow-x:auto;margin:1rem 0} +pre code{background:none;color:inherit;padding:0} +blockquote{border-left:4px solid #ff9900;background:#fff8e1;padding:.75rem 1rem;margin:1rem 0;border-radius:0 8px 8px 0} +hr{border:none;border-top:2px solid #e8e8e8;margin:2rem 0} +a{color:#0073bb;text-decoration:none}a:hover{text-decoration:underline} +li{margin:.3rem 0 .3rem 1.5rem} +ul,ol{margin:.5rem 0 1rem} +strong{color:#16213e} +p{margin:.5rem 0} +.red{background:#fde8e8;color:#b71c1c;padding:2px 8px;border-radius:4px;font-weight:600;display:inline-block;font-size:.85em;min-width:60px;text-align:center} +.amber{background:#fff3e0;color:#e65100;padding:2px 8px;border-radius:4px;font-weight:600;display:inline-block;font-size:.85em;min-width:60px;text-align:center} +.green{background:#e8f5e9;color:#1b5e20;padding:2px 8px;border-radius:4px;font-weight:600;display:inline-block;font-size:.85em;min-width:60px;text-align:center} +.unknown{background:#e8eaf6;color:#283593;padding:2px 8px;border-radius:4px;font-weight:600;display:inline-block;font-size:.85em;min-width:60px;text-align:center} +.score-bar{display:flex;gap:4px;margin:1rem 0} +.score-bar div{height:28px;border-radius:4px;display:flex;align-items:center;justify-content:center;color:#fff;font-weight:600;font-size:.8rem} +.critical-box{background:#fde8e8;border:2px solid #ef5350;border-radius:8px;padding:1rem 1.25rem;margin:1rem 0} +.quick-win{background:#e8f5e9;border:2px solid #66bb6a;border-radius:8px;padding:1rem 1.25rem;margin:1rem 0} +.internal-banner{background:#fff3e0;border:2px solid #ff9900;border-radius:8px;padding:.75rem 1.25rem;margin:1rem 0;font-weight:600;color:#e65100} +@media print{body{background:#fff;padding:0}main{box-shadow:none;padding:1rem}} +""" + + +def escape(text): + return html.escape(text, quote=True) + + +def _safe_href(url): + """Return the URL if its scheme is allowlisted, otherwise '#'. + + Relative links (no scheme) and fragments are treated as safe. + Control characters are stripped because browsers historically ignored + them inside schemes, allowing `java\tscript:` to still execute. + """ + cleaned = ''.join(c for c in url.strip() if c >= ' ' and c != '\x7f') + try: + parsed = urllib.parse.urlparse(cleaned) + except ValueError: + return '#' + scheme = parsed.scheme.lower() + if scheme and scheme not in _ALLOWED_URL_SCHEMES: + return '#' + return cleaned + + +def _link_sub(match): + text = match.group(1) + url = _safe_href(match.group(2)) + return f'{text}' + + +def inline_format(text): + """Process inline markdown: bold, code, links, emoji badges. + + Input must already be HTML-escaped with quote=True by the caller. + Link hrefs are validated against a scheme allowlist; because `"` and + `&` in the URL are already escaped to `"`/`&`, cluster-derived + strings cannot break out of the href attribute. + """ + # Links: [text](url) — scheme-allowlisted, href-escaped + text = re.sub(r'\[([^\]]+)\]\(([^)]+)\)', _link_sub, text) + # Bold: **text** + text = re.sub(r'\*\*(.+?)\*\*', r'\1', text) + # Inline code: `text` + text = re.sub(r'`([^`]+)`', r'\1', text) + # RAG emoji badges + text = text.replace('🔴', 'RED') + text = text.replace('🟡', 'AMBER') + text = text.replace('🟢', 'GREEN') + text = text.replace('⬜', 'UNKNOWN') + return text + + +def parse_table(lines): + """Convert markdown table lines to HTML table.""" + if len(lines) < 2: + return "" + headers = [c.strip() for c in lines[0].strip('|').split('|')] + rows = [] + for line in lines[2:]: # skip separator + cells = [c.strip() for c in line.strip('|').split('|')] + rows.append(cells) + + out = '\n' + for h in headers: + out += f'' + out += '\n\n' + for row in rows: + out += '' + for cell in row: + out += f'' + out += '\n' + out += '
{inline_format(escape(h))}
{inline_format(escape(cell))}
\n' + return out + + +def convert(md_text): + """Convert markdown to HTML.""" + lines = md_text.split('\n') + out = [] + i = 0 + in_list = None # 'ul' or 'ol' + + def close_list(): + nonlocal in_list + if in_list: + out.append(f'') + in_list = None + + while i < len(lines): + line = lines[i] + stripped = line.strip() + + # Fenced code block + if stripped.startswith('```'): + close_list() + lang_match = re.match(r'^```(\w+)?', stripped) + lang = lang_match.group(1) if lang_match and lang_match.group(1) else None + lang_attr = f' class="language-{lang}"' if lang else '' + i += 1 + code_lines = [] + while i < len(lines) and not lines[i].strip().startswith('```'): + code_lines.append(escape(lines[i])) + i += 1 + i += 1 # skip closing ``` + out.append(f'
{chr(10).join(code_lines)}
') + continue + + # Blank line + if not stripped: + close_list() + i += 1 + continue + + # Table: collect consecutive | lines + if stripped.startswith('|') and '|' in stripped[1:]: + close_list() + table_lines = [] + while i < len(lines) and lines[i].strip().startswith('|'): + table_lines.append(lines[i]) + i += 1 + out.append(parse_table(table_lines)) + continue + + # Headings + m = re.match(r'^(#{1,4})\s+(.+)$', stripped) + if m: + close_list() + level = len(m.group(1)) + text = inline_format(escape(m.group(2))) + out.append(f'{text}') + i += 1 + continue + + # Horizontal rule + if re.match(r'^-{3,}$', stripped) or re.match(r'^\*{3,}$', stripped): + close_list() + out.append('
') + i += 1 + continue + + # Ordered list + m = re.match(r'^(\d+)\.\s+(.+)$', stripped) + if m: + if in_list != 'ol': + close_list() + in_list = 'ol' + out.append('
    ') + out.append(f'
  1. {inline_format(escape(m.group(2)))}
    2. ') + i += 1 + continue + + # Unordered list + if stripped.startswith('- ') or stripped.startswith('* '): + if in_list != 'ul': + close_list() + in_list = 'ul' + out.append('
      ') + out.append(f'
    • {inline_format(escape(stripped[2:]))}
      • ') + i += 1 + continue + + # Blockquote + if stripped.startswith('> '): + close_list() + out.append(f'
      {inline_format(escape(stripped[2:]))}
      ') + i += 1 + continue + + # Internal banner detection + if '⚠️' in stripped and 'INTERNAL' in stripped.upper(): + close_list() + out.append(f'
      {inline_format(escape(stripped))}
      ') + i += 1 + continue + + # Paragraph + close_list() + out.append(f'

      {inline_format(escape(stripped))}

      ') + i += 1 + + close_list() + return '\n'.join(out) + + +def md_to_html(md_text, title="EKS Operation Review"): + body = convert(md_text) + return f""" + +{escape(title)} +
      +{body} +
      """ + + +def main(): + args = sys.argv[1:] + if not args or args[0] in ('-h', '--help'): + print(__doc__.strip()) + sys.exit(0) + + output_path = None + files = [] + i = 0 + while i < len(args): + if args[i] == '-o' and i + 1 < len(args): + output_path = args[i + 1] + i += 2 + else: + files.append(args[i]) + i += 1 + + if not files: + print("Error: no input files specified", file=sys.stderr) + sys.exit(1) + + for f in files: + p = Path(f) + if not p.exists(): + print(f"Error: {f} not found", file=sys.stderr) + continue + + md = p.read_text(encoding='utf-8') + + # Extract title from first H1 + m = re.search(r'^#\s+(.+)$', md, re.MULTILINE) + title = m.group(1) if m else p.stem + + result = md_to_html(md, title) + + if output_path and len(files) == 1: + out = Path(output_path) + else: + out = p.with_suffix('.html') + + out.write_text(result, encoding='utf-8') + print(f"✓ {p.name} → {out.name}") + + +if __name__ == '__main__': + main()