diff --git a/README.md b/README.md index 95c5970..becdf4f 100644 --- a/README.md +++ b/README.md @@ -316,6 +316,19 @@ aliases resolve to their base. Explicit `-cluster` flags are applied last, so they override or extend the manifest (e.g. `-cluster events_recent=@absent`). A cluster that references an undeclared role is rejected. +#### Validating a whole environment + +With `-manifest`/`-env` and **no** `-layer`/`-config`, validate runs in +manifest-driven mode: it validates **every role** in the manifest for that env, +each against the cluster set derived from the whole manifest. One command checks +the entire environment instead of a shell loop over nodes; failures are prefixed +with the role they came from. + +```sh +hclexp validate -manifest roles.hcl -env prod-us -layer-root . +# validation error: [role data] posthog.web_stats: Distributed table column ... +``` + ### Distributed proxy columns Once a `Distributed` remote resolves to an inspectable table (locally or via a diff --git a/cmd/hclexp/hclexp.go b/cmd/hclexp/hclexp.go index affc56b..36fd019 100644 --- a/cmd/hclexp/hclexp.go +++ b/cmd/hclexp/hclexp.go @@ -193,41 +193,108 @@ func buildManifestClusters(cs *hclload.ClusterSet, path, env, layerRoot string) if err != nil { return err } - layersByRole := make(map[string][]string, len(roles)) + clusters, err := parseManifestClusters(path) + if err != nil { + return err + } + // Load only the roles some cluster references (single-node mode needs the + // mappings, not every role's schema). + referenced := map[string]bool{} + for _, cl := range clusters { + for _, role := range cl.Roles { + referenced[role] = true + } + } + needed := make([]manifestRole, 0, len(referenced)) for _, r := range roles { - layersByRole[r.Role] = r.Layers + if referenced[r.Role] { + needed = append(needed, r) + } } - - clusters, err := parseManifestClusters(path) + schemas, err := loadManifestRoleSchemas(needed, layerRoot) if err != nil { return err } + clusterSetFromRoles(cs, clusters, schemas) + return nil +} + +// loadManifestRoleSchemas loads and resolves each role's composition for the +// selected env (layer paths under layerRoot) into a schema, keyed by role name. +func loadManifestRoleSchemas(roles []manifestRole, layerRoot string) (map[string]*hclload.Schema, error) { + schemas := make(map[string]*hclload.Schema, len(roles)) + for _, r := range roles { + dirs := make([]string, len(r.Layers)) + for i, l := range r.Layers { + dirs[i] = filepath.Join(layerRoot, l) + } + schema, err := hclload.LoadLayers(dirs) + if err != nil { + return nil, fmt.Errorf("role %q: loading %v: %w", r.Role, dirs, err) + } + if err := hclload.Resolve(schema); err != nil { + return nil, fmt.Errorf("role %q: resolving %v: %w", r.Role, dirs, err) + } + schemas[r.Role] = schema + } + return schemas, nil +} + +// clusterSetFromRoles registers each cluster's mapping in cs: its schema is the +// union of its member roles' (preloaded) compositions, and each alias maps to +// @alias=cluster. A member role absent from schemas (not deployed in the env) +// contributes nothing. +func clusterSetFromRoles(cs *hclload.ClusterSet, clusters []manifestClusterBlock, schemas map[string]*hclload.Schema) { for _, cl := range clusters { var dbs []hclload.DatabaseSpec for _, role := range cl.Roles { - layers, ok := layersByRole[role] - if !ok { - continue // role not deployed in this env - } - dirs := make([]string, len(layers)) - for i, l := range layers { - dirs[i] = filepath.Join(layerRoot, l) - } - schema, err := hclload.LoadLayers(dirs) - if err != nil { - return fmt.Errorf("cluster %q role %q: loading %v: %w", cl.Name, role, dirs, err) + if s, ok := schemas[role]; ok { + dbs = append(dbs, s.Databases...) } - if err := hclload.Resolve(schema); err != nil { - return fmt.Errorf("cluster %q role %q: resolving %v: %w", cl.Name, role, dirs, err) - } - dbs = append(dbs, schema.Databases...) } cs.Add(cl.Name, dbs) for _, alias := range cl.Aliases { cs.AddAlias(alias, cl.Name) } } - return nil +} + +// roleValidation holds the validation errors for one role's composition. +type roleValidation struct { + Role string + Errs []hclload.ValidationError +} + +// validateManifest validates every role's composition in the manifest for env, +// each against the cluster set derived from the whole manifest (member-role +// unions + aliases, with flagEntries applied last). Results are returned per +// role in manifest order. +func validateManifest(path, env, layerRoot string, skip hclload.SkipSet, opts hclload.ValidateOptions, flagEntries []clusterEntry) ([]roleValidation, error) { + roles, err := parseManifest(path, env) + if err != nil { + return nil, err + } + clusters, err := parseManifestClusters(path) + if err != nil { + return nil, err + } + schemas, err := loadManifestRoleSchemas(roles, layerRoot) + if err != nil { + return nil, err + } + + cs := hclload.NewClusterSet() + clusterSetFromRoles(&cs, clusters, schemas) + if err := applyClusterEntries(&cs, flagEntries); err != nil { + return nil, err + } + + results := make([]roleValidation, 0, len(roles)) + for _, r := range roles { + errs := hclload.ValidateOpts(schemas[r.Role].Databases, skip, cs, opts) + results = append(results, roleValidation{Role: r.Role, Errs: errs}) + } + return results, nil } // runValidate loads an HCL config (single file or layers), resolves it, and @@ -249,16 +316,22 @@ func buildManifestClusters(cs *hclload.ClusterSet, path, env, layerRoot string) // additionally requires the remote's columns to all exist on the proxy. // // Instead of listing every cluster as a -cluster flag, -manifest/-env derive the -// mappings from the same role manifest `plan` uses: each role's `cluster` (and -// `aliases`) maps to its composed layer stack for -env. Explicit -cluster flags -// are applied last, so they override or extend the manifest (e.g. NAME=@absent). +// mappings from the same role manifest `plan` uses: each `cluster` block maps to +// the union of its member roles' composed layer stacks for -env. Explicit +// -cluster flags are applied last, so they override or extend the manifest +// (e.g. NAME=@absent). +// +// With -manifest/-env and no explicit node (-layer/-config), validate runs in +// manifest-driven mode: it validates every role in the manifest, each against +// the cluster set derived from the whole manifest — one command to check a +// whole environment. func runValidate(args []string) { fs := flag.NewFlagSet("hclexp validate", flag.ExitOnError) configFlag := fs.String("config", "./cmd/hclexp/node.conf", "path to a single HCL config file (mutually exclusive with -layer)") layersFlag := fs.String("layer", "", "comma-separated list of layer directories (loaded in order)") skipFlag := fs.String("skip-validation", "", "comma-separated dependent object names to skip, or \"*\" for all") strictProxyCols := fs.Bool("strict-proxy-columns", false, "require Distributed proxy and remote to have exactly the same columns (default: proxy columns need only be a subset)") - manifestFlag := fs.String("manifest", "", "HCL role manifest to derive -cluster mappings from (each role's cluster/aliases -> its -env layer stack); requires -env") + manifestFlag := fs.String("manifest", "", "HCL role manifest to derive -cluster mappings from; requires -env. With no -layer/-config, validates every role in the manifest") envFlag := fs.String("env", "", "environment selecting each role's layer stack in -manifest") layerRootFlag := fs.String("layer-root", ".", "root directory the manifest's layer paths resolve under") var clusters clusterFlag @@ -270,6 +343,15 @@ func runValidate(args []string) { os.Exit(2) } + // Manifest-driven mode: -manifest/-env with no explicit node (-layer or + // -config) validates every role in the manifest, each against the clusters + // derived from the whole manifest. + if *manifestFlag != "" && *layersFlag == "" && !flagWasSet(fs, "config") { + runValidateManifest(*manifestFlag, *envFlag, *layerRootFlag, + hclload.ParseSkipSet(*skipFlag), hclload.ValidateOptions{StrictProxyColumns: *strictProxyCols}, clusters.entries) + return + } + schema, err := load(*configFlag, *layersFlag) if err != nil { slog.Error("failed to load config", "err", err) @@ -307,6 +389,42 @@ func runValidate(args []string) { slog.Info("schema validation passed", "databases", len(schema.Databases)) } +// flagWasSet reports whether the named flag was explicitly provided on the +// command line (as opposed to left at its default). +func flagWasSet(fs *flag.FlagSet, name string) bool { + set := false + fs.Visit(func(f *flag.Flag) { + if f.Name == name { + set = true + } + }) + return set +} + +// runValidateManifest validates every role in the manifest for env, each +// against the cluster set derived from the whole manifest. Errors are printed +// per role and it exits non-zero if any role fails. +func runValidateManifest(manifestPath, env, layerRoot string, skip hclload.SkipSet, opts hclload.ValidateOptions, flagEntries []clusterEntry) { + results, err := validateManifest(manifestPath, env, layerRoot, skip, opts, flagEntries) + if err != nil { + slog.Error("failed to validate manifest", "file", manifestPath, "env", env, "err", err) + os.Exit(1) + } + + total := 0 + for _, r := range results { + for _, e := range r.Errs { + fmt.Fprintf(os.Stderr, "validation error: [role %s] %s\n", r.Role, e.Error()) + } + total += len(r.Errs) + } + if total > 0 { + slog.Error("schema validation failed", "env", env, "roles", len(results), "errors", total) + os.Exit(1) + } + slog.Info("schema validation passed", "env", env, "roles", len(results)) +} + // runLoad loads an HCL config (single file or layers), resolves it, and // optionally writes the resolved schema back out as canonical HCL. func runLoad(args []string) { diff --git a/cmd/hclexp/hclexp_test.go b/cmd/hclexp/hclexp_test.go index b07618f..b68896f 100644 --- a/cmd/hclexp/hclexp_test.go +++ b/cmd/hclexp/hclexp_test.go @@ -641,6 +641,99 @@ cluster "aux" { "manifest alias resolves via its base cluster") } +// validateManifest validates every role's composition, resolving cross-cluster +// references against the manifest-derived cluster set. Clean roles pass; a role +// with a broken reference reports it, while its cross-cluster proxy resolves. +func TestValidateManifest_AllRoles(t *testing.T) { + root := t.TempDir() + // aux role: a clean storage table. + writeLayer(t, root, "layers/aux/aux.hcl", ` +database "posthog" { + table "sharded_web_stats" { + order_by = ["day"] + column "day" { type = "Date" } + engine "merge_tree" {} + } +}`) + // data role: a Distributed proxy into aux (resolves cross-cluster) plus a + // view referencing a table that exists nowhere (an error). + writeLayer(t, root, "layers/data/data.hcl", ` +database "posthog" { + table "web_stats" { + engine "distributed" { + cluster_name = "aux" + remote_database = "posthog" + remote_table = "sharded_web_stats" + } + column "day" { type = "Date" } + } + view "broken" { + query = "SELECT day FROM posthog.nonexistent" + } +}`) + manifest := writeTemp(t, "roles.hcl", ` +role "data" { + env "prod-us" { layers = ["layers/data"] } +} +role "aux" { + env "prod-us" { layers = ["layers/aux"] } +} +cluster "aux" { roles = ["aux"] } +cluster "posthog" { roles = ["data"] }`) + + results, err := validateManifest(manifest, "prod-us", root, hclload.ParseSkipSet(""), hclload.ValidateOptions{}, nil) + require.NoError(t, err) + require.Len(t, results, 2, "every deployed role is validated") + + byRole := map[string][]hclload.ValidationError{} + for _, r := range results { + byRole[r.Role] = r.Errs + } + require.Empty(t, byRole["aux"], "aux role is clean") + require.Len(t, byRole["data"], 1, "data role has exactly the broken view (its proxy resolved cross-cluster)") + require.Equal(t, "broken", byRole["data"][0].Object.Name) + require.Equal(t, "nonexistent", byRole["data"][0].Missing.Name) +} + +// A fully consistent manifest yields no errors on any role. +func TestValidateManifest_AllClean(t *testing.T) { + root := t.TempDir() + writeLayer(t, root, "layers/aux/aux.hcl", ` +database "posthog" { + table "sharded_web_stats" { + order_by = ["day"] + column "day" { type = "Date" } + engine "merge_tree" {} + } +}`) + writeLayer(t, root, "layers/data/data.hcl", ` +database "posthog" { + table "web_stats" { + engine "distributed" { + cluster_name = "aux" + remote_database = "posthog" + remote_table = "sharded_web_stats" + } + column "day" { type = "Date" } + } +}`) + manifest := writeTemp(t, "roles.hcl", ` +role "data" { + env "prod-us" { layers = ["layers/data"] } +} +role "aux" { + env "prod-us" { layers = ["layers/aux"] } +} +cluster "aux" { roles = ["aux"] }`) + + results, err := validateManifest(manifest, "prod-us", root, hclload.ParseSkipSet(""), hclload.ValidateOptions{}, nil) + require.NoError(t, err) + require.Len(t, results, 2) + for _, r := range results { + require.Empty(t, r.Errs, "role %s should be clean", r.Role) + } +} + // writeLayer writes content to root/rel, creating parent dirs. func writeLayer(t *testing.T, root, rel, content string) { t.Helper() diff --git a/docs/plans/2026-07-08-validate-manifest-driven.md b/docs/plans/2026-07-08-validate-manifest-driven.md new file mode 100644 index 0000000..6463cce --- /dev/null +++ b/docs/plans/2026-07-08-validate-manifest-driven.md @@ -0,0 +1,49 @@ +# validate: manifest-driven mode (validate a whole environment) + +Status: implemented 2026-07-08. +Origin: `validate -manifest` (the prior change) validates **one** node passed via +`-layer`/`-config`, using the manifest only as cross-cluster resolution context. +To check a whole environment you had to loop `validate` per node in a shell. This +adds a first-class mode that validates every role in the manifest in one command. + +## Behavior + +`hclexp validate -manifest roles.hcl -env prod-us [-layer-root .]` with **no** +`-layer`/`-config`: + +- Loads and resolves every role's composition for `-env`. +- Builds the cluster set from the manifest (member-role unions + aliases), plus + any `-cluster` flags applied last. +- Runs the full validation on **each** role's schema against that cluster set, + and reports errors prefixed with the role: `[role data] ...`. +- Exits non-zero if any role has errors. + +Mode is selected when `-manifest` is set and neither `-layer` nor an explicit +`-config` is given (detected via `flag.FlagSet.Visit`). With `-layer`/`-config`, +the existing single-node behavior (manifest as context) is unchanged. + +## Implementation (`cmd/hclexp/hclexp.go`) + +Shared helpers factored so both modes load role schemas once: + +- `loadManifestRoleSchemas(roles, layerRoot)` — role name → resolved `*Schema`. +- `clusterSetFromRoles(cs, clusters, schemas)` — union member-role databases per + cluster + aliases. `buildManifestClusters` (single-node) reuses these, loading + only cluster-referenced roles. +- `validateManifest(...) []roleValidation` — loads all roles, builds the cluster + set, validates each; returns per-role errors (testable, no `os.Exit`). +- `runValidateManifest(...)` — prints per-role errors and sets the exit code. +- `flagWasSet(fs, name)` — detects an explicit `-config`. + +## Tests + +- `validateManifest`: every role validated; a clean role passes while a role with + a broken reference reports it *and* its cross-cluster proxy still resolves. +- all-clean manifest → no errors on any role. +- Binary end-to-end: one command flags `[role data]` for a broken view, then + passes for the whole env once fixed. + +## Downstream + +posthog `check.sh` collapses its per-node validate loop into a single +`validate -manifest -env` per environment.