From 01e2749e94d6ae34d9f811fe8a89f0485e5717f8 Mon Sep 17 00:00:00 2001 From: Rafael Richards Date: Sat, 30 May 2026 22:40:52 -0400 Subject: [PATCH 1/5] harness: P0 frame contract + SplitFrame (resident harness, 5.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New internal/harness package — the host-side trigger/render client for the resident pure-M harness (design §3.1/§3.2). SplitFrame splits the deterministic ##-envelope into per-suite ^STDASSERT blocks + the LCOV block + provenance/ summary FrameMeta. It is delimiter-scanning only: no test/coverage parse logic lives here. Each suite Body is verbatim ^STDASSERT fed to the UNCHANGED mtest.ParseOutput; the ##LCOV block is verbatim LCOV fed to mcov. This is what structurally guarantees G4 cross-engine parity — both tiers speak one dialect. - ErrNoFrame (missing header) and ErrTruncated (missing/mismatched trailer, so a dropped connection is detectable); partial results returned alongside. - Unknown ## directives skip gracefully (forward-compat). - mcov.ParseLCOV: inverse of mcov.LCOV, so a coverage payload arriving as text (the ##LCOV block) joins the same ByFile/Percent consumers a host-side mcov.Run produces. Tolerates TN:/LF:/LH: records. Golden-frame tests prove each split block round-trips through the unchanged mtest.ParseOutput and the LCOV block through mcov.ByFile. Host-side, no engine. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/harness/harness.go | 188 ++++++++++++++++++++++ internal/harness/harness_test.go | 128 +++++++++++++++ internal/harness/testdata/frame_basic.txt | 23 +++ internal/mcov/parse.go | 40 +++++ internal/mcov/parse_test.go | 55 +++++++ 5 files changed, 434 insertions(+) create mode 100644 internal/harness/harness.go create mode 100644 internal/harness/harness_test.go create mode 100644 internal/harness/testdata/frame_basic.txt create mode 100644 internal/mcov/parse.go create mode 100644 internal/mcov/parse_test.go diff --git a/internal/harness/harness.go b/internal/harness/harness.go new file mode 100644 index 0000000..5394a8c --- /dev/null +++ b/internal/harness/harness.go @@ -0,0 +1,188 @@ +// Package harness is the host-side trigger/render client for the resident +// pure-M harness (design §3.1, spec §9). The harness's contract is its output +// *frame*, not its transport: a deterministic line-delimited envelope of +// verbatim ^STDASSERT per-suite blocks + a verbatim LCOV block + provenance +// tags (§3.2). This package is a thin frame-splitter — it owns NO test/coverage +// parsing. Each suite block is handed to the unchanged mtest.ParseOutput and the +// LCOV block to the unchanged mcov consumers, which is what structurally +// guarantees G4 cross-engine parity (the resident tier and the file-side tier +// speak the exact same dialect). +package harness + +import ( + "errors" + "strconv" + "strings" +) + +// Frame delimiter lines. They never collide with ^STDASSERT / LCOV content, +// which never begin with "##". +const ( + hdrHarness = "##M-HARNESS" // header: frame=/tier=/engine=/ns= + hdrSuite = "##SUITE" // ##SUITE ^NAME + hdrEnd = "##END" // ##END ^NAME exit=N (closes a suite) + hdrLCOV = "##LCOV" // ##LCOV … verbatim LCOV tracefile + hdrTrailer = "##END-HARNESS" // trailer: suites=/pass=/fail= cross-check +) + +var ( + // ErrNoFrame means the input had no ##M-HARNESS header — not a frame. + ErrNoFrame = errors.New("harness: missing ##M-HARNESS header") + // ErrTruncated means the stream ended before the ##END-HARNESS trailer, or + // the trailer's suite count disagrees with the blocks parsed — a dropped + // connection. Partial results are still returned alongside it. + ErrTruncated = errors.New("harness: truncated frame (missing or mismatched trailer)") +) + +// SuiteBlock is one per-suite payload: Name is the suite (the ##SUITE token with +// any leading ^ stripped, so it matches mtest.TestSuite.Name), Body is the +// verbatim ^STDASSERT text between the ##SUITE and ##END lines (fed unchanged to +// mtest.ParseOutput), Exit is the engine exit code from the ##END line. +type SuiteBlock struct { + Name string + Body string + Exit int +} + +// FrameMeta carries the header provenance (render label) and the trailer +// cross-check totals. +type FrameMeta struct { + Frame int + Tier string + Engine string + NS string + Suites int + Pass int + Fail int +} + +// SplitFrame splits the result frame (§3.2) into per-suite ^STDASSERT blocks, +// the LCOV block (empty when coverage was not requested), and the provenance / +// summary metadata. It is delimiter-scanning only — no test or coverage parsing +// happens here. Unrecognized ## directives are skipped (forward-compat). A +// missing header is ErrNoFrame; a missing/mismatched trailer is ErrTruncated, +// returned alongside whatever was parsed so a caller can still render it. +func SplitFrame(frame string) ([]SuiteBlock, string, FrameMeta, error) { + var ( + meta FrameMeta + suites []SuiteBlock + lcov strings.Builder + sawHeader bool + sawTrailer bool + inLCOV bool + cur *SuiteBlock + body strings.Builder + ) + + flushSuite := func() { + if cur != nil { + cur.Body = body.String() + suites = append(suites, *cur) + cur = nil + body.Reset() + } + } + + for _, line := range strings.Split(frame, "\n") { + switch { + case strings.HasPrefix(line, hdrHarness): + sawHeader = true + parseHeader(line, &meta) + case strings.HasPrefix(line, hdrTrailer): + flushSuite() + inLCOV = false + sawTrailer = true + parseTrailer(line, &meta) + case strings.HasPrefix(line, hdrSuite): + flushSuite() + inLCOV = false + name := strings.TrimSpace(strings.TrimPrefix(line, hdrSuite)) + name = strings.TrimPrefix(name, "^") + cur = &SuiteBlock{Name: name, Exit: -1} + case strings.HasPrefix(line, hdrEnd) && !strings.HasPrefix(line, hdrTrailer): + if cur != nil { + cur.Exit = parseExit(line) + cur.Body = body.String() + suites = append(suites, *cur) + cur = nil + body.Reset() + } + case line == hdrLCOV || strings.HasPrefix(line, hdrLCOV+" "): + flushSuite() + inLCOV = true + case strings.HasPrefix(line, "##"): + // Unknown directive — skip, never fatal (forward-compat). + case inLCOV: + lcov.WriteString(line) + lcov.WriteByte('\n') + case cur != nil: + body.WriteString(line) + body.WriteByte('\n') + } + } + flushSuite() + + if !sawHeader { + return suites, lcov.String(), meta, ErrNoFrame + } + if !sawTrailer || meta.Suites != len(suites) { + return suites, lcov.String(), meta, ErrTruncated + } + return suites, lcov.String(), meta, nil +} + +// parseHeader reads `##M-HARNESS frame=1 tier=integration engine=iris ns=VEHU`. +func parseHeader(line string, m *FrameMeta) { + for k, v := range kvFields(strings.TrimPrefix(line, hdrHarness)) { + switch k { + case "frame": + m.Frame = atoi(v) + case "tier": + m.Tier = v + case "engine": + m.Engine = v + case "ns": + m.NS = v + } + } +} + +// parseTrailer reads `##END-HARNESS suites=2 pass=2 fail=1`. +func parseTrailer(line string, m *FrameMeta) { + for k, v := range kvFields(strings.TrimPrefix(line, hdrTrailer)) { + switch k { + case "suites": + m.Suites = atoi(v) + case "pass": + m.Pass = atoi(v) + case "fail": + m.Fail = atoi(v) + } + } +} + +// parseExit reads the exit code from `##END ^NAME exit=N`. +func parseExit(line string) int { + for k, v := range kvFields(line) { + if k == "exit" { + return atoi(v) + } + } + return -1 +} + +// kvFields splits a run of space-separated key=value tokens into a map. +func kvFields(s string) map[string]string { + out := map[string]string{} + for _, tok := range strings.Fields(s) { + if eq := strings.IndexByte(tok, '='); eq > 0 { + out[tok[:eq]] = tok[eq+1:] + } + } + return out +} + +func atoi(s string) int { + n, _ := strconv.Atoi(s) + return n +} diff --git a/internal/harness/harness_test.go b/internal/harness/harness_test.go new file mode 100644 index 0000000..42d5b22 --- /dev/null +++ b/internal/harness/harness_test.go @@ -0,0 +1,128 @@ +package harness_test + +import ( + "errors" + "os" + "strings" + "testing" + + "github.com/vista-cloud-dev/m-cli/internal/harness" + "github.com/vista-cloud-dev/m-cli/internal/mcov" + "github.com/vista-cloud-dev/m-cli/internal/mtest" +) + +func goldenFrame(t *testing.T) string { + t.Helper() + b, err := os.ReadFile("testdata/frame_basic.txt") + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestSplitFrameMeta(t *testing.T) { + suites, lcov, meta, err := harness.SplitFrame(goldenFrame(t)) + if err != nil { + t.Fatalf("SplitFrame: %v", err) + } + if meta.Frame != 1 || meta.Tier != "integration" || meta.Engine != "iris" || meta.NS != "VEHU" { + t.Errorf("meta header = %+v", meta) + } + if meta.Suites != 2 || meta.Pass != 2 || meta.Fail != 1 { + t.Errorf("meta trailer = suites %d pass %d fail %d, want 2/2/1", meta.Suites, meta.Pass, meta.Fail) + } + if len(suites) != 2 { + t.Fatalf("got %d suite blocks, want 2", len(suites)) + } + if lcov == "" || !strings.Contains(lcov, "SF:MATH.m") { + t.Errorf("lcov block missing SF:MATH.m: %q", lcov) + } +} + +// The whole point of the frame-as-contract: each per-suite block is verbatim +// ^STDASSERT text that the UNCHANGED mtest.ParseOutput consumes — no new parse +// logic in the splitter. +func TestSplitFrameSuiteBlocksRoundTripThroughMtest(t *testing.T) { + suites, _, _, err := harness.SplitFrame(goldenFrame(t)) + if err != nil { + t.Fatalf("SplitFrame: %v", err) + } + want := map[string]struct { + total, passed, failed int + ok bool + }{ + "MATHTST": {2, 1, 1, false}, + "FILEMANTST": {1, 1, 0, true}, + } + for _, sb := range suites { + w, ok := want[sb.Name] + if !ok { + t.Errorf("unexpected suite %q", sb.Name) + continue + } + if sb.Exit != 0 { + t.Errorf("%s exit = %d, want 0", sb.Name, sb.Exit) + } + s := mtest.ParseOutput(sb.Body) + if s.Total != w.total || s.Passed != w.passed || s.Failed != w.failed || s.OK != w.ok { + t.Errorf("%s ParseOutput = %d/%d/%d ok=%v, want %d/%d/%d ok=%v", + sb.Name, s.Total, s.Passed, s.Failed, s.OK, w.total, w.passed, w.failed, w.ok) + } + } +} + +// The ##LCOV block is verbatim LCOV the UNCHANGED mcov consumers understand. +func TestSplitFrameLCOVRoundTripsThroughMcov(t *testing.T) { + _, lcov, _, err := harness.SplitFrame(goldenFrame(t)) + if err != nil { + t.Fatalf("SplitFrame: %v", err) + } + r, err := mcov.ParseLCOV(lcov) + if err != nil { + t.Fatalf("ParseLCOV: %v", err) + } + bf := mcov.ByFile(r) + if len(bf) != 1 || bf[0].Path != "MATH.m" || bf[0].Total != 2 || bf[0].Covered != 1 { + t.Errorf("ByFile = %+v, want MATH.m 1/2", bf) + } +} + +func TestSplitFrameTruncatedStreamDetected(t *testing.T) { + full := goldenFrame(t) + // Drop the ##END-HARNESS trailer — a dropped connection. + cut := full[:strings.Index(full, "##END-HARNESS")] + suites, _, _, err := harness.SplitFrame(cut) + if !errors.Is(err, harness.ErrTruncated) { + t.Fatalf("err = %v, want ErrTruncated", err) + } + // Partial results still come back so a caller can render what arrived. + if len(suites) != 2 { + t.Errorf("got %d suites, want 2 (partial)", len(suites)) + } +} + +func TestSplitFrameMissingHeader(t *testing.T) { + _, _, _, err := harness.SplitFrame("##SUITE ^X\nAll tests passed.\n##END ^X exit=0\n") + if !errors.Is(err, harness.ErrNoFrame) { + t.Fatalf("err = %v, want ErrNoFrame", err) + } +} + +// Unrecognized ## lines degrade gracefully (skipped, not fatal) — forward-compat +// with frame versions that add directives. +func TestSplitFrameUnknownDirectiveSkipped(t *testing.T) { + frame := "##M-HARNESS frame=2 tier=pure-logic engine=ydb ns=USER\n" + + "##FUTURE something\n" + + "##SUITE ^XTST\nAll tests passed.\nResults: 1 tests 1 passed 0 failed\n##END ^XTST exit=0\n" + + "##END-HARNESS suites=1 pass=1 fail=0\n" + suites, _, meta, err := harness.SplitFrame(frame) + if err != nil { + t.Fatalf("SplitFrame: %v", err) + } + if meta.Frame != 2 || meta.Tier != "pure-logic" { + t.Errorf("meta = %+v", meta) + } + if len(suites) != 1 || suites[0].Name != "XTST" { + t.Fatalf("suites = %+v, want one XTST", suites) + } +} diff --git a/internal/harness/testdata/frame_basic.txt b/internal/harness/testdata/frame_basic.txt new file mode 100644 index 0000000..4951408 --- /dev/null +++ b/internal/harness/testdata/frame_basic.txt @@ -0,0 +1,23 @@ +##M-HARNESS frame=1 tier=integration engine=iris ns=VEHU +##SUITE ^MATHTST + PASS add: 2+2 + FAIL div: 1/0 traps + expected: 0 + actual: +Results: 2 tests 1 passed 1 failed +1 test(s) FAILED. +##END ^MATHTST exit=0 +##SUITE ^FILEMANTST + PASS patient #1 DFN resolves +Results: 1 tests 1 passed 0 failed +All tests passed. +##END ^FILEMANTST exit=0 +##LCOV +TN: +SF:MATH.m +DA:3,1 +DA:4,0 +LF:2 +LH:1 +end_of_record +##END-HARNESS suites=2 pass=2 fail=1 diff --git a/internal/mcov/parse.go b/internal/mcov/parse.go new file mode 100644 index 0000000..06c779b --- /dev/null +++ b/internal/mcov/parse.go @@ -0,0 +1,40 @@ +package mcov + +import ( + "strconv" + "strings" +) + +// ParseLCOV reads an LCOV tracefile into a Result. It is the inverse of LCOV: +// a coverage payload that arrives as text (the resident harness ##LCOV frame +// block, §3.2) parses back here so it feeds the same ByFile / Percent / gutter +// consumers a host-side mcov.Run produces — the parity contract. Only SF: and +// DA: lines carry data; TN:/LF:/LH:/end_of_record and anything else are ignored +// (so output of this package's own LCOV, with its TN:/LF:/LH: lines, round-trips +// and a foreign producer's extra records do not break the parse). +func ParseLCOV(text string) (Result, error) { + var r Result + path := "" + for _, raw := range strings.Split(text, "\n") { + line := strings.TrimRight(raw, "\r") + switch { + case strings.HasPrefix(line, "SF:"): + path = line[len("SF:"):] + case strings.HasPrefix(line, "DA:") && path != "": + rest := line[len("DA:"):] + comma := strings.IndexByte(rest, ',') + if comma < 0 { + continue + } + ln, err1 := strconv.Atoi(strings.TrimSpace(rest[:comma])) + hits, err2 := strconv.Atoi(strings.TrimSpace(rest[comma+1:])) + if err1 != nil || err2 != nil { + continue + } + r.Lines = append(r.Lines, LineCov{Path: path, Line: ln, Hits: hits}) + case line == "end_of_record": + path = "" + } + } + return r, nil +} diff --git a/internal/mcov/parse_test.go b/internal/mcov/parse_test.go new file mode 100644 index 0000000..7b13010 --- /dev/null +++ b/internal/mcov/parse_test.go @@ -0,0 +1,55 @@ +package mcov + +import "testing" + +// ParseLCOV reads an LCOV tracefile back into a Result so a coverage payload +// that arrived as text (e.g. the resident harness's ##LCOV frame block) joins +// the same ByFile / Percent consumers a host-side mcov.Run produces. +func TestParseLCOVRoundTrip(t *testing.T) { + orig := Result{Lines: []LineCov{ + {Path: "MATH.m", Line: 3, Hits: 1}, + {Path: "MATH.m", Line: 4, Hits: 0}, + {Path: "STR.m", Line: 7, Hits: 3}, + }} + got, err := ParseLCOV(LCOV(orig)) + if err != nil { + t.Fatalf("ParseLCOV: %v", err) + } + // ByFile rollup must match the original — the parity contract. + a, b := ByFile(orig), ByFile(got) + if len(a) != len(b) { + t.Fatalf("ByFile len %d != %d", len(a), len(b)) + } + for i := range a { + if a[i].Path != b[i].Path || a[i].Total != b[i].Total || a[i].Covered != b[i].Covered { + t.Errorf("file %d: got %+v, want %+v", i, b[i], a[i]) + } + } +} + +func TestParseLCOVToleratesExtraRecords(t *testing.T) { + // TN:/LF:/LH:/blank lines and an unrelated leading comment are ignored; + // only SF:/DA:/end_of_record carry data. + text := "TN:\nSF:FOO.m\nDA:1,2\nDA:2,0\nLF:2\nLH:1\nend_of_record\n" + r, err := ParseLCOV(text) + if err != nil { + t.Fatalf("ParseLCOV: %v", err) + } + if r.Total() != 2 || r.Covered() != 1 { + t.Errorf("got %d/%d, want 1/2", r.Covered(), r.Total()) + } + bf := ByFile(r) + if len(bf) != 1 || bf[0].Path != "FOO.m" { + t.Fatalf("ByFile = %+v, want one FOO.m", bf) + } +} + +func TestParseLCOVEmpty(t *testing.T) { + r, err := ParseLCOV("") + if err != nil { + t.Fatalf("ParseLCOV(empty): %v", err) + } + if r.Total() != 0 { + t.Errorf("Total = %d, want 0", r.Total()) + } +} From 4109c1fa9fec35eec6e0e9b519da742de26c4a6b Mon Sep 17 00:00:00 2001 From: Rafael Richards Date: Sat, 30 May 2026 23:03:34 -0400 Subject: [PATCH 2/5] harness: P1 Trigger + live cross-engine parity test (resident harness, 5.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Trigger(ctx, eng, scope) invokes RUN^STDHARN via the engine adapter and returns the raw result frame (design §3.1 CLI trigger path). Pure delegation; scope rides $ZCMDLINE, the frame comes back on stdout, and the same frame travels over T.1's WebSocket unchanged. - TestResidentParityYDB (opt-in, M_TEST_LIVE) is stage 5.1's gate G4: the SAME *TST suites yield IDENTICAL mtest.Summary file-side (one process per suite) vs resident (one RUN^STDHARN, split via SplitFrame). Pure-M makes this exercisable on YDB with no IRIS. Covers a deliberately-failing fixture (testdata/PARITYFAILTST.m) to prove FAIL-path parity through the no-halt mechanism — the failing suite frames without halting the orchestrator. Verified live against the m-test-engine YDB container: STDMATHTST/STDSTRTST/ STDSEMVERTST/STDHEXTST + PARITYFAILTST all parity-match across both tiers. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/harness/parity_live_test.go | 136 ++++++++++++++++++++++ internal/harness/testdata/PARITYFAILTST.m | 18 +++ internal/harness/trigger.go | 26 +++++ 3 files changed, 180 insertions(+) create mode 100644 internal/harness/parity_live_test.go create mode 100644 internal/harness/testdata/PARITYFAILTST.m create mode 100644 internal/harness/trigger.go diff --git a/internal/harness/parity_live_test.go b/internal/harness/parity_live_test.go new file mode 100644 index 0000000..07c1bef --- /dev/null +++ b/internal/harness/parity_live_test.go @@ -0,0 +1,136 @@ +package harness_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/vista-cloud-dev/m-cli/internal/engine" + "github.com/vista-cloud-dev/m-cli/internal/harness" + "github.com/vista-cloud-dev/m-cli/internal/mtest" + "github.com/vista-cloud-dev/m-parse/parse" +) + +// TestResidentParityYDB is stage 5.1's gate (G4): the SAME *TST suites must +// yield IDENTICAL mtest.Summary results whether run file-side (host-orchestrated, +// one process per suite) or resident (RUN^STDHARN, all suites in one process, +// framed). Portable pure-M makes this exercisable on YDB with no IRIS. Opt-in: +// +// M_TEST_LIVE=1 M_STDLIB_SRC=$HOME/vista-cloud-dev/m-stdlib/src \ +// go test ./internal/harness/ -run TestResidentParityYDB +func TestResidentParityYDB(t *testing.T) { + if os.Getenv("M_TEST_LIVE") == "" { + t.Skip("set M_TEST_LIVE=1 (+ M_STDLIB_SRC) to run the live YDB parity test") + } + stdlib := os.Getenv("M_STDLIB_SRC") + if stdlib == "" { + t.Skip("set M_STDLIB_SRC to the m-stdlib src dir (provides STDHARN/STDASSERT)") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available") + } + container := os.Getenv("M_TEST_ENGINE_CONTAINER") + if container == "" { + container = "m-test-engine" + } + ctx := context.Background() + + // Pure-logic suites (deterministic, no shared global state) so running them + // in one resident process matches one-process-per-suite file-side. Plus the + // deliberately-failing fixture, to prove FAIL-path parity through no-halt. + suiteNames := []string{"STDMATHTST", "STDSTRTST", "STDSEMVERTST", "STDHEXTST", "PARITYFAILTST"} + + stageDir := "/m-work/harness-parity" + var files []string + srcEntries, err := os.ReadDir(stdlib) + if err != nil { + t.Fatalf("read M_STDLIB_SRC: %v", err) + } + for _, e := range srcEntries { + if !e.IsDir() && filepath.Ext(e.Name()) == ".m" { + files = append(files, filepath.Join(stdlib, e.Name())) + } + } + testsDir := filepath.Join(filepath.Dir(stdlib), "tests") + var suiteFiles []string + for _, n := range suiteNames { + var p string + if n == "PARITYFAILTST" { + p = filepath.Join("testdata", n+".m") + } else { + p = filepath.Join(testsDir, n+".m") + } + if _, err := os.Stat(p); err != nil { + t.Fatalf("suite file missing: %s", p) + } + files = append(files, p) + suiteFiles = append(suiteFiles, p) + } + + if err := engine.DockerStage(ctx, container, stageDir, files); err != nil { + t.Fatalf("stage: %v", err) + } + defer engine.DockerUnstage(ctx, container, stageDir) + + eng := engine.New(engine.YDB, engine.Options{Runner: engine.DockerRunner(container, stageDir)}) + + // File-side tier: discover + run each suite host-orchestrated. + p, err := parse.New(ctx) + if err != nil { + t.Fatalf("parse.New: %v", err) + } + defer func() { _ = p.Close(ctx) }() + suites, err := mtest.Discover(p, suiteFiles) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fileSide, err := mtest.Run(ctx, eng, suites) + if err != nil { + t.Fatalf("file-side Run: %v", err) + } + fileByName := map[string]mtest.Summary{} + for _, r := range fileSide { + fileByName[r.Suite] = r.Summary + } + + // Resident tier: one RUN^STDHARN, split the frame, parse each block. + frame, err := harness.Trigger(ctx, eng, suiteNames) + if err != nil { + t.Fatalf("Trigger: %v", err) + } + blocks, _, meta, err := harness.SplitFrame(frame) + if err != nil { + t.Fatalf("SplitFrame: %v\nframe:\n%s", err, frame) + } + if meta.Suites != len(suiteNames) { + t.Errorf("trailer suites=%d, want %d", meta.Suites, len(suiteNames)) + } + resByName := map[string]mtest.Summary{} + for _, b := range blocks { + resByName[b.Name] = mtest.ParseOutput(b.Body) + } + + // G4: per-suite Summary must be identical across tiers. + for _, n := range suiteNames { + fs, ok := fileByName[n] + if !ok { + t.Errorf("%s: missing from file-side results", n) + continue + } + rs, ok := resByName[n] + if !ok { + t.Errorf("%s: missing from resident frame", n) + continue + } + if fs.Total != rs.Total || fs.Passed != rs.Passed || fs.Failed != rs.Failed || fs.OK != rs.OK { + t.Errorf("%s parity MISMATCH:\n file-side: %d/%d/%d ok=%v\n resident: %d/%d/%d ok=%v", + n, fs.Total, fs.Passed, fs.Failed, fs.OK, rs.Total, rs.Passed, rs.Failed, rs.OK) + } + } + // The failing fixture must read as a failure on both tiers (not a false pass). + if rs := resByName["PARITYFAILTST"]; rs.OK || rs.Failed != 1 { + t.Errorf("PARITYFAILTST resident = %+v, want a failure (Failed=1, OK=false)", rs) + } +} diff --git a/internal/harness/testdata/PARITYFAILTST.m b/internal/harness/testdata/PARITYFAILTST.m new file mode 100644 index 0000000..665cb32 --- /dev/null +++ b/internal/harness/testdata/PARITYFAILTST.m @@ -0,0 +1,18 @@ +PARITYFAILTST ; Parity fixture: 1 pass + 1 fail. + ; Proves the resident no-halt path frames a FAILING suite identically to + ; the file-side per-process runner (where report^STDASSERT halts). Kept + ; out of m-stdlib's own tests/ so `make test` never runs it standalone. + new pass,fail + do start^STDASSERT(.pass,.fail) + do tParityPasses(.pass,.fail) + do tParityFails(.pass,.fail) + do report^STDASSERT(pass,fail) + quit + ; +tParityPasses(pass,fail) ;@TEST "an assertion that passes" + do eq^STDASSERT(.pass,.fail,1,1,"one is one") + quit + ; +tParityFails(pass,fail) ;@TEST "an assertion that fails on purpose" + do eq^STDASSERT(.pass,.fail,1,2,"one is two (deliberate)") + quit diff --git a/internal/harness/trigger.go b/internal/harness/trigger.go new file mode 100644 index 0000000..47c3afa --- /dev/null +++ b/internal/harness/trigger.go @@ -0,0 +1,26 @@ +package harness + +import ( + "context" + "strings" + + "github.com/vista-cloud-dev/m-cli/internal/engine" +) + +// Trigger invokes the resident orchestrator RUN^STDHARN over the engine adapter +// and returns the raw result frame (design §3.1, the CLI trigger path). It is +// pure delegation — the scope (suite routine names) is passed as the engine's +// command line ($ZCMDLINE on YDB), and the frame comes back on the engine's +// stdout. No engine-specific logic beyond what the adapter abstracts; the same +// frame travels over T.1's WebSocket transport unchanged. +// +// The routines (STDHARN, STDASSERT, the suites) must already be available to the +// engine — staged/loaded by the caller, exactly as the host-orchestrated path +// arranges them. +func Trigger(ctx context.Context, eng engine.Engine, scope []string) (string, error) { + res, err := eng.RunRoutine(ctx, "RUN^STDHARN", strings.Join(scope, " ")) + if err != nil { + return "", err + } + return res.Stdout, nil +} From bdab321ed450141d41987342cb037e6050d9545a Mon Sep 17 00:00:00 2001 From: Rafael Richards Date: Sat, 30 May 2026 23:12:48 -0400 Subject: [PATCH 3/5] harness: portable Trigger + IRIS parity test (G4 test-tier on both engines) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drive run^STDHARN via RunScript (an M argument) instead of RUN^STDHARN's $ZCMDLINE — IRIS has no $ZCMDLINE (), so the script path is engine- portable. Generalize TestResidentParity into an engine-parameterized helper + add TestResidentParityIRIS (stage dir under /tmp on IRIS, IrisStageLoad). Result: the SAME suites + the deliberately-failing fixture yield IDENTICAL mtest.Summary resident vs file-side on BOTH the YDB file-side tier AND the IRIS resident tier — the test half of G4 across both engines. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/harness/parity_live_test.go | 96 ++++++++++++++++++++-------- internal/harness/trigger.go | 18 ++++-- 2 files changed, 81 insertions(+), 33 deletions(-) diff --git a/internal/harness/parity_live_test.go b/internal/harness/parity_live_test.go index 07c1bef..5f0d96a 100644 --- a/internal/harness/parity_live_test.go +++ b/internal/harness/parity_live_test.go @@ -13,6 +13,11 @@ import ( "github.com/vista-cloud-dev/m-parse/parse" ) +// Pure-logic suites (deterministic, no shared global state) so running them in +// one resident process matches one-process-per-suite file-side. Plus the +// deliberately-failing fixture, to prove FAIL-path parity through no-halt. +var paritySuites = []string{"STDMATHTST", "STDSTRTST", "STDSEMVERTST", "STDHEXTST", "PARITYFAILTST"} + // TestResidentParityYDB is stage 5.1's gate (G4): the SAME *TST suites must // yield IDENTICAL mtest.Summary results whether run file-side (host-orchestrated, // one process per suite) or resident (RUN^STDHARN, all suites in one process, @@ -21,8 +26,28 @@ import ( // M_TEST_LIVE=1 M_STDLIB_SRC=$HOME/vista-cloud-dev/m-stdlib/src \ // go test ./internal/harness/ -run TestResidentParityYDB func TestResidentParityYDB(t *testing.T) { + stdlib := liveStdlibSrc(t) + residentParity(t, engine.YDB, envOr("M_TEST_ENGINE_CONTAINER", "m-test-engine"), "", stdlib) +} + +// TestResidentParityIRIS is the IRIS half of G4: the same parity but on the +// resident IRIS tier (the integration tier's real substrate). Opt-in: +// +// M_TEST_LIVE=1 M_STDLIB_SRC=$HOME/vista-cloud-dev/m-stdlib/src \ +// M_IRIS_CONTAINER=vista-iris go test ./internal/harness/ -run TestResidentParityIRIS +func TestResidentParityIRIS(t *testing.T) { + stdlib := liveStdlibSrc(t) + container := os.Getenv("M_IRIS_CONTAINER") + if container == "" { + t.Skip("set M_IRIS_CONTAINER (e.g. vista-iris) to run the IRIS parity test") + } + residentParity(t, engine.IRIS, container, envOr("M_IRIS_NAMESPACE", "USER"), stdlib) +} + +func liveStdlibSrc(t *testing.T) string { + t.Helper() if os.Getenv("M_TEST_LIVE") == "" { - t.Skip("set M_TEST_LIVE=1 (+ M_STDLIB_SRC) to run the live YDB parity test") + t.Skip("set M_TEST_LIVE=1 (+ M_STDLIB_SRC) to run the live parity test") } stdlib := os.Getenv("M_STDLIB_SRC") if stdlib == "" { @@ -31,18 +56,28 @@ func TestResidentParityYDB(t *testing.T) { if _, err := exec.LookPath("docker"); err != nil { t.Skip("docker not available") } - container := os.Getenv("M_TEST_ENGINE_CONTAINER") - if container == "" { - container = "m-test-engine" - } - ctx := context.Background() + return stdlib +} - // Pure-logic suites (deterministic, no shared global state) so running them - // in one resident process matches one-process-per-suite file-side. Plus the - // deliberately-failing fixture, to prove FAIL-path parity through no-halt. - suiteNames := []string{"STDMATHTST", "STDSTRTST", "STDSEMVERTST", "STDHEXTST", "PARITYFAILTST"} +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} +// residentParity runs the parity gate against one engine: stage the suites + +// their deps, run them file-side (one process per suite) and resident (one +// RUN^STDHARN), and assert identical per-suite Summaries. +func residentParity(t *testing.T, kind engine.Kind, container, namespace, stdlib string) { + t.Helper() + ctx := context.Background() + // IRIS stages under /tmp (writable); YDB under the m-test-engine /m-work mount. stageDir := "/m-work/harness-parity" + if kind == engine.IRIS { + stageDir = "/tmp/harness-parity" + } + var files []string srcEntries, err := os.ReadDir(stdlib) if err != nil { @@ -55,12 +90,10 @@ func TestResidentParityYDB(t *testing.T) { } testsDir := filepath.Join(filepath.Dir(stdlib), "tests") var suiteFiles []string - for _, n := range suiteNames { - var p string + for _, n := range paritySuites { + p := filepath.Join(testsDir, n+".m") if n == "PARITYFAILTST" { p = filepath.Join("testdata", n+".m") - } else { - p = filepath.Join(testsDir, n+".m") } if _, err := os.Stat(p); err != nil { t.Fatalf("suite file missing: %s", p) @@ -69,12 +102,20 @@ func TestResidentParityYDB(t *testing.T) { suiteFiles = append(suiteFiles, p) } - if err := engine.DockerStage(ctx, container, stageDir, files); err != nil { - t.Fatalf("stage: %v", err) + var eng engine.Engine + if kind == engine.IRIS { + eng = engine.New(engine.IRIS, engine.Options{Runner: engine.DockerRunner(container, ""), Namespace: namespace}) + if err := engine.IrisStageLoad(ctx, eng, container, stageDir, files); err != nil { + t.Fatalf("iris stage: %v", err) + } + defer engine.DockerUnstage(ctx, container, stageDir) + } else { + if err := engine.DockerStage(ctx, container, stageDir, files); err != nil { + t.Fatalf("stage: %v", err) + } + defer engine.DockerUnstage(ctx, container, stageDir) + eng = engine.New(engine.YDB, engine.Options{Runner: engine.DockerRunner(container, stageDir)}) } - defer engine.DockerUnstage(ctx, container, stageDir) - - eng := engine.New(engine.YDB, engine.Options{Runner: engine.DockerRunner(container, stageDir)}) // File-side tier: discover + run each suite host-orchestrated. p, err := parse.New(ctx) @@ -96,7 +137,7 @@ func TestResidentParityYDB(t *testing.T) { } // Resident tier: one RUN^STDHARN, split the frame, parse each block. - frame, err := harness.Trigger(ctx, eng, suiteNames) + frame, err := harness.Trigger(ctx, eng, paritySuites) if err != nil { t.Fatalf("Trigger: %v", err) } @@ -104,8 +145,11 @@ func TestResidentParityYDB(t *testing.T) { if err != nil { t.Fatalf("SplitFrame: %v\nframe:\n%s", err, frame) } - if meta.Suites != len(suiteNames) { - t.Errorf("trailer suites=%d, want %d", meta.Suites, len(suiteNames)) + if meta.Suites != len(paritySuites) { + t.Errorf("trailer suites=%d, want %d", meta.Suites, len(paritySuites)) + } + if meta.Engine != string(kind) { + t.Errorf("frame engine=%q, want %q", meta.Engine, kind) } resByName := map[string]mtest.Summary{} for _, b := range blocks { @@ -113,7 +157,7 @@ func TestResidentParityYDB(t *testing.T) { } // G4: per-suite Summary must be identical across tiers. - for _, n := range suiteNames { + for _, n := range paritySuites { fs, ok := fileByName[n] if !ok { t.Errorf("%s: missing from file-side results", n) @@ -125,12 +169,12 @@ func TestResidentParityYDB(t *testing.T) { continue } if fs.Total != rs.Total || fs.Passed != rs.Passed || fs.Failed != rs.Failed || fs.OK != rs.OK { - t.Errorf("%s parity MISMATCH:\n file-side: %d/%d/%d ok=%v\n resident: %d/%d/%d ok=%v", - n, fs.Total, fs.Passed, fs.Failed, fs.OK, rs.Total, rs.Passed, rs.Failed, rs.OK) + t.Errorf("%s parity MISMATCH on %s:\n file-side: %d/%d/%d ok=%v\n resident: %d/%d/%d ok=%v", + n, kind, fs.Total, fs.Passed, fs.Failed, fs.OK, rs.Total, rs.Passed, rs.Failed, rs.OK) } } // The failing fixture must read as a failure on both tiers (not a false pass). if rs := resByName["PARITYFAILTST"]; rs.OK || rs.Failed != 1 { - t.Errorf("PARITYFAILTST resident = %+v, want a failure (Failed=1, OK=false)", rs) + t.Errorf("PARITYFAILTST resident (%s) = %+v, want a failure (Failed=1, OK=false)", kind, rs) } } diff --git a/internal/harness/trigger.go b/internal/harness/trigger.go index 47c3afa..7eee64f 100644 --- a/internal/harness/trigger.go +++ b/internal/harness/trigger.go @@ -7,18 +7,22 @@ import ( "github.com/vista-cloud-dev/m-cli/internal/engine" ) -// Trigger invokes the resident orchestrator RUN^STDHARN over the engine adapter +// Trigger invokes the resident orchestrator run^STDHARN over the engine adapter // and returns the raw result frame (design §3.1, the CLI trigger path). It is -// pure delegation — the scope (suite routine names) is passed as the engine's -// command line ($ZCMDLINE on YDB), and the frame comes back on the engine's -// stdout. No engine-specific logic beyond what the adapter abstracts; the same -// frame travels over T.1's WebSocket transport unchanged. +// pure delegation — the scope (suite routine names) is passed straight to +// run^STDHARN and the frame comes back on the engine's stdout. The same frame +// travels over T.1's WebSocket transport unchanged. +// +// It drives run^STDHARN through RunScript (direct mode) rather than RunRoutine +// so it is engine-portable: the YDB-only RUN^STDHARN entry reads $ZCMDLINE, +// which IRIS lacks, whereas passing scope as an M argument works on both. // // The routines (STDHARN, STDASSERT, the suites) must already be available to the // engine — staged/loaded by the caller, exactly as the host-orchestrated path -// arranges them. +// arranges them. Suite names are routine names (no quoting hazard). func Trigger(ctx context.Context, eng engine.Engine, scope []string) (string, error) { - res, err := eng.RunRoutine(ctx, "RUN^STDHARN", strings.Join(scope, " ")) + script := "do run^STDHARN(\"" + strings.Join(scope, " ") + "\")\nhalt\n" + res, err := eng.RunScript(ctx, script) if err != nil { return "", err } From 954722c28bf0f0e85aa6b25f6a2b004ef96ccdf3 Mon Sep 17 00:00:00 2001 From: Rafael Richards Date: Sat, 30 May 2026 23:26:20 -0400 Subject: [PATCH 4/5] harness/mcov: resident coverage join + ##MON frame block (P2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - SplitFrame extracts the raw ##MON line-monitor block (alongside suites/LCOV); ##END-MON never closes a suite ##END. TestSplitFrameMonBlock covers it. - mcov.FromMonitor joins a ##MON block (MLINE:routine:line:count) to the parse- tree executable lines — the host-side half of resident IRIS coverage. Shares joinMon with the host-orchestrated runIris, so resident == file-side coverage by construction. TestFromMonitor (host-side, no engine). - harness.TriggerCoverage drives cov^STDHARN (RunScript, portable). Gate (G4 coverage half): TestResidentCoverageParityIRIS proves resident IRIS coverage (cov^STDHARN → ##MON → FromMonitor) ByFile == host-orchestrated IRIS coverage (mcov.Run → %Monitor): STDMATH 29/30, host == resident. With 4.1's host IRIS==YDB LCOV parity, coverage parity holds across both tiers. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/harness/coverage_parity_live_test.go | 107 ++++++++++++++++++ internal/harness/harness.go | 37 ++++-- internal/harness/harness_test.go | 33 +++++- internal/harness/parity_live_test.go | 2 +- internal/harness/trigger.go | 14 +++ internal/mcov/frommonitor_test.go | 30 +++++ internal/mcov/iris.go | 27 ++++- 7 files changed, 229 insertions(+), 21 deletions(-) create mode 100644 internal/harness/coverage_parity_live_test.go create mode 100644 internal/mcov/frommonitor_test.go diff --git a/internal/harness/coverage_parity_live_test.go b/internal/harness/coverage_parity_live_test.go new file mode 100644 index 0000000..b305fee --- /dev/null +++ b/internal/harness/coverage_parity_live_test.go @@ -0,0 +1,107 @@ +package harness_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/vista-cloud-dev/m-cli/internal/engine" + "github.com/vista-cloud-dev/m-cli/internal/harness" + "github.com/vista-cloud-dev/m-cli/internal/mcov" + "github.com/vista-cloud-dev/m-parse/parse" +) + +// TestResidentCoverageParityIRIS is the coverage half of G4 on the resident IRIS +// tier: resident coverage (cov^STDHARN → raw ##MON block → mcov.FromMonitor) must +// roll up to the SAME ByFile coverage as the host-orchestrated IRIS path +// (mcov.Run → %Monitor). Both use the same monitor data and the same parse-tree +// denominator, so they agree by construction. Resident coverage is IRIS-only +// (YDB stays the host-side view "TRACE" path), so this test is IRIS-bound. Opt-in: +// +// M_TEST_LIVE=1 M_STDLIB_SRC=$HOME/vista-cloud-dev/m-stdlib/src \ +// M_IRIS_CONTAINER=vista-iris go test ./internal/harness/ -run TestResidentCoverageParityIRIS +func TestResidentCoverageParityIRIS(t *testing.T) { + stdlib := liveStdlibSrc(t) + container := os.Getenv("M_IRIS_CONTAINER") + if container == "" { + t.Skip("set M_IRIS_CONTAINER (e.g. vista-iris) to run the IRIS coverage parity test") + } + ns := envOr("M_IRIS_NAMESPACE", "USER") + ctx := context.Background() + stageDir := "/tmp/harness-cov-parity" + + // Cover STDMATH (a pure-logic module) exercised by STDMATHTST. + covRoutine := "STDMATH" + suite := "STDMATHTST" + routinePath := filepath.Join(stdlib, covRoutine+".m") + suitePath := filepath.Join(filepath.Dir(stdlib), "tests", suite+".m") + + var files []string + srcEntries, err := os.ReadDir(stdlib) + if err != nil { + t.Fatalf("read M_STDLIB_SRC: %v", err) + } + for _, e := range srcEntries { + if !e.IsDir() && filepath.Ext(e.Name()) == ".m" { + files = append(files, filepath.Join(stdlib, e.Name())) + } + } + files = append(files, suitePath) + + eng := engine.New(engine.IRIS, engine.Options{Runner: engine.DockerRunner(container, ""), Namespace: ns}) + if err := engine.IrisStageLoad(ctx, eng, container, stageDir, files); err != nil { + t.Fatalf("iris stage: %v", err) + } + defer engine.DockerUnstage(ctx, container, stageDir) + + p, err := parse.New(ctx) + if err != nil { + t.Fatalf("parse.New: %v", err) + } + defer func() { _ = p.Close(ctx) }() + + // Host-orchestrated IRIS coverage (mcov.Run → %Monitor). + hostRes, err := mcov.Run(ctx, p, eng, []string{routinePath}, []string{suite}) + if err != nil { + t.Fatalf("host mcov.Run: %v", err) + } + hostBF := mcov.ByFile(hostRes) + + // Resident coverage: cov^STDHARN → ##MON block → mcov.FromMonitor. + frame, err := harness.TriggerCoverage(ctx, eng, []string{suite}, []string{covRoutine}) + if err != nil { + t.Fatalf("TriggerCoverage: %v", err) + } + _, _, mon, meta, err := harness.SplitFrame(frame) + if err != nil { + t.Fatalf("SplitFrame: %v\nframe:\n%s", err, frame) + } + if meta.Engine != "iris" { + t.Errorf("frame engine=%q, want iris", meta.Engine) + } + if mon == "" { + t.Fatalf("resident ##MON block is empty; frame:\n%s", frame) + } + resRes, err := mcov.FromMonitor(p, mon, []string{routinePath}) + if err != nil { + t.Fatalf("FromMonitor: %v", err) + } + resBF := mcov.ByFile(resRes) + + // The test must be meaningful (some lines, some covered). + if hostRes.Total() == 0 || hostRes.Covered() == 0 { + t.Fatalf("host coverage trivial: %d/%d", hostRes.Covered(), hostRes.Total()) + } + + // G4 coverage: resident ByFile == host ByFile. + if len(hostBF) != len(resBF) { + t.Fatalf("ByFile len: host %d, resident %d", len(hostBF), len(resBF)) + } + for i := range hostBF { + if hostBF[i].Path != resBF[i].Path || hostBF[i].Covered != resBF[i].Covered || hostBF[i].Total != resBF[i].Total { + t.Errorf("coverage parity MISMATCH:\n host: %+v\n resident: %+v", hostBF[i], resBF[i]) + } + } + t.Logf("coverage parity OK: %s %d/%d lines (host == resident)", covRoutine, hostRes.Covered(), hostRes.Total()) +} diff --git a/internal/harness/harness.go b/internal/harness/harness.go index 5394a8c..cd608e5 100644 --- a/internal/harness/harness.go +++ b/internal/harness/harness.go @@ -22,6 +22,8 @@ const ( hdrSuite = "##SUITE" // ##SUITE ^NAME hdrEnd = "##END" // ##END ^NAME exit=N (closes a suite) hdrLCOV = "##LCOV" // ##LCOV … verbatim LCOV tracefile + hdrMon = "##MON" // ##MON … raw IRIS line-monitor counts (MLINE:…) + hdrEndMon = "##END-MON" // closes the ##MON block hdrTrailer = "##END-HARNESS" // trailer: suites=/pass=/fail= cross-check ) @@ -57,19 +59,22 @@ type FrameMeta struct { } // SplitFrame splits the result frame (§3.2) into per-suite ^STDASSERT blocks, -// the LCOV block (empty when coverage was not requested), and the provenance / -// summary metadata. It is delimiter-scanning only — no test or coverage parsing -// happens here. Unrecognized ## directives are skipped (forward-compat). A -// missing header is ErrNoFrame; a missing/mismatched trailer is ErrTruncated, -// returned alongside whatever was parsed so a caller can still render it. -func SplitFrame(frame string) ([]SuiteBlock, string, FrameMeta, error) { +// the LCOV block, the raw ##MON line-monitor block (both empty when coverage was +// not requested), and the provenance / summary metadata. It is delimiter- +// scanning only — no test or coverage parsing happens here. Unrecognized ## +// directives are skipped (forward-compat). A missing header is ErrNoFrame; a +// missing/mismatched trailer is ErrTruncated, returned alongside whatever was +// parsed so a caller can still render it. +func SplitFrame(frame string) ([]SuiteBlock, string, string, FrameMeta, error) { var ( meta FrameMeta suites []SuiteBlock lcov strings.Builder + mon strings.Builder sawHeader bool sawTrailer bool inLCOV bool + inMon bool cur *SuiteBlock body strings.Builder ) @@ -90,15 +95,20 @@ func SplitFrame(frame string) ([]SuiteBlock, string, FrameMeta, error) { parseHeader(line, &meta) case strings.HasPrefix(line, hdrTrailer): flushSuite() - inLCOV = false + inLCOV, inMon = false, false sawTrailer = true parseTrailer(line, &meta) case strings.HasPrefix(line, hdrSuite): flushSuite() - inLCOV = false + inLCOV, inMon = false, false name := strings.TrimSpace(strings.TrimPrefix(line, hdrSuite)) name = strings.TrimPrefix(name, "^") cur = &SuiteBlock{Name: name, Exit: -1} + case line == hdrMon || strings.HasPrefix(line, hdrMon+" "): + flushSuite() + inLCOV, inMon = false, true + case strings.HasPrefix(line, hdrEndMon): + inMon = false case strings.HasPrefix(line, hdrEnd) && !strings.HasPrefix(line, hdrTrailer): if cur != nil { cur.Exit = parseExit(line) @@ -109,9 +119,12 @@ func SplitFrame(frame string) ([]SuiteBlock, string, FrameMeta, error) { } case line == hdrLCOV || strings.HasPrefix(line, hdrLCOV+" "): flushSuite() - inLCOV = true + inLCOV, inMon = true, false case strings.HasPrefix(line, "##"): // Unknown directive — skip, never fatal (forward-compat). + case inMon: + mon.WriteString(line) + mon.WriteByte('\n') case inLCOV: lcov.WriteString(line) lcov.WriteByte('\n') @@ -123,12 +136,12 @@ func SplitFrame(frame string) ([]SuiteBlock, string, FrameMeta, error) { flushSuite() if !sawHeader { - return suites, lcov.String(), meta, ErrNoFrame + return suites, lcov.String(), mon.String(), meta, ErrNoFrame } if !sawTrailer || meta.Suites != len(suites) { - return suites, lcov.String(), meta, ErrTruncated + return suites, lcov.String(), mon.String(), meta, ErrTruncated } - return suites, lcov.String(), meta, nil + return suites, lcov.String(), mon.String(), meta, nil } // parseHeader reads `##M-HARNESS frame=1 tier=integration engine=iris ns=VEHU`. diff --git a/internal/harness/harness_test.go b/internal/harness/harness_test.go index 42d5b22..73c47ca 100644 --- a/internal/harness/harness_test.go +++ b/internal/harness/harness_test.go @@ -21,7 +21,7 @@ func goldenFrame(t *testing.T) string { } func TestSplitFrameMeta(t *testing.T) { - suites, lcov, meta, err := harness.SplitFrame(goldenFrame(t)) + suites, lcov, _, meta, err := harness.SplitFrame(goldenFrame(t)) if err != nil { t.Fatalf("SplitFrame: %v", err) } @@ -43,7 +43,7 @@ func TestSplitFrameMeta(t *testing.T) { // ^STDASSERT text that the UNCHANGED mtest.ParseOutput consumes — no new parse // logic in the splitter. func TestSplitFrameSuiteBlocksRoundTripThroughMtest(t *testing.T) { - suites, _, _, err := harness.SplitFrame(goldenFrame(t)) + suites, _, _, _, err := harness.SplitFrame(goldenFrame(t)) if err != nil { t.Fatalf("SplitFrame: %v", err) } @@ -73,7 +73,7 @@ func TestSplitFrameSuiteBlocksRoundTripThroughMtest(t *testing.T) { // The ##LCOV block is verbatim LCOV the UNCHANGED mcov consumers understand. func TestSplitFrameLCOVRoundTripsThroughMcov(t *testing.T) { - _, lcov, _, err := harness.SplitFrame(goldenFrame(t)) + _, lcov, _, _, err := harness.SplitFrame(goldenFrame(t)) if err != nil { t.Fatalf("SplitFrame: %v", err) } @@ -91,7 +91,7 @@ func TestSplitFrameTruncatedStreamDetected(t *testing.T) { full := goldenFrame(t) // Drop the ##END-HARNESS trailer — a dropped connection. cut := full[:strings.Index(full, "##END-HARNESS")] - suites, _, _, err := harness.SplitFrame(cut) + suites, _, _, _, err := harness.SplitFrame(cut) if !errors.Is(err, harness.ErrTruncated) { t.Fatalf("err = %v, want ErrTruncated", err) } @@ -102,7 +102,7 @@ func TestSplitFrameTruncatedStreamDetected(t *testing.T) { } func TestSplitFrameMissingHeader(t *testing.T) { - _, _, _, err := harness.SplitFrame("##SUITE ^X\nAll tests passed.\n##END ^X exit=0\n") + _, _, _, _, err := harness.SplitFrame("##SUITE ^X\nAll tests passed.\n##END ^X exit=0\n") if !errors.Is(err, harness.ErrNoFrame) { t.Fatalf("err = %v, want ErrNoFrame", err) } @@ -115,7 +115,7 @@ func TestSplitFrameUnknownDirectiveSkipped(t *testing.T) { "##FUTURE something\n" + "##SUITE ^XTST\nAll tests passed.\nResults: 1 tests 1 passed 0 failed\n##END ^XTST exit=0\n" + "##END-HARNESS suites=1 pass=1 fail=0\n" - suites, _, meta, err := harness.SplitFrame(frame) + suites, _, _, meta, err := harness.SplitFrame(frame) if err != nil { t.Fatalf("SplitFrame: %v", err) } @@ -126,3 +126,24 @@ func TestSplitFrameUnknownDirectiveSkipped(t *testing.T) { t.Fatalf("suites = %+v, want one XTST", suites) } } + +// The raw ##MON line-monitor block (resident IRIS coverage) is captured verbatim +// for the host to join via mcov.FromMonitor — and it never collides with the +// suite/LCOV blocks (##END-MON is not a suite ##END). +func TestSplitFrameMonBlock(t *testing.T) { + frame := "##M-HARNESS frame=1 tier=integration engine=iris ns=USER\n" + + "##SUITE ^XTST\nAll tests passed.\nResults: 1 tests 1 passed 0 failed\n##END ^XTST exit=0\n" + + "##MON\nMLINE:STDMATH:41:14\nMLINE:STDMATH:42:0\n##END-MON\n" + + "##END-HARNESS suites=1 pass=1 fail=0\n" + suites, _, mon, _, err := harness.SplitFrame(frame) + if err != nil { + t.Fatalf("SplitFrame: %v", err) + } + if len(suites) != 1 || suites[0].Name != "XTST" || suites[0].Exit != 0 { + t.Fatalf("suites = %+v, want one XTST exit=0 (##END-MON must not close it)", suites) + } + want := "MLINE:STDMATH:41:14\nMLINE:STDMATH:42:0\n" + if mon != want { + t.Errorf("mon = %q, want %q", mon, want) + } +} diff --git a/internal/harness/parity_live_test.go b/internal/harness/parity_live_test.go index 5f0d96a..6817e4f 100644 --- a/internal/harness/parity_live_test.go +++ b/internal/harness/parity_live_test.go @@ -141,7 +141,7 @@ func residentParity(t *testing.T, kind engine.Kind, container, namespace, stdlib if err != nil { t.Fatalf("Trigger: %v", err) } - blocks, _, meta, err := harness.SplitFrame(frame) + blocks, _, _, meta, err := harness.SplitFrame(frame) if err != nil { t.Fatalf("SplitFrame: %v\nframe:\n%s", err, frame) } diff --git a/internal/harness/trigger.go b/internal/harness/trigger.go index 7eee64f..0d36ced 100644 --- a/internal/harness/trigger.go +++ b/internal/harness/trigger.go @@ -28,3 +28,17 @@ func Trigger(ctx context.Context, eng engine.Engine, scope []string) (string, er } return res.Stdout, nil } + +// TriggerCoverage invokes cov^STDHARN, which runs the scope under the IRIS line +// monitor over the named routines and adds a raw ##MON block to the frame (the +// host joins it via mcov.FromMonitor). On YDB the ##MON block is empty by design +// (YDB coverage stays the host-side view "TRACE" path); resident coverage is the +// IRIS tier. +func TriggerCoverage(ctx context.Context, eng engine.Engine, scope, routines []string) (string, error) { + script := "do cov^STDHARN(\"" + strings.Join(scope, " ") + "\",\"" + strings.Join(routines, " ") + "\")\nhalt\n" + res, err := eng.RunScript(ctx, script) + if err != nil { + return "", err + } + return res.Stdout, nil +} diff --git a/internal/mcov/frommonitor_test.go b/internal/mcov/frommonitor_test.go new file mode 100644 index 0000000..d6ab04a --- /dev/null +++ b/internal/mcov/frommonitor_test.go @@ -0,0 +1,30 @@ +package mcov + +import "testing" + +// FromMonitor joins a raw ##MON block (MLINE:routine:line:count) to the parse- +// tree executable lines — the host-side half of resident IRIS coverage. It must +// produce the same ByFile rollup the host-orchestrated runIris would from the +// same monitor data (the G4-by-construction guarantee). +func TestFromMonitor(t *testing.T) { + // MATH.m executable lines: 3 (add) and 5 (sub). Monitor: 3 ran, 5 did not. + mon := "MLINE:MATH:3:5\nMLINE:MATH:5:0\n" + r, err := FromMonitor(mustParser(t), mon, []string{"testdata/MATH.m"}) + if err != nil { + t.Fatalf("FromMonitor: %v", err) + } + if r.Total() != 2 || r.Covered() != 1 { + t.Fatalf("coverage = %d/%d, want 1/2", r.Covered(), r.Total()) + } + bf := ByFile(r) + if len(bf) != 1 || bf[0].Path != "testdata/MATH.m" || bf[0].Covered != 1 || bf[0].Total != 2 { + t.Errorf("ByFile = %+v, want MATH.m 1/2", bf) + } + // The executable-line denominator comes from the parse tree, not the monitor: + // a comment/label line the monitor never reports is still absent from the set. + for _, l := range r.Lines { + if l.Line != 3 && l.Line != 5 { + t.Errorf("unexpected covered line %d (only exec lines 3,5 belong)", l.Line) + } + } +} diff --git a/internal/mcov/iris.go b/internal/mcov/iris.go index 84dbbef..b28fb5a 100644 --- a/internal/mcov/iris.go +++ b/internal/mcov/iris.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/vista-cloud-dev/m-cli/internal/engine" + "github.com/vista-cloud-dev/m-parse/parse" ) // IRIS line coverage uses the %Monitor.System.LineByLine monitor (^%MONLBL), @@ -73,7 +74,15 @@ func runIris(ctx context.Context, eng engine.Engine, execs []ExecLine, routineNa if err != nil { return Result{}, err } - hits := parseMon(res.Stdout) + return joinMon(execs, res.Stdout), nil +} + +// joinMon joins raw per-line monitor counts (MLINE:routine:line:count) to the +// executable lines — the parse-tree denominator. Shared by the host-orchestrated +// runIris and the resident FromMonitor, so both produce the same Result from the +// same monitor data. +func joinMon(execs []ExecLine, monStdout string) Result { + hits := parseMon(monStdout) lines := make([]LineCov, 0, len(execs)) for _, ex := range execs { lines = append(lines, LineCov{ @@ -81,5 +90,19 @@ func runIris(ctx context.Context, eng engine.Engine, execs []ExecLine, routineNa Hits: hits[monKey{strings.ToUpper(ex.Routine), ex.Line}], }) } - return Result{Lines: lines, Stdout: res.Stdout}, nil + return Result{Lines: lines, Stdout: monStdout} +} + +// FromMonitor builds a coverage Result from a raw ##MON block (the MLINE lines +// the resident harness's IRIS line monitor emits, design §3.2) joined to the +// executable lines of routinePaths. The executable-line denominator stays +// host-side (the parse tree) and the monitor data is identical to what the +// host-orchestrated runIris collects, so resident coverage == file-side coverage +// by construction (G4). The host owns the parse tree the resident M side lacks. +func FromMonitor(p *parse.Parser, monText string, routinePaths []string) (Result, error) { + execs, err := DiscoverExecutables(p, routinePaths) + if err != nil { + return Result{}, err + } + return joinMon(execs, monText), nil } From 24a9feec255f51fb14f288aed575aa766bcaa76a Mon Sep 17 00:00:00 2001 From: Rafael Richards Date: Sat, 30 May 2026 23:33:00 -0400 Subject: [PATCH 5/5] harness: two-tier reconciliation + m test --resident (P3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves design Q5 (tier marker) and implements the §9.1-Q6 reconciliation. - mtest: TestSuite.Tier + DetectTier read a `;; tier: integration` directive at discovery (untagged ⇒ pure-logic, the safe default that keeps a suite the file-side PR gate). Inline or own-line comment, case-insensitive. - harness.Reconcile: one verdict by provenance — file-side authoritative for pure-logic, resident for integration; resident wins a same-suite conflict (reality); OK = UNION (any failure on either tier ⇒ non-zero exit). - harness.RunResident: trigger → SplitFrame → []mtest.RunResult (OK = summary.OK && ##END exit==0, same rule as mtest.RunSuite); truncated frame ⇒ error. - m test --resident: partitions discovered suites by tier, runs pure-logic file-side + integration via the resident harness, reconciles, reports each suite tagged by tier, exits on the union. Verified live on vista-iris: a pure-logic suite runs file-side and an integration-tagged suite runs resident in one reconciled report; a failing integration suite drives exit 3 (union). Host-side unit tests cover DetectTier, Reconcile (union + conflict), and RunResident (mapping + truncation). Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/harness/reconcile.go | 56 +++++++++++++++++++++ internal/harness/reconcile_test.go | 64 +++++++++++++++++++++++ internal/harness/run.go | 44 ++++++++++++++++ internal/harness/run_test.go | 65 ++++++++++++++++++++++++ internal/mtest/discovery.go | 32 +++++++++++- internal/mtest/mtest_test.go | 16 ++++++ main.go | 81 ++++++++++++++++++++++++++---- 7 files changed, 346 insertions(+), 12 deletions(-) create mode 100644 internal/harness/reconcile.go create mode 100644 internal/harness/reconcile_test.go create mode 100644 internal/harness/run.go create mode 100644 internal/harness/run_test.go diff --git a/internal/harness/reconcile.go b/internal/harness/reconcile.go new file mode 100644 index 0000000..95c95ac --- /dev/null +++ b/internal/harness/reconcile.go @@ -0,0 +1,56 @@ +package harness + +import "github.com/vista-cloud-dev/m-cli/internal/mtest" + +// Source names which tier produced a result. +const ( + SourceFileSide = "file-side" + SourceResident = "resident" +) + +// Provenanced is one suite's result tagged with the tier it is authoritative for +// and which tier actually produced it. +type Provenanced struct { + Result mtest.RunResult + Tier string // mtest.TierPureLogic | mtest.TierIntegration + Source string // SourceFileSide | SourceResident +} + +// Merged is the reconciled two-tier verdict (design §3.4, spec §9.1-Q6). +type Merged struct { + Results []Provenanced // one per suite, file-side first then resident-only, in order + OK bool // union: false if ANY suite failed on either tier +} + +// Reconcile produces one verdict by provenance. The file-side tier is +// authoritative for pure-logic suites (the deterministic PR gate); the resident +// IRIS tier is authoritative for integration suites (the live DD + data). The +// host runs each suite on exactly one tier, so the two result sets are normally +// disjoint; on conflict for the same suite the integration (resident) verdict +// wins — reality beats the file-side approximation. The OK is the UNION: any +// failure on either tier ⇒ not OK, so `m test`'s exit is non-zero (spec §3.3). +func Reconcile(fileSide, resident []mtest.RunResult) Merged { + var out []Provenanced + idx := map[string]int{} + for _, r := range fileSide { + idx[r.Suite] = len(out) + out = append(out, Provenanced{Result: r, Tier: mtest.TierPureLogic, Source: SourceFileSide}) + } + for _, r := range resident { + p := Provenanced{Result: r, Tier: mtest.TierIntegration, Source: SourceResident} + if i, ok := idx[r.Suite]; ok { + out[i] = p // resident (integration) wins the conflict + continue + } + idx[r.Suite] = len(out) + out = append(out, p) + } + merged := Merged{Results: out, OK: true} + for _, p := range out { + if !p.Result.OK { + merged.OK = false + break + } + } + return merged +} diff --git a/internal/harness/reconcile_test.go b/internal/harness/reconcile_test.go new file mode 100644 index 0000000..e1c6495 --- /dev/null +++ b/internal/harness/reconcile_test.go @@ -0,0 +1,64 @@ +package harness_test + +import ( + "testing" + + "github.com/vista-cloud-dev/m-cli/internal/harness" + "github.com/vista-cloud-dev/m-cli/internal/mtest" +) + +func ok(suite string) mtest.RunResult { + return mtest.RunResult{Suite: suite, OK: true, Summary: mtest.Summary{Total: 1, Passed: 1, OK: true}} +} +func fail(suite string) mtest.RunResult { + return mtest.RunResult{Suite: suite, OK: false, Summary: mtest.Summary{Total: 1, Failed: 1, OK: false}} +} + +// Reconcile is the §9.1-Q6 rule: file-side is authoritative for pure-logic +// suites, resident for integration suites; the verdict is the UNION (any failure +// on either tier ⇒ not OK), and each suite is tagged by provenance. +func TestReconcileUnionAndProvenance(t *testing.T) { + fileSide := []mtest.RunResult{ok("MATHTST"), ok("STRTST")} + resident := []mtest.RunResult{fail("DGINTTST")} + + m := harness.Reconcile(fileSide, resident) + if m.OK { + t.Error("OK = true, want false (DGINTTST failed on the resident tier — union)") + } + if len(m.Results) != 3 { + t.Fatalf("got %d results, want 3", len(m.Results)) + } + byName := map[string]harness.Provenanced{} + for _, r := range m.Results { + byName[r.Result.Suite] = r + } + if p := byName["MATHTST"]; p.Tier != mtest.TierPureLogic || p.Source != harness.SourceFileSide { + t.Errorf("MATHTST = %+v, want pure-logic/file-side", p) + } + if p := byName["DGINTTST"]; p.Tier != mtest.TierIntegration || p.Source != harness.SourceResident { + t.Errorf("DGINTTST = %+v, want integration/resident", p) + } +} + +func TestReconcileAllPass(t *testing.T) { + m := harness.Reconcile([]mtest.RunResult{ok("A")}, []mtest.RunResult{ok("B")}) + if !m.OK { + t.Errorf("OK = false, want true (both tiers passed)") + } +} + +// On conflict for the SAME suite, the integration (resident) verdict wins — +// reality from the live DD beats the file-side approximation. +func TestReconcileConflictResidentWins(t *testing.T) { + m := harness.Reconcile([]mtest.RunResult{ok("X")}, []mtest.RunResult{fail("X")}) + if len(m.Results) != 1 { + t.Fatalf("got %d results, want 1 (deduped)", len(m.Results)) + } + r := m.Results[0] + if r.Source != harness.SourceResident || r.Tier != mtest.TierIntegration { + t.Errorf("X = %+v, want resident/integration (server wins conflict)", r) + } + if m.OK { + t.Error("OK = true, want false (resident verdict for X is a failure)") + } +} diff --git a/internal/harness/run.go b/internal/harness/run.go new file mode 100644 index 0000000..538b532 --- /dev/null +++ b/internal/harness/run.go @@ -0,0 +1,44 @@ +package harness + +import ( + "context" + + "github.com/vista-cloud-dev/m-cli/internal/engine" + "github.com/vista-cloud-dev/m-cli/internal/mtest" +) + +// RunResident triggers the resident orchestrator for the given suites and maps +// the framed result back into the SAME mtest.RunResult shape the file-side runner +// produces — so the two tiers reconcile (Reconcile) and render through one path. +// A suite is ok only when its parsed summary is ok AND its ##END exit is 0 (a +// mid-suite crash reads as a fail), identical to mtest.RunSuite. A structurally +// bad frame (no header, truncated stream) is an error. +func RunResident(ctx context.Context, eng engine.Engine, suites []mtest.TestSuite) ([]mtest.RunResult, error) { + if len(suites) == 0 { + return nil, nil + } + names := make([]string, len(suites)) + for i, s := range suites { + names[i] = s.Name + } + frame, err := Trigger(ctx, eng, names) + if err != nil { + return nil, err + } + blocks, _, _, _, err := SplitFrame(frame) + if err != nil { + return nil, err + } + out := make([]mtest.RunResult, 0, len(blocks)) + for _, b := range blocks { + s := mtest.ParseOutput(b.Body) + out = append(out, mtest.RunResult{ + Suite: b.Name, + Summary: s, + OK: s.OK && b.Exit == 0, + ExitCode: b.Exit, + Stdout: b.Body, + }) + } + return out, nil +} diff --git a/internal/harness/run_test.go b/internal/harness/run_test.go new file mode 100644 index 0000000..c2a45ee --- /dev/null +++ b/internal/harness/run_test.go @@ -0,0 +1,65 @@ +package harness_test + +import ( + "context" + "testing" + + "github.com/vista-cloud-dev/m-cli/internal/engine" + "github.com/vista-cloud-dev/m-cli/internal/harness" + "github.com/vista-cloud-dev/m-cli/internal/mtest" +) + +// scriptEngine is a fake engine whose RunScript returns a canned frame. +type scriptEngine struct{ frame string } + +func (scriptEngine) Kind() engine.Kind { return engine.YDB } +func (scriptEngine) EnsureLoaded(context.Context, string) error { return nil } +func (scriptEngine) RunRoutine(context.Context, string, ...string) (engine.Result, error) { + return engine.Result{}, nil +} +func (scriptEngine) RunXCmd(context.Context, string) (engine.Result, error) { + return engine.Result{}, nil +} +func (e scriptEngine) RunScript(context.Context, string) (engine.Result, error) { + return engine.Result{Stdout: e.frame}, nil +} + +func TestRunResidentMapsFrameToRunResults(t *testing.T) { + frame := "##M-HARNESS frame=1 tier=integration engine=iris ns=VEHU\n" + + "##SUITE ^AINTTST\nResults: 2 tests 2 passed 0 failed\nAll tests passed.\n##END ^AINTTST exit=0\n" + + "##SUITE ^BINTTST\n FAIL x\nResults: 1 tests 0 passed 1 failed\n1 test(s) FAILED.\n##END ^BINTTST exit=0\n" + + "##END-HARNESS suites=2 pass=2 fail=1\n" + eng := scriptEngine{frame: frame} + suites := []mtest.TestSuite{{Name: "AINTTST"}, {Name: "BINTTST"}} + + got, err := harness.RunResident(context.Background(), eng, suites) + if err != nil { + t.Fatalf("RunResident: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d results, want 2", len(got)) + } + if !got[0].OK || got[0].Summary.Passed != 2 { + t.Errorf("AINTTST = %+v, want ok 2 passed", got[0]) + } + if got[1].OK || got[1].Summary.Failed != 1 { + t.Errorf("BINTTST = %+v, want not-ok 1 failed", got[1]) + } +} + +func TestRunResidentTruncatedFrameErrors(t *testing.T) { + // No trailer ⇒ truncated; RunResident surfaces it rather than reporting a + // partial pass. + eng := scriptEngine{frame: "##M-HARNESS frame=1 tier=integration engine=iris ns=X\n" + + "##SUITE ^AINTTST\nAll tests passed.\n##END ^AINTTST exit=0\n"} + if _, err := harness.RunResident(context.Background(), eng, []mtest.TestSuite{{Name: "AINTTST"}}); err == nil { + t.Error("want an error for a truncated frame, got nil") + } +} + +func TestRunResidentEmpty(t *testing.T) { + got, err := harness.RunResident(context.Background(), scriptEngine{}, nil) + if err != nil || got != nil { + t.Errorf("RunResident(nil) = %v, %v; want nil, nil", got, err) + } +} diff --git a/internal/mtest/discovery.go b/internal/mtest/discovery.go index 88e7f43..848f67a 100644 --- a/internal/mtest/discovery.go +++ b/internal/mtest/discovery.go @@ -20,15 +20,45 @@ type TestCase struct { Line int } +// Tier classifies where a suite is authoritative (spec §9.1-Q6, design §3.4). +const ( + // TierPureLogic suites are deterministic, with no live DD/data dependency. + // They run file-side (host-orchestrated) and are the PR gate. The safe + // default for an untagged suite. + TierPureLogic = "pure-logic" + // TierIntegration suites exist because of the live FileMan DD + populated + // globals and cannot be reproduced file-side. They run resident (in the dev + // IRIS, next to the data). + TierIntegration = "integration" +) + // TestSuite is one *TST.m file with its cases. type TestSuite struct { Name string Path string Protocol string // routine hosting start/report (STDASSERT, TESTRUN, …) + Tier string // TierPureLogic (default) | TierIntegration — see DetectTier Cases []TestCase Deps []string // external routines the suite calls (upper, sorted) — for affected-test selection } +// reTier matches a suite-level `; tier: ` directive inside any M comment +// (one or more `;`, case-insensitive), so a suite can opt into the integration +// tier on its own line or inline on the first label line. +var reTier = regexp.MustCompile(`(?i);[;\s]*tier\s*:\s*(\S+)`) + +// DetectTier reads the suite's tier directive. An untagged suite is +// TierPureLogic — the safe default that keeps it the file-side PR gate; only an +// explicit `;; tier: integration` moves it to the resident tier. +func DetectTier(src []byte) string { + if m := reTier.FindSubmatch(src); m != nil { + if strings.EqualFold(string(m[1]), TierIntegration) { + return TierIntegration + } + } + return TierPureLogic +} + var ( reSuiteName = regexp.MustCompile(`^[A-Z][A-Z0-9]*TST$`) reTestLabel = regexp.MustCompile(`^t[A-Z][A-Za-z0-9]*$`) @@ -164,7 +194,7 @@ func Discover(p *parse.Parser, paths []string) ([]TestSuite, error) { if err != nil { return nil, err } - suites = append(suites, TestSuite{Name: name, Path: f, Protocol: DetectProtocol(src), Cases: cases, Deps: deps}) + suites = append(suites, TestSuite{Name: name, Path: f, Protocol: DetectProtocol(src), Tier: DetectTier(src), Cases: cases, Deps: deps}) } sort.Slice(suites, func(i, j int) bool { return suites[i].Name < suites[j].Name }) return suites, nil diff --git a/internal/mtest/mtest_test.go b/internal/mtest/mtest_test.go index a0aae26..754ed60 100644 --- a/internal/mtest/mtest_test.go +++ b/internal/mtest/mtest_test.go @@ -42,6 +42,22 @@ func TestDetectProtocol(t *testing.T) { } } +func TestDetectTier(t *testing.T) { + cases := map[string]string{ + "FOOTST\t; suite\n\t;; tier: integration\n": mtest.TierIntegration, + "FOOTST\t; suite\n\t; tier: integration\n": mtest.TierIntegration, + "FOOTST\t;; tier:integration\n": mtest.TierIntegration, + "FOOTST\t;; tier: pure-logic\n": mtest.TierPureLogic, + "FOOTST\t; an ordinary suite, no directive\n": mtest.TierPureLogic, // untagged ⇒ safe default + "FOOTST\t;; TIER: Integration\n": mtest.TierIntegration, // case-insensitive + } + for src, want := range cases { + if got := mtest.DetectTier([]byte(src)); got != want { + t.Errorf("DetectTier(%q) = %q, want %q", src, got, want) + } + } +} + func TestFindCases(t *testing.T) { src, err := os.ReadFile("testdata/SAMPLETST.m") if err != nil { diff --git a/main.go b/main.go index cf12899..6f0ca76 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,7 @@ import ( "github.com/vista-cloud-dev/m-cli/internal/config" "github.com/vista-cloud-dev/m-cli/internal/dispatch" "github.com/vista-cloud-dev/m-cli/internal/engine" + "github.com/vista-cloud-dev/m-cli/internal/harness" "github.com/vista-cloud-dev/m-cli/internal/lint" "github.com/vista-cloud-dev/m-cli/internal/lsp" "github.com/vista-cloud-dev/m-cli/internal/mcov" @@ -412,10 +413,12 @@ type testCmd struct { Docker string `help:"Run inside this running container via docker exec (e.g. m-test-engine, vista-iris)."` Routines []string `help:"Extra source dirs to stage (e.g. m-stdlib/src for ^STDASSERT). Repeatable."` Namespace string `help:"IRIS namespace (default USER)."` + Resident bool `help:"Run ';; tier: integration' suites via the resident harness (RUN^STDHARN) and reconcile with file-side pure-logic suites (spec §9)."` } type suiteResult struct { Suite string `json:"suite"` + Tier string `json:"tier,omitempty"` Passed int `json:"passed"` Failed int `json:"failed"` Total int `json:"total"` @@ -494,21 +497,22 @@ func (c *testCmd) Run(cc *clikit.Context) error { } else { eng = engine.New(kind, engine.Options{Namespace: c.Namespace}) } - results, runErr := mtest.Run(ctx, eng, suites) - if runErr != nil { - return clikit.Fail(clikit.ExitRuntime, "ENGINE_RUN", runErr.Error(), - "m test runs on a live engine — is ydb/iris installed and reachable?") + var rows []suiteResult + if c.Resident { + rows, err = runTwoTier(ctx, eng, suites) + } else { + rows, err = runOneTier(ctx, eng, suites) } - for _, r := range results { - report.Passed += r.Summary.Passed - report.Failed += r.Summary.Failed + if err != nil { + return err + } + for _, r := range rows { + report.Passed += r.Passed + report.Failed += r.Failed if !r.OK { failedSuites++ } - report.Results = append(report.Results, suiteResult{ - Suite: r.Suite, Passed: r.Summary.Passed, Failed: r.Summary.Failed, - Total: r.Summary.Total, OK: r.OK, - }) + report.Results = append(report.Results, r) } } @@ -541,6 +545,61 @@ func (c *testCmd) Run(cc *clikit.Context) error { return nil } +// runOneTier is the default host-orchestrated path: every suite runs file-side. +func runOneTier(ctx context.Context, eng engine.Engine, suites []mtest.TestSuite) ([]suiteResult, error) { + results, err := mtest.Run(ctx, eng, suites) + if err != nil { + return nil, clikit.Fail(clikit.ExitRuntime, "ENGINE_RUN", err.Error(), + "m test runs on a live engine — is ydb/iris installed and reachable?") + } + tier := map[string]string{} + for _, s := range suites { + tier[s.Name] = s.Tier + } + rows := make([]suiteResult, 0, len(results)) + for _, r := range results { + rows = append(rows, toRow(r, tier[r.Suite])) + } + return rows, nil +} + +// runTwoTier (--resident) runs pure-logic suites file-side and integration +// suites via the resident harness, then reconciles by provenance (spec §9.1-Q6): +// one verdict per suite, exit = union. +func runTwoTier(ctx context.Context, eng engine.Engine, suites []mtest.TestSuite) ([]suiteResult, error) { + var pureLogic, integration []mtest.TestSuite + for _, s := range suites { + if s.Tier == mtest.TierIntegration { + integration = append(integration, s) + } else { + pureLogic = append(pureLogic, s) + } + } + fileResults, err := mtest.Run(ctx, eng, pureLogic) + if err != nil { + return nil, clikit.Fail(clikit.ExitRuntime, "ENGINE_RUN", err.Error(), + "m test runs on a live engine — is ydb/iris installed and reachable?") + } + resResults, err := harness.RunResident(ctx, eng, integration) + if err != nil { + return nil, clikit.Fail(clikit.ExitRuntime, "RESIDENT_RUN", err.Error(), + "the resident harness needs RUN^STDHARN staged — add --routines ") + } + merged := harness.Reconcile(fileResults, resResults) + rows := make([]suiteResult, 0, len(merged.Results)) + for _, p := range merged.Results { + rows = append(rows, toRow(p.Result, p.Tier)) + } + return rows, nil +} + +func toRow(r mtest.RunResult, tier string) suiteResult { + return suiteResult{ + Suite: r.Suite, Tier: tier, + Passed: r.Summary.Passed, Failed: r.Summary.Failed, Total: r.Summary.Total, OK: r.OK, + } +} + // --- coverage ---------------------------------------------------------------- type coverageCmd struct {