Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 0 additions & 22 deletions .tbd/workspaces/outbox/issues/is-01khx9xwpv03xfg2cc5aqtwn8g.md

This file was deleted.

21 changes: 0 additions & 21 deletions .tbd/workspaces/outbox/issues/is-01khx9y5kgsweghjsdnt8wze22.md

This file was deleted.

19 changes: 0 additions & 19 deletions .tbd/workspaces/outbox/issues/is-01khx9yd02anf0w14rkt37mcej.md

This file was deleted.

19 changes: 0 additions & 19 deletions .tbd/workspaces/outbox/issues/is-01khx9ym9e1d2dyb3nh3dqs7qe.md

This file was deleted.

19 changes: 0 additions & 19 deletions .tbd/workspaces/outbox/issues/is-01khx9yvwgs8af6m7q1yp8jb6s.md

This file was deleted.

17 changes: 0 additions & 17 deletions .tbd/workspaces/outbox/issues/is-01khx9z3qgwx2s3gfnwbsnv45j.md

This file was deleted.

15 changes: 0 additions & 15 deletions .tbd/workspaces/outbox/issues/is-01khy5qjtzdbp2189eh4x1xkea.md

This file was deleted.

7 changes: 0 additions & 7 deletions .tbd/workspaces/outbox/mappings/ids.yml

This file was deleted.

5 changes: 3 additions & 2 deletions docs/markform-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -450,8 +450,9 @@ A row with at least one non-empty cell is retained.
| `min` / `max` | `number`, `year`, `date` | Value bounds |
| `integer` | `number` | Must be integer |

**Row sparseness warning:** When a non-empty row has the majority of its cells empty
(strictly more than half), a validation warning is emitted.
**Row sparseness warning:** When a non-empty row has more than half of its cells empty
(e.g., 1 of 3 filled, or 1 of 4 filled; even splits like 2 of 4 don’t trigger this), a
validation warning is emitted.
This helps catch cases where an agent produced sparse or incomplete data.

**Sentinel values in cells:** Use `%SKIP%` or `%ABORT%` with optional reasons:
Expand Down
5 changes: 3 additions & 2 deletions docs/markform-spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -493,8 +493,9 @@ columnTypes=[{type: "string", required: true}, "number", "url"]
| `max` | `number`, `year`, `date` | Maximum value |
| `integer` | `number` | Value must be an integer |

**Row sparseness warning:** When a non-empty row has the majority of its cells empty
(strictly more than half), a validation warning is emitted.
**Row sparseness warning:** When a non-empty row has more than half of its cells empty
(e.g., 1 of 3 filled, or 1 of 4 filled; even splits like 2 of 4 don’t trigger this), a
validation warning is emitted.
This helps catch cases where an agent produced sparse or incomplete data.

**Example with per-column constraints:**
Expand Down
32 changes: 22 additions & 10 deletions packages/markform/src/engine/table/parseTable.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,23 +137,35 @@ export function parseCellValue(rawValue: string, columnType: ColumnTypeName): Ce
}

// =============================================================================
// Empty Row Detection
// Empty Cell Detection
// =============================================================================

/**
* Check if a single cell is empty.
* Used by both row normalization and validation to ensure consistent logic.
*
* - Skipped cells are empty
* - Aborted cells are NOT empty (they carry intentional signal)
* - Answered cells are empty only if value is undefined/null/empty string
* - Unknown states are treated conservatively as NOT empty
*/
export function isCellEmpty(cell: CellResponse | undefined): boolean {
if (!cell || cell.state === 'skipped') return true;
if (cell.state === 'aborted') return false;
// Only 'answered' cells can be empty based on value check
if (cell.state === 'answered') {
return cell.value === undefined || cell.value === null || cell.value === '';
}
// Unknown state - treat conservatively as NOT empty
return false;
}

/**
* Check if a table row is fully empty (all cells skipped/empty).
* Used during normalization to drop rows that carry no data.
*
* Aborted cells are NOT considered empty — they carry intentional signal
* (the agent explicitly declined to fill them, possibly with a reason).
*/
export function isRowFullyEmpty(row: TableRowResponse): boolean {
return Object.values(row).every((cell) => {
if (!cell || cell.state === 'skipped') return true;
if (cell.state === 'aborted') return false;
// 'answered' cell with no meaningful value
return cell.value === undefined || cell.value === null || cell.value === '';
});
return Object.values(row).every(isCellEmpty);
}

// =============================================================================
Expand Down
11 changes: 2 additions & 9 deletions packages/markform/src/engine/validate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import type {
YearField,
YearValue,
} from './coreTypes.js';
import { isCellEmpty } from './table/parseTable.js';

// =============================================================================
// Validation Options and Results
Expand Down Expand Up @@ -975,15 +976,7 @@ function validateTableRow(

// Check for mostly-empty rows (more than half of cells empty in a non-empty row)
const totalCells = columns.length;
const filledCells = columns.filter((col) => {
const cell = row[col.id];
return (
cell?.state === 'answered' &&
cell.value !== undefined &&
cell.value !== null &&
cell.value !== ''
);
}).length;
const filledCells = columns.filter((col) => !isCellEmpty(row[col.id])).length;

if (filledCells > 0 && filledCells < totalCells && totalCells - filledCells > totalCells / 2) {
issues.push({
Expand Down
47 changes: 47 additions & 0 deletions packages/markform/tests/unit/engine/parseTable.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
extractColumnsFromTable,
parseInlineTable,
isRowFullyEmpty,
isCellEmpty,
} from '../../../src/engine/table/parseTable.js';
import type {
TableColumn,
Expand All @@ -19,6 +20,35 @@ import type {
} from '../../../src/engine/coreTypes.js';

describe('parseTable', () => {
describe('isCellEmpty', () => {
it('returns true for undefined cell', () => {
expect(isCellEmpty(undefined)).toBe(true);
});

it('returns true for skipped cell', () => {
expect(isCellEmpty({ state: 'skipped' })).toBe(true);
});

it('returns false for aborted cell', () => {
expect(isCellEmpty({ state: 'aborted' })).toBe(false);
});

it('returns true for answered cell with empty value', () => {
expect(isCellEmpty({ state: 'answered', value: '' })).toBe(true);
expect(isCellEmpty({ state: 'answered', value: undefined })).toBe(true);
expect(isCellEmpty({ state: 'answered', value: null as any })).toBe(true);
});

it('returns false for answered cell with meaningful value', () => {
expect(isCellEmpty({ state: 'answered', value: 'test' })).toBe(false);
expect(isCellEmpty({ state: 'answered', value: 0 })).toBe(false);
});

it('returns false for unknown state (conservative)', () => {
expect(isCellEmpty({ state: 'unknown' as any })).toBe(false);
});
});

describe('parseCellValue', () => {
// [rawValue, columnType, expectedState, expectedValue]
type CellCase = [
Expand Down Expand Up @@ -462,6 +492,23 @@ describe('parseTable', () => {
false,
);
});

it('treats malformed cell with unknown state conservatively (not empty)', () => {
// Simulate a malformed/corrupted cell with an unexpected state
// This should be treated as NOT empty (conservative approach)
const malformedRow = {
name: { state: 'unknown' as any, value: 'test' },
};
expect(isRowFullyEmpty(malformedRow)).toBe(false);
});

it('treats malformed cell with unknown state and no value conservatively (not empty)', () => {
// Even with no value, unknown state should be treated conservatively
const malformedRow = {
name: { state: 'unknown' as any, value: undefined },
};
expect(isRowFullyEmpty(malformedRow)).toBe(false);
});
});

describe('empty row filtering in parseMarkdownTable', () => {
Expand Down
Loading