diff --git a/runtime/parser/parse_node.go b/runtime/parser/parse_node.go index 4e950135a35..9685bb4ef4f 100644 --- a/runtime/parser/parse_node.go +++ b/runtime/parser/parse_node.go @@ -248,26 +248,26 @@ func (p *Parser) parseStem(paths []string, ymlPath, yml, sqlPath, sql string) (* v, ok := v.(string) if !ok { err = fmt.Errorf("invalid type %T for property 'type'", v) - break - } - res.Kind, err = ParseResourceKind(v) - if err != nil { - break + } else { + res.Kind, err = ParseResourceKind(v) } case "name": v, ok := v.(string) if !ok { err = fmt.Errorf("invalid type %T for property 'name'", v) - break + } else { + res.Name = v } - res.Name = v case "connector": v, ok := v.(string) if !ok { err = fmt.Errorf("invalid type %T for property 'connector'", v) - break + } else { + res.Connector = v } - res.Connector = v + } + if err != nil { + break } } if err != nil { diff --git a/runtime/parser/parser.go b/runtime/parser/parser.go index f5f25d4fbd7..51bc9b799d0 100644 --- a/runtime/parser/parser.go +++ b/runtime/parser/parser.go @@ -634,12 +634,13 @@ func (p *Parser) parsePaths(ctx context.Context, paths []string) error { // NOTE 2: Using a map since the two-way check (necessary for reparses) may match the same resource twice. modelsWithNameErrs := make(map[ResourceName]string) for _, r := range p.insertedResources { - if r.Name.Kind == ResourceKindSource { + switch r.Name.Kind { + case ResourceKindSource: n := ResourceName{Kind: ResourceKindModel, Name: r.Name.Name}.Normalized() if _, ok := p.Resources[n]; ok { modelsWithNameErrs[n] = r.Name.Name } - } else if r.Name.Kind == ResourceKindModel { + case ResourceKindModel: n := ResourceName{Kind: ResourceKindSource, Name: r.Name.Name}.Normalized() if r2, ok := p.Resources[n]; ok { modelsWithNameErrs[r.Name.Normalized()] = r2.Name.Name diff --git a/web-admin/src/features/projects/status/ActionsCell.svelte b/web-admin/src/features/projects/status/ActionsCell.svelte index 401987577f6..6d66cbc607d 100644 --- a/web-admin/src/features/projects/status/ActionsCell.svelte +++ b/web-admin/src/features/projects/status/ActionsCell.svelte @@ -8,10 +8,12 @@ export let resourceKind: string; export let resourceName: string; export let canRefresh: boolean; + export let isIncremental: boolean = false; + export let hasErroredPartitions: boolean = false; export let onClickRefreshDialog: ( resourceName: string, resourceKind: string, - refreshType: "full" | "incremental", + refreshType: "full" | "incremental" | "errored-partitions", ) => void; export let isDropdownOpen: boolean; export let onDropdownOpenChange: (isOpen: boolean) => void; @@ -37,17 +39,36 @@ Full Refresh - { - onClickRefreshDialog(resourceName, resourceKind, "incremental"); - }} - > -
- - Incremental Refresh -
- + {#if isIncremental} + { + onClickRefreshDialog(resourceName, resourceKind, "incremental"); + }} + > +
+ + Incremental Refresh +
+ + {/if} + {#if hasErroredPartitions} + { + onClickRefreshDialog( + resourceName, + resourceKind, + "errored-partitions", + ); + }} + > +
+ + Refresh Errored Partitions +
+ + {/if} {:else} diff --git a/web-admin/src/features/projects/status/ProjectResourcesTable.svelte b/web-admin/src/features/projects/status/ProjectResourcesTable.svelte index 4b2517a74a2..77d70790949 100644 --- a/web-admin/src/features/projects/status/ProjectResourcesTable.svelte +++ b/web-admin/src/features/projects/status/ProjectResourcesTable.svelte @@ -23,7 +23,7 @@ let isConfirmDialogOpen = false; let dialogResourceName = ""; let dialogResourceKind = ""; - let dialogRefreshType: "full" | "incremental" = "full"; + let dialogRefreshType: "full" | "incremental" | "errored-partitions" = "full"; let openDropdownResourceKey = ""; @@ -33,7 +33,7 @@ const openRefreshDialog = ( resourceName: string, resourceKind: string, - refreshType: "full" | "incremental", + refreshType: "full" | "incremental" | "errored-partitions", ) => { dialogResourceName = resourceName; dialogResourceKind = resourceKind; @@ -62,6 +62,7 @@ { model: dialogResourceName, full: dialogRefreshType === "full", + allErroredPartitions: dialogRefreshType === "errored-partitions", }, ], }, @@ -85,7 +86,7 @@ closeRefreshDialog(); }; - // Create columns definition as a constant to prevent unnecessary re-creation + // Create columns definition as a constant - key block handles re-renders const columns: ColumnDef[] = [ { accessorKey: "title", @@ -165,12 +166,23 @@ status === V1ReconcileStatus.RECONCILE_STATUS_RUNNING; if (!isRowReconciling) { const resourceKey = `${row.original.meta.name.kind}:${row.original.meta.name.name}`; + // Check if model is incremental and has errored partitions + const isModel = row.original.meta.name.kind === ResourceKind.Model; + const modelSpec = row.original.model?.spec; + const modelState = row.original.model?.state; + const isIncremental = isModel && !!modelSpec?.incremental; + const hasErroredPartitions = + isModel && + !!modelState?.partitionsModelId && + !!modelState?.partitionsHaveErrors; return flexRender(ActionsCell, { resourceKind: row.original.meta.name.kind, resourceName: row.original.meta.name.name, canRefresh: row.original.meta.name.kind === ResourceKind.Model || row.original.meta.name.kind === ResourceKind.Source, + isIncremental, + hasErroredPartitions, onClickRefreshDialog: openRefreshDialog, isDropdownOpen: isDropdownOpen(resourceKey), onDropdownOpenChange: (isOpen: boolean) => @@ -193,7 +205,7 @@ void; - export let refreshType: "full" | "incremental" = "full"; + export let refreshType: "full" | "incremental" | "errored-partitions" = + "full"; + + function getDialogTitle(type: typeof refreshType): string { + switch (type) { + case "full": + return "Full Refresh"; + case "incremental": + return "Incremental Refresh"; + case "errored-partitions": + return "Refresh Errored Partitions"; + } + } function handleRefresh() { try { @@ -28,7 +40,7 @@ - {refreshType === "full" ? "Full Refresh" : "Incremental Refresh"} + {getDialogTitle(refreshType)} {name}? @@ -38,6 +50,9 @@ This operation can take a significant amount of time and will update all dependent resources. Only proceed if you're certain this is necessary. + {:else if refreshType === "errored-partitions"} + This will re-run all partitions that failed during their last + execution. Successfully completed partitions will not be affected. {:else} Refreshing this resource will update all dependent resources. {/if} diff --git a/web-admin/src/features/projects/status/ResourceErrorMessage.svelte b/web-admin/src/features/projects/status/ResourceErrorMessage.svelte index 9bddfc5904a..c9e9d60ef5e 100644 --- a/web-admin/src/features/projects/status/ResourceErrorMessage.svelte +++ b/web-admin/src/features/projects/status/ResourceErrorMessage.svelte @@ -7,18 +7,44 @@ import TooltipContent from "@rilldata/web-common/components/tooltip/TooltipContent.svelte"; import { copyToClipboard } from "@rilldata/web-common/lib/actions/copy-to-clipboard"; import { V1ReconcileStatus } from "@rilldata/web-common/runtime-client"; + import { AlertTriangleIcon } from "lucide-svelte"; export let message: string; export let status: V1ReconcileStatus; + export let testErrors: string[] = []; + + $: hasTestErrors = testErrors.length > 0; + $: testErrorMessage = testErrors.join("\n");
{#if status === V1ReconcileStatus.RECONCILE_STATUS_PENDING || status === V1ReconcileStatus.RECONCILE_STATUS_RUNNING} + {:else if hasTestErrors} + + + +

{testErrorMessage}

+ + {:else if message} + + + + diff --git a/web-admin/src/features/projects/status/model-overview/utils.spec.ts b/web-admin/src/features/projects/status/model-overview/utils.spec.ts new file mode 100644 index 00000000000..2e0f96ea735 --- /dev/null +++ b/web-admin/src/features/projects/status/model-overview/utils.spec.ts @@ -0,0 +1,291 @@ +import { describe, expect, it } from "vitest"; +import { + filterTemporaryTables, + parseSizeForSorting, + compareSizesDescending, + formatLogTime, + getLogLevelClass, + getLogLevelLabel, + formatModelSize, + isModelPartitioned, + isModelIncremental, + hasModelErroredPartitions, + shouldFilterByErrored, + shouldFilterByPending, +} from "./utils"; +import type { + V1OlapTableInfo, + V1Resource, +} from "@rilldata/web-common/runtime-client"; + +describe("model-overview utils", () => { + describe("filterTemporaryTables", () => { + it("filters out __rill_tmp_ prefixed tables", () => { + const tables: V1OlapTableInfo[] = [ + { name: "users" }, + { name: "__rill_tmp_123" }, + { name: "orders" }, + { name: "__rill_tmp_abc" }, + ]; + + const result = filterTemporaryTables(tables); + + expect(result).toEqual([{ name: "users" }, { name: "orders" }]); + }); + + it("filters out tables with empty names", () => { + const tables: V1OlapTableInfo[] = [ + { name: "users" }, + { name: "" }, + { name: undefined }, + { name: "orders" }, + ]; + + const result = filterTemporaryTables(tables); + + expect(result).toEqual([{ name: "users" }, { name: "orders" }]); + }); + + it("returns empty array for undefined input", () => { + expect(filterTemporaryTables(undefined)).toEqual([]); + }); + + it("returns empty array for empty input", () => { + expect(filterTemporaryTables([])).toEqual([]); + }); + + it("keeps all tables when none are temporary", () => { + const tables: V1OlapTableInfo[] = [ + { name: "users" }, + { name: "orders" }, + { name: "products" }, + ]; + + const result = filterTemporaryTables(tables); + + expect(result).toEqual(tables); + }); + }); + + describe("parseSizeForSorting", () => { + it("returns -1 for undefined", () => { + expect(parseSizeForSorting(undefined)).toBe(-1); + }); + + it("returns -1 for empty string", () => { + expect(parseSizeForSorting("")).toBe(-1); + }); + + it("returns -1 for '-1' string", () => { + expect(parseSizeForSorting("-1")).toBe(-1); + }); + + it("returns number as-is", () => { + expect(parseSizeForSorting(1024)).toBe(1024); + expect(parseSizeForSorting(0)).toBe(-1); // 0 is falsy + }); + + it("parses string to number", () => { + expect(parseSizeForSorting("1024")).toBe(1024); + expect(parseSizeForSorting("999999")).toBe(999999); + }); + }); + + describe("compareSizesDescending", () => { + it("sorts larger sizes first (descending)", () => { + expect(compareSizesDescending(100, 200)).toBeGreaterThan(0); + expect(compareSizesDescending(200, 100)).toBeLessThan(0); + expect(compareSizesDescending(100, 100)).toBe(0); + }); + + it("handles string sizes", () => { + expect(compareSizesDescending("100", "200")).toBeGreaterThan(0); + expect(compareSizesDescending("200", "100")).toBeLessThan(0); + }); + + it("handles mixed string and number", () => { + expect(compareSizesDescending(100, "200")).toBeGreaterThan(0); + expect(compareSizesDescending("200", 100)).toBeLessThan(0); + }); + + it("puts undefined/invalid values last", () => { + expect(compareSizesDescending(100, undefined)).toBeLessThan(0); + expect(compareSizesDescending(undefined, 100)).toBeGreaterThan(0); + expect(compareSizesDescending(undefined, undefined)).toBe(0); + }); + }); + + describe("formatLogTime", () => { + it("returns empty string for undefined", () => { + expect(formatLogTime(undefined)).toBe(""); + }); + + it("returns empty string for empty string", () => { + expect(formatLogTime("")).toBe(""); + }); + + it("formats ISO timestamp to locale time", () => { + // Use a fixed timestamp to test formatting + const result = formatLogTime("2024-01-15T14:30:45.000Z"); + // Result will vary by locale, but should contain time components + expect(result).toMatch(/\d{1,2}:\d{2}:\d{2}/); + }); + }); + + describe("getLogLevelClass", () => { + const testCases: [string | undefined, string][] = [ + ["LOG_LEVEL_ERROR", "text-red-600"], + ["LOG_LEVEL_FATAL", "text-red-600"], + ["LOG_LEVEL_WARN", "text-yellow-600"], + ["LOG_LEVEL_INFO", "text-fg-muted"], + ["LOG_LEVEL_DEBUG", "text-fg-muted"], + [undefined, "text-fg-muted"], + ["UNKNOWN", "text-fg-muted"], + ]; + + for (const [level, expectedClass] of testCases) { + it(`getLogLevelClass(${JSON.stringify(level)}) = ${expectedClass}`, () => { + expect(getLogLevelClass(level)).toBe(expectedClass); + }); + } + }); + + describe("getLogLevelLabel", () => { + const testCases: [string | undefined, string][] = [ + ["LOG_LEVEL_ERROR", "ERROR"], + ["LOG_LEVEL_FATAL", "FATAL"], + ["LOG_LEVEL_WARN", "WARN"], + ["LOG_LEVEL_INFO", "INFO"], + ["LOG_LEVEL_DEBUG", "DEBUG"], + [undefined, "INFO"], + ["UNKNOWN", "INFO"], + ]; + + for (const [level, expectedLabel] of testCases) { + it(`getLogLevelLabel(${JSON.stringify(level)}) = ${expectedLabel}`, () => { + expect(getLogLevelLabel(level)).toBe(expectedLabel); + }); + } + }); + + describe("formatModelSize", () => { + it("returns '-' for undefined", () => { + expect(formatModelSize(undefined)).toBe("-"); + }); + + it("returns '-' for null", () => { + expect(formatModelSize(null as unknown as undefined)).toBe("-"); + }); + + it("returns '-' for '-1' string", () => { + expect(formatModelSize("-1")).toBe("-"); + }); + + it("returns '-' for negative numbers", () => { + expect(formatModelSize(-100)).toBe("-"); + }); + + it("returns '-' for NaN", () => { + expect(formatModelSize("not a number")).toBe("-"); + }); + + it("formats valid byte sizes", () => { + expect(formatModelSize(0)).toBe("0"); + expect(formatModelSize(1024)).toBe("1.0KB"); + expect(formatModelSize("1048576")).toBe("1.0MB"); + }); + }); + + describe("isModelPartitioned", () => { + it("returns false for undefined resource", () => { + expect(isModelPartitioned(undefined)).toBe(false); + }); + + it("returns false when no partitionsResolver", () => { + const resource: V1Resource = { + model: { spec: {} }, + }; + expect(isModelPartitioned(resource)).toBe(false); + }); + + it("returns true when partitionsResolver exists", () => { + const resource: V1Resource = { + model: { spec: { partitionsResolver: "some-resolver" } }, + }; + expect(isModelPartitioned(resource)).toBe(true); + }); + }); + + describe("isModelIncremental", () => { + it("returns false for undefined resource", () => { + expect(isModelIncremental(undefined)).toBe(false); + }); + + it("returns false when incremental is false", () => { + const resource: V1Resource = { + model: { spec: { incremental: false } }, + }; + expect(isModelIncremental(resource)).toBe(false); + }); + + it("returns true when incremental is true", () => { + const resource: V1Resource = { + model: { spec: { incremental: true } }, + }; + expect(isModelIncremental(resource)).toBe(true); + }); + }); + + describe("hasModelErroredPartitions", () => { + it("returns false for undefined resource", () => { + expect(hasModelErroredPartitions(undefined)).toBe(false); + }); + + it("returns false when no partitionsModelId", () => { + const resource: V1Resource = { + model: { state: { partitionsHaveErrors: true } }, + }; + expect(hasModelErroredPartitions(resource)).toBe(false); + }); + + it("returns false when partitionsHaveErrors is false", () => { + const resource: V1Resource = { + model: { + state: { partitionsModelId: "123", partitionsHaveErrors: false }, + }, + }; + expect(hasModelErroredPartitions(resource)).toBe(false); + }); + + it("returns true when both conditions are met", () => { + const resource: V1Resource = { + model: { + state: { partitionsModelId: "123", partitionsHaveErrors: true }, + }, + }; + expect(hasModelErroredPartitions(resource)).toBe(true); + }); + }); + + describe("shouldFilterByErrored", () => { + it("returns true for 'errors' filter", () => { + expect(shouldFilterByErrored("errors")).toBe(true); + }); + + it("returns false for other filters", () => { + expect(shouldFilterByErrored("all")).toBe(false); + expect(shouldFilterByErrored("pending")).toBe(false); + }); + }); + + describe("shouldFilterByPending", () => { + it("returns true for 'pending' filter", () => { + expect(shouldFilterByPending("pending")).toBe(true); + }); + + it("returns false for other filters", () => { + expect(shouldFilterByPending("all")).toBe(false); + expect(shouldFilterByPending("errors")).toBe(false); + }); + }); +}); diff --git a/web-admin/src/features/projects/status/model-overview/utils.ts b/web-admin/src/features/projects/status/model-overview/utils.ts new file mode 100644 index 00000000000..e53fd5e6701 --- /dev/null +++ b/web-admin/src/features/projects/status/model-overview/utils.ts @@ -0,0 +1,162 @@ +import type { + V1OlapTableInfo, + V1Resource, +} from "@rilldata/web-common/runtime-client"; +import { formatMemorySize } from "@rilldata/web-common/lib/number-formatting/memory-size"; + +/** + * Filters out temporary tables (e.g., __rill_tmp_ prefixed tables) + */ +export function filterTemporaryTables( + tables: V1OlapTableInfo[] | undefined, +): V1OlapTableInfo[] { + return ( + tables?.filter( + (t): t is V1OlapTableInfo => + !!t.name && !t.name.startsWith("__rill_tmp_"), + ) ?? [] + ); +} + +/** + * Parses a size value (string or number) to a number for sorting. + * Returns -1 for invalid/missing values. + */ +export function parseSizeForSorting(size: string | number | undefined): number { + if (!size || size === "-1") { + return -1; + } + return typeof size === "number" ? size : parseInt(size, 10); +} + +/** + * Compares two size values for descending sort order. + * Used for sorting tables by database size. + */ +export function compareSizesDescending( + sizeA: string | number | undefined, + sizeB: string | number | undefined, +): number { + const numA = parseSizeForSorting(sizeA); + const numB = parseSizeForSorting(sizeB); + return numB - numA; +} + +/** + * Formats a timestamp string to locale time (HH:MM:SS). + */ +export function formatLogTime(time: string | undefined): string { + if (!time) return ""; + const date = new Date(time); + return date.toLocaleTimeString(undefined, { + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); +} + +/** + * Returns the CSS class for a log level. + */ +export function getLogLevelClass(level: string | undefined): string { + switch (level) { + case "LOG_LEVEL_ERROR": + case "LOG_LEVEL_FATAL": + return "text-red-600"; + case "LOG_LEVEL_WARN": + return "text-yellow-600"; + default: + return "text-fg-muted"; + } +} + +/** + * Returns the display label for a log level. + */ +export function getLogLevelLabel(level: string | undefined): string { + switch (level) { + case "LOG_LEVEL_ERROR": + return "ERROR"; + case "LOG_LEVEL_FATAL": + return "FATAL"; + case "LOG_LEVEL_WARN": + return "WARN"; + case "LOG_LEVEL_INFO": + return "INFO"; + case "LOG_LEVEL_DEBUG": + return "DEBUG"; + default: + return "INFO"; + } +} + +// ============================================ +// Model Size Utils +// ============================================ + +/** + * Formats a byte size for display. Returns "-" for invalid values. + */ +export function formatModelSize(bytes: string | number | undefined): string { + if (bytes === undefined || bytes === null || bytes === "-1") return "-"; + + let numBytes: number; + if (typeof bytes === "number") { + numBytes = bytes; + } else { + numBytes = parseInt(bytes, 10); + } + + if (isNaN(numBytes) || numBytes < 0) return "-"; + return formatMemorySize(numBytes); +} + +// ============================================ +// Model Actions Utils +// ============================================ + +/** + * Checks if a model resource is partitioned. + */ +export function isModelPartitioned(resource: V1Resource | undefined): boolean { + return !!resource?.model?.spec?.partitionsResolver; +} + +/** + * Checks if a model resource is incremental. + */ +export function isModelIncremental(resource: V1Resource | undefined): boolean { + return !!resource?.model?.spec?.incremental; +} + +/** + * Checks if a model resource has errored partitions. + */ +export function hasModelErroredPartitions( + resource: V1Resource | undefined, +): boolean { + return ( + !!resource?.model?.state?.partitionsModelId && + !!resource?.model?.state?.partitionsHaveErrors + ); +} + +// ============================================ +// Model Partitions Filter Utils +// ============================================ + +export type PartitionFilterType = "all" | "errors" | "pending"; + +/** + * Returns whether to filter by errored partitions based on filter selection. + */ +export function shouldFilterByErrored(filter: PartitionFilterType): boolean { + return filter === "errors"; +} + +/** + * Returns whether to filter by pending partitions based on filter selection. + */ +export function shouldFilterByPending(filter: PartitionFilterType): boolean { + return filter === "pending"; +} diff --git a/web-admin/src/features/projects/status/selectors.ts b/web-admin/src/features/projects/status/selectors.ts index cebe50a8f2f..569b1c3c602 100644 --- a/web-admin/src/features/projects/status/selectors.ts +++ b/web-admin/src/features/projects/status/selectors.ts @@ -4,11 +4,27 @@ import { } from "@rilldata/web-admin/client"; import { createRuntimeServiceListResources, + createConnectorServiceOLAPListTables, + createConnectorServiceOLAPGetTable, + createQueryServiceTableCardinality, type V1ListResourcesResponse, + type V1OlapTableInfo, + type V1Resource, } from "@rilldata/web-common/runtime-client"; import { ResourceKind } from "@rilldata/web-common/features/entity-management/resource-selectors"; +import { readable, type Readable } from "svelte/store"; import { smartRefetchIntervalFunc } from "@rilldata/web-admin/lib/refetch-interval-store"; +/** Type for the table metadata store result */ +export type TableMetadataResult = { + data: { + isView: Map; + columnCount: Map; + }; + isLoading: boolean; + isError: boolean; +}; + export function useProjectDeployment(orgName: string, projName: string) { return createAdminServiceGetProject( orgName, @@ -47,3 +63,238 @@ export function useResources(instanceId: string) { }, ); } + +export function useTablesList(instanceId: string, connector: string = "") { + return createConnectorServiceOLAPListTables( + { + instanceId, + connector, + }, + { + query: { + enabled: !!instanceId, + }, + }, + ); +} + +/** + * Fetches metadata (view status) for each table. + * + * Note: This creates a separate query per table (N+1 pattern). This is acceptable here because: + * 1. The OLAPGetTable API doesn't support batch requests + * 2. Tables are typically few in number on the status page + * 3. Queries are cached and run in parallel via svelte-query + * + * If performance becomes an issue with many tables, consider adding a batch API endpoint. + */ +export function useTableMetadata( + instanceId: string, + connector: string = "", + tables: V1OlapTableInfo[] | undefined, +): Readable { + // If no tables, return empty store immediately + if (!tables || tables.length === 0) { + return readable( + { + data: { + isView: new Map(), + columnCount: new Map(), + }, + isLoading: false, + isError: false, + }, + () => {}, + ); + } + + return readable( + { + data: { + isView: new Map(), + columnCount: new Map(), + }, + isLoading: true, + isError: false, + }, + (set) => { + const isView = new Map(); + const columnCount = new Map(); + const tableNames = (tables ?? []) + .map((t) => t.name) + .filter((n) => !!n) as string[]; + const subscriptions: Array<() => void> = []; + + let completedCount = 0; + const totalOperations = tableNames.length; + + // Helper to update and notify + const updateAndNotify = () => { + const isLoading = completedCount < totalOperations; + set({ + data: { isView, columnCount }, + isLoading, + isError: false, + }); + }; + + // Fetch view status for each table in parallel + for (const tableName of tableNames) { + const tableQuery = createConnectorServiceOLAPGetTable( + { + instanceId, + connector, + table: tableName, + }, + { + query: { + enabled: !!instanceId && !!tableName, + }, + }, + ); + + const unsubscribe = tableQuery.subscribe((result) => { + // Capture the view field from the response + if (result.data?.view !== undefined) { + isView.set(tableName, result.data.view); + } + // Capture the column count from the schema + if (result.data?.schema?.fields !== undefined) { + columnCount.set(tableName, result.data.schema.fields.length); + } + completedCount++; + updateAndNotify(); + }); + + subscriptions.push(unsubscribe); + } + + // Return cleanup function + return () => { + subscriptions.forEach((unsub) => unsub()); + }; + }, + ); +} + +/** Type for the table cardinality store result */ +export type TableCardinalityResult = { + data: { + rowCount: Map; + }; + isLoading: boolean; + isError: boolean; +}; + +/** + * Fetches row count (cardinality) for each table. + */ +export function useTableCardinality( + instanceId: string, + tables: V1OlapTableInfo[] | undefined, +): Readable { + // If no tables, return empty store immediately + if (!tables || tables.length === 0) { + return readable( + { + data: { + rowCount: new Map(), + }, + isLoading: false, + isError: false, + }, + () => {}, + ); + } + + return readable( + { + data: { + rowCount: new Map(), + }, + isLoading: true, + isError: false, + }, + (set) => { + const rowCount = new Map(); + const tableNames = (tables ?? []) + .map((t) => t.name) + .filter((n) => !!n) as string[]; + const subscriptions: Array<() => void> = []; + + let completedCount = 0; + const totalOperations = tableNames.length; + + // Helper to update and notify + const updateAndNotify = () => { + const isLoading = completedCount < totalOperations; + set({ + data: { rowCount }, + isLoading, + isError: false, + }); + }; + + // Fetch cardinality for each table in parallel + for (const tableName of tableNames) { + const cardinalityQuery = createQueryServiceTableCardinality( + instanceId, + tableName, + {}, + { + query: { + enabled: !!instanceId && !!tableName, + }, + }, + ); + + const unsubscribe = cardinalityQuery.subscribe((result) => { + if (result.data?.cardinality !== undefined) { + rowCount.set(tableName, parseInt(result.data.cardinality, 10) || 0); + } + completedCount++; + updateAndNotify(); + }); + + subscriptions.push(unsubscribe); + } + + // Return cleanup function + return () => { + subscriptions.forEach((unsub) => unsub()); + }; + }, + ); +} + +/** + * Fetches model resources and maps them by their result table name. + * This allows looking up model resource data by the OLAP table name. + */ +export function useModelResources(instanceId: string) { + return createRuntimeServiceListResources( + instanceId, + { kind: ResourceKind.Model }, + { + query: { + select: (data: V1ListResourcesResponse) => { + const map = new Map(); + data.resources?.forEach((resource) => { + // Index by resultTable (the actual output table name) + const tableName = resource.model?.state?.resultTable; + if (tableName) { + map.set(tableName.toLowerCase(), resource); + } + // Also index by model name as fallback + const modelName = resource.meta?.name?.name; + if (modelName) { + map.set(modelName.toLowerCase(), resource); + } + }); + return map; + }, + enabled: !!instanceId, + }, + }, + ); +} diff --git a/web-admin/src/routes/[organization]/[project]/-/status/+layout.svelte b/web-admin/src/routes/[organization]/[project]/-/status/+layout.svelte new file mode 100644 index 00000000000..fcecd72c182 --- /dev/null +++ b/web-admin/src/routes/[organization]/[project]/-/status/+layout.svelte @@ -0,0 +1,42 @@ + + + + + +
+ + +
+ + + diff --git a/web-admin/src/routes/[organization]/[project]/-/status/+page.ts b/web-admin/src/routes/[organization]/[project]/-/status/+page.ts new file mode 100644 index 00000000000..4d875360bc9 --- /dev/null +++ b/web-admin/src/routes/[organization]/[project]/-/status/+page.ts @@ -0,0 +1,8 @@ +import { redirect } from "@sveltejs/kit"; + +export const load = ({ params }) => { + throw redirect( + 307, + `/${params.organization}/${params.project}/-/status/project-status`, + ); +}; diff --git a/web-admin/src/routes/[organization]/[project]/-/status/model-overview/+page.svelte b/web-admin/src/routes/[organization]/[project]/-/status/model-overview/+page.svelte new file mode 100644 index 00000000000..6dc119f768a --- /dev/null +++ b/web-admin/src/routes/[organization]/[project]/-/status/model-overview/+page.svelte @@ -0,0 +1,123 @@ + + +
+
+

Model Overview

+
+
+
+ Tables (Materialized Models) + + + + + +
+ + {$tableMetadata?.isLoading ? "-" : tableCount} + +
+
+ Views + + {$tableMetadata?.isLoading ? "-" : viewCount} + +
+
+
+ OLAP Engine + + + + + +
+ {#if $instanceQuery.isLoading} + - + {:else if olapConnector} + + {formatConnectorName(olapConnector.type)} + +
+ + {olapConnector.provision ? "Rill-Managed" : "Self-Managed"} + +
+ {:else} + - + {/if} +
+
+ + +
+
diff --git a/web-admin/src/routes/[organization]/[project]/-/status/+page.svelte b/web-admin/src/routes/[organization]/[project]/-/status/project-status/+page.svelte similarity index 57% rename from web-admin/src/routes/[organization]/[project]/-/status/+page.svelte rename to web-admin/src/routes/[organization]/[project]/-/status/project-status/+page.svelte index ba2ab04ec94..4ba3986f36c 100644 --- a/web-admin/src/routes/[organization]/[project]/-/status/+page.svelte +++ b/web-admin/src/routes/[organization]/[project]/-/status/project-status/+page.svelte @@ -1,6 +1,5 @@ - -
-
- - -
- - - +
+
+ +
- + + + +
diff --git a/web-admin/tests/project-status-refresh.spec.ts b/web-admin/tests/project-status-refresh.spec.ts new file mode 100644 index 00000000000..e11b4f8ba6f --- /dev/null +++ b/web-admin/tests/project-status-refresh.spec.ts @@ -0,0 +1,253 @@ +import { expect } from "@playwright/test"; +import { test } from "./setup/base"; + +test.describe("Project Status - Resource Refresh (openrtb)", () => { + // Increase timeout for tests that interact with virtualized tables + test.setTimeout(60_000); + + test.beforeEach(async ({ adminPage }) => { + // Navigate to the project status page + await adminPage.goto("/e2e/openrtb/-/status"); + // Wait for the resources table to load with actual data + await expect(adminPage.getByText("Resources")).toBeVisible(); + // Wait for a specific model name to appear (indicates data is loaded) + // Note: VirtualizedTable uses div.row elements, not role="row" + await expect(adminPage.getByText("auction_data_model")).toBeVisible({ + timeout: 60_000, + }); + }); + + test("should show Full Refresh option for models", async ({ adminPage }) => { + // Find the auction_data_model row and click its actions menu + // The row contains both the model name and a "Model" badge + const modelRow = adminPage.locator(".row").filter({ + hasText: "auction_data_model", + }); + // Target the dropdown menu trigger specifically (rows may have multiple buttons) + await modelRow.locator("[data-melt-dropdown-menu-trigger]").click(); + + // Verify "Full Refresh" is visible + await expect( + adminPage.getByRole("menuitem", { name: "Full Refresh" }), + ).toBeVisible(); + }); + + test("should show Full Refresh option for sources", async ({ adminPage }) => { + // Wait for the source row to be visible before interacting + // Look for bids_data_raw source which should be in the openrtb test project + await expect(adminPage.getByText("bids_data_raw")).toBeVisible({ + timeout: 60_000, + }); + + // Find a source row and click its actions menu + const sourceRow = adminPage.locator(".row").filter({ + hasText: "bids_data_raw", + }); + // Target the dropdown menu trigger specifically (rows may have multiple buttons) + await sourceRow.locator("[data-melt-dropdown-menu-trigger]").click(); + + // Verify "Full Refresh" is visible + await expect( + adminPage.getByRole("menuitem", { name: "Full Refresh" }), + ).toBeVisible(); + + // Incremental Refresh should NOT be visible for sources + await expect( + adminPage.getByRole("menuitem", { name: "Incremental Refresh" }), + ).not.toBeVisible(); + }); + + test("should not show Incremental Refresh for non-incremental models", async ({ + adminPage, + }) => { + // Find the auction_data_model row and click its actions menu + const modelRow = adminPage.locator(".row").filter({ + hasText: "auction_data_model", + }); + // Target the dropdown menu trigger specifically (rows may have multiple buttons) + await modelRow.locator("[data-melt-dropdown-menu-trigger]").click(); + + // Verify "Full Refresh" is visible + await expect( + adminPage.getByRole("menuitem", { name: "Full Refresh" }), + ).toBeVisible(); + + // For non-incremental models, "Incremental Refresh" should not be visible + await expect( + adminPage.getByRole("menuitem", { name: "Incremental Refresh" }), + ).not.toBeVisible(); + }); + + test("should not show Refresh Errored Partitions for models without errors", async ({ + adminPage, + }) => { + // Find the auction_data_model row and click its actions menu + const modelRow = adminPage.locator(".row").filter({ + hasText: "auction_data_model", + }); + // Target the dropdown menu trigger specifically (rows may have multiple buttons) + await modelRow.locator("[data-melt-dropdown-menu-trigger]").click(); + + // "Refresh Errored Partitions" should not be visible for models without errored partitions + await expect( + adminPage.getByRole("menuitem", { name: "Refresh Errored Partitions" }), + ).not.toBeVisible(); + }); + + test("should show correct dialog for Full Refresh", async ({ adminPage }) => { + // Find the auction_data_model row and click its actions menu + const modelRow = adminPage.locator(".row").filter({ + hasText: "auction_data_model", + }); + // Target the dropdown menu trigger specifically (rows may have multiple buttons) + await modelRow.locator("[data-melt-dropdown-menu-trigger]").click(); + + // Click "Full Refresh" + await adminPage.getByRole("menuitem", { name: "Full Refresh" }).click(); + + // Verify the dialog shows "Full Refresh" in the title + await expect(adminPage.getByRole("alertdialog")).toBeVisible(); + await expect( + adminPage.getByRole("heading", { name: /Full Refresh/ }), + ).toBeVisible(); + + // Verify the warning message about full refresh + await expect( + adminPage.getByText(/Warning.*will re-ingest ALL data from scratch/), + ).toBeVisible(); + + // Close dialog by clicking cancel + await adminPage.getByRole("button", { name: "Cancel" }).click(); + await expect(adminPage.getByRole("alertdialog")).not.toBeVisible(); + }); +}); + +test.describe("Project Status - Incremental Model Refresh (incremental-test)", () => { + // Increase timeout for tests that interact with virtualized tables + test.setTimeout(60_000); + + test.beforeEach(async ({ adminPage }) => { + // Navigate to the incremental-test project status page + await adminPage.goto("/e2e/incremental-test/-/status"); + // Wait for the resources table to load with actual data + await expect(adminPage.getByText("Resources")).toBeVisible(); + // Wait for the specific model rows to appear (indicates data is loaded) + // Note: VirtualizedTable uses div.row elements, not role="row" + await expect(adminPage.getByText("success_partition")).toBeVisible({ + timeout: 60_000, + }); + }); + + test("should show Incremental Refresh option for incremental models", async ({ + adminPage, + }) => { + // Find the success_partition model row and click its actions menu + const modelRow = adminPage.locator(".row").filter({ + hasText: "success_partition", + }); + // Target the dropdown menu trigger specifically (rows may have multiple buttons) + await modelRow.locator("[data-melt-dropdown-menu-trigger]").click(); + + // Verify both "Full Refresh" and "Incremental Refresh" are visible + await expect( + adminPage.getByRole("menuitem", { name: "Full Refresh" }), + ).toBeVisible(); + await expect( + adminPage.getByRole("menuitem", { name: "Incremental Refresh" }), + ).toBeVisible(); + }); + + test("should show correct dialog for Incremental Refresh", async ({ + adminPage, + }) => { + // Find the success_partition model row and click its actions menu + const modelRow = adminPage.locator(".row").filter({ + hasText: "success_partition", + }); + // Target the dropdown menu trigger specifically (rows may have multiple buttons) + await modelRow.locator("[data-melt-dropdown-menu-trigger]").click(); + + // Click "Incremental Refresh" + await adminPage + .getByRole("menuitem", { name: "Incremental Refresh" }) + .click(); + + // Verify the dialog shows "Incremental Refresh" in the title + await expect(adminPage.getByRole("alertdialog")).toBeVisible(); + await expect( + adminPage.getByRole("heading", { name: /Incremental Refresh/ }), + ).toBeVisible(); + + // Verify the message about updating dependent resources + await expect( + adminPage.getByText("will update all dependent resources"), + ).toBeVisible(); + + // Close dialog by clicking cancel + await adminPage.getByRole("button", { name: "Cancel" }).click(); + await expect(adminPage.getByRole("alertdialog")).not.toBeVisible(); + }); + + test("should show Refresh Errored Partitions for models with errored partitions", async ({ + adminPage, + }) => { + // Find the failed_partition model row and click its actions menu + const modelRow = adminPage.locator(".row").filter({ + hasText: "failed_partition", + }); + // Target the dropdown menu trigger specifically (rows may have multiple buttons) + await modelRow.locator("[data-melt-dropdown-menu-trigger]").click(); + + // Verify "Refresh Errored Partitions" is visible for models with errors + await expect( + adminPage.getByRole("menuitem", { name: "Refresh Errored Partitions" }), + ).toBeVisible(); + }); + + test("should show correct dialog for Refresh Errored Partitions", async ({ + adminPage, + }) => { + // Find the failed_partition model row and click its actions menu + const modelRow = adminPage.locator(".row").filter({ + hasText: "failed_partition", + }); + // Target the dropdown menu trigger specifically (rows may have multiple buttons) + await modelRow.locator("[data-melt-dropdown-menu-trigger]").click(); + + // Click "Refresh Errored Partitions" + await adminPage + .getByRole("menuitem", { name: "Refresh Errored Partitions" }) + .click(); + + // Verify the dialog shows "Refresh Errored Partitions" in the title + await expect(adminPage.getByRole("alertdialog")).toBeVisible(); + await expect( + adminPage.getByRole("heading", { name: /Refresh Errored Partitions/ }), + ).toBeVisible(); + + // Verify the message about re-running failed partitions + await expect( + adminPage.getByText("re-run all partitions that failed"), + ).toBeVisible(); + + // Close dialog by clicking cancel + await adminPage.getByRole("button", { name: "Cancel" }).click(); + await expect(adminPage.getByRole("alertdialog")).not.toBeVisible(); + }); + + test("should not show Refresh Errored Partitions for successful incremental models", async ({ + adminPage, + }) => { + // Find the success_partition model row and click its actions menu + const modelRow = adminPage.locator(".row").filter({ + hasText: "success_partition", + }); + // Target the dropdown menu trigger specifically (rows may have multiple buttons) + await modelRow.locator("[data-melt-dropdown-menu-trigger]").click(); + + // "Refresh Errored Partitions" should NOT be visible for models without errors + await expect( + adminPage.getByRole("menuitem", { name: "Refresh Errored Partitions" }), + ).not.toBeVisible(); + }); +}); diff --git a/web-admin/tests/setup/setup.ts b/web-admin/tests/setup/setup.ts index ce2ddd7dd07..c4bd104e8ca 100644 --- a/web-admin/tests/setup/setup.ts +++ b/web-admin/tests/setup/setup.ts @@ -325,4 +325,48 @@ setup.describe("global setup", () => { adminPage.getByRole("link", { name: "Adbids dashboard" }), ).toBeVisible(); }); + + setup("should deploy the incremental-test project", async ({ adminPage }) => { + // Deploy the incremental-test project (for testing incremental model refresh) + const { match } = await spawnAndMatch( + "rill", + [ + "deploy", + "--path", + "../web-common/tests/projects/incremental-test", + "--project", + "incremental-test", + "--archive", + "--interactive=false", + ], + /https?:\/\/[^\s]+/, + ); + + // Navigate to the project URL and expect to see the successful deployment + const url = match[0]; + await adminPage.goto(url); + await expect( + adminPage.getByRole("link", { name: RILL_ORG_NAME }), + ).toBeVisible(); // Organization breadcrumb + await expect( + adminPage.getByRole("link", { name: "incremental-test", exact: true }), + ).toBeVisible(); // Project breadcrumb + + // Expect to land on the project home page + await adminPage.waitForURL(`/${RILL_ORG_NAME}/incremental-test`); + + // Wait for the project to be ready by checking the status page + await adminPage.goto(`/${RILL_ORG_NAME}/incremental-test/-/status`); + await expect(adminPage.getByText("Resources")).toBeVisible({ + timeout: 60_000, + }); + + // Verify the incremental models are listed + await expect(adminPage.getByText("success_partition")).toBeVisible({ + timeout: 30_000, + }); + await expect(adminPage.getByText("failed_partition")).toBeVisible({ + timeout: 30_000, + }); + }); }); diff --git a/web-common/src/components/table/VirtualizedTable.svelte b/web-common/src/components/table/VirtualizedTable.svelte index 28f72ddc6a1..a0201c1fcad 100644 --- a/web-common/src/components/table/VirtualizedTable.svelte +++ b/web-common/src/components/table/VirtualizedTable.svelte @@ -24,6 +24,7 @@ export let rowHeight = 46; export let containerHeight = 400; export let overscan = 1; + export let tableId: string | undefined = undefined; let containerElement: HTMLDivElement; let sorting: SortingState = []; @@ -108,6 +109,7 @@
diff --git a/web-common/src/features/models/partitions/PartitionsTable.svelte b/web-common/src/features/models/partitions/PartitionsTable.svelte index 5bd23df3cbe..bcd8b2f7629 100644 --- a/web-common/src/features/models/partitions/PartitionsTable.svelte +++ b/web-common/src/features/models/partitions/PartitionsTable.svelte @@ -135,6 +135,7 @@ flexRender(TriggerPartition, { partitionKey: (row as Row).original .key as string, + resource, }), }, ] diff --git a/web-common/src/features/models/partitions/TriggerPartition.svelte b/web-common/src/features/models/partitions/TriggerPartition.svelte index 51b13cf3a45..8acef2221a1 100644 --- a/web-common/src/features/models/partitions/TriggerPartition.svelte +++ b/web-common/src/features/models/partitions/TriggerPartition.svelte @@ -1,10 +1,13 @@