diff --git a/packages/cubejs-api-gateway/openspec.yml b/packages/cubejs-api-gateway/openspec.yml index 0a5c71ddddc04..5fe9925a30bac 100644 --- a/packages/cubejs-api-gateway/openspec.yml +++ b/packages/cubejs-api-gateway/openspec.yml @@ -4,6 +4,30 @@ info: version: "1.0.0" title: "Cube.js" paths: + "/v1/granularities": + get: + summary: "List the granularities available for this deployment" + description: "Returns the granularities enabled in this deployment — built-ins plus any custom granularities defined via `CUBEJS_GRANULARITIES` or `config.granularities`. Evaluated per request context." + operationId: "granularitiesV1" + responses: + "200": + description: "successful operation" + content: + application/json: + schema: + $ref: "#/components/schemas/V1GranularitiesResponse" + "4XX": + description: "Request could not be completed" + content: + application/json: + schema: + $ref: "#/components/schemas/V1Error" + "5XX": + description: "Internal Server Error" + content: + application/json: + schema: + $ref: "#/components/schemas/V1Error" "/v1/meta": get: summary: "Load Metadata" @@ -115,8 +139,17 @@ components: properties: name: type: "string" + type: + type: "string" + description: "Built-in (year/quarter/month/...) or user-defined custom granularity." + enum: + - built-in + - custom title: type: "string" + format: + type: "string" + description: "d3-time-format string used by clients to display bucketed timestamps." interval: type: "string" sql: @@ -125,6 +158,17 @@ components: type: "string" origin: type: "string" + V1GranularitiesResponse: + type: "object" + description: "Response shape of GET /v1/granularities." + properties: + data: + type: "object" + properties: + granularities: + type: array + items: + $ref: "#/components/schemas/V1CubeMetaDimensionGranularity" V1CubeMetaDimension: type: "object" required: diff --git a/packages/cubejs-api-gateway/package.json b/packages/cubejs-api-gateway/package.json index 2af6c980b391a..17ca99e78e558 100644 --- a/packages/cubejs-api-gateway/package.json +++ b/packages/cubejs-api-gateway/package.json @@ -29,6 +29,7 @@ "dependencies": { "@cubejs-backend/native": "1.6.49", "@cubejs-backend/query-orchestrator": "1.6.49", + "@cubejs-backend/schema-compiler": "1.6.49", "@cubejs-backend/shared": "1.6.49", "@ungap/structured-clone": "^0.3.4", "assert-never": "^1.4.0", diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 070342511a6f1..436fb8a51891a 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -32,6 +32,13 @@ import type { import { createProxyMiddleware } from 'http-proxy-middleware'; import { QueryBody } from '@cubejs-backend/query-orchestrator'; +import { + resolveGlobalGranularities, + resolveDimensionGranularities, + normalizeGranularitiesBlock, + buildBuiltInsCatalog, + BUILT_IN_GRANULARITIES, +} from '@cubejs-backend/schema-compiler'; import { QueryType, ApiScopes, @@ -154,6 +161,8 @@ class ApiGateway { protected readonly extendContext?: ExtendContextFn; + protected readonly granularitiesOption?: ApiGatewayOptions['granularities']; + protected readonly dataSourceStorage: any; public readonly checkAuthFn: PreparedCheckAuthFn; @@ -207,6 +216,7 @@ class ApiGateway { this.subscriptionStore = options.subscriptionStore || new LocalSubscriptionStore(); this.enforceSecurityChecks = options.enforceSecurityChecks || (process.env.NODE_ENV === 'production'); this.extendContext = options.extendContext; + this.granularitiesOption = options.granularities; this.checkAuthFn = this.createCheckAuthFn(options); this.checkAuthSystemFn = this.createCheckAuthSystemFn(); @@ -453,6 +463,17 @@ class ApiGateway { }) ); + app.get( + `${this.basePath}/v1/granularities`, + userMiddlewares, + userAsyncHandler(async (req, res) => { + await this.granularities({ + context: req.context, + res: this.resToResultFn(res), + }); + }) + ); + app.post( `${this.basePath}/v1/cubesql`, userMiddlewares, @@ -668,7 +689,9 @@ class ApiGateway { const cubesConfig = onlyViews ? metaConfig.cubes.filter((c: any) => c.config?.type === 'view') : metaConfig.cubes; - const cubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config); + const filteredCubes = this.filterVisibleItemsInMeta(context, cubesConfig).map(cube => cube.config); + // Apply after the visibility filter so we only enrich what the client will actually receive. + const cubes = await this.applyGlobalGranularitiesToMetaCubes(context, filteredCubes); const visibleCubeNames = new Set(cubes.map(c => c.name)); const viewGroups = (metaConfig.viewGroups || []) .map(group => ({ @@ -695,6 +718,47 @@ class ApiGateway { } } + public async granularities({ context, res }: { + context: RequestContext, + res: ResponseResultFn, + }) { + const requestStarted = new Date(); + try { + await this.assertApiScope('meta', context.securityContext); + const globalConfig = await this.resolveGlobalGranularitiesForRequest(context); + const builtInsCatalog = buildBuiltInsCatalog(globalConfig); + + const granularities: any[] = []; + for (const [name, entry] of Object.entries(builtInsCatalog)) { + granularities.push({ type: 'built-in', name, ...entry }); + } + for (const [name, def] of Object.entries(globalConfig.customGranularities)) { + // Skip names already emitted by `buildBuiltInsCatalog` (their inline overrides are folded in there). + if (!(name in BUILT_IN_GRANULARITIES)) { + const entry: any = { + type: 'custom', + name, + title: def.title || name, + }; + if (def.interval !== undefined) entry.interval = def.interval; + if (def.origin !== undefined) entry.origin = def.origin; + if (def.offset !== undefined) entry.offset = def.offset; + if (def.format !== undefined) entry.format = def.format; + granularities.push(entry); + } + } + res({ data: { granularities } }); + } catch (e: any) { + this.handleError({ + e, + context, + // @ts-ignore + res, + requestStarted, + }); + } + } + public async metaExtended({ context, res, onlyViews }: { context: ExtendedRequestContext, res: ResponseResultFn, @@ -1998,6 +2062,13 @@ class ApiGateway { }); metaConfigResult = this.filterVisibleItemsInMeta(context, metaConfigResult); + // Annotation reads from this meta. Without enrichment, /v1/load and /v1/cubesql would + // omit the type/title/format/interval fields that /v1/meta exposes. + const enrichedCubes = await this.applyGlobalGranularitiesToMetaCubes( + context, + metaConfigResult.map((m: any) => m.config), + ); + metaConfigResult = metaConfigResult.map((m: any, i: number) => ({ ...m, config: enrichedCubes[i] })); const sqlQueries = await this.getSqlQueriesInternal(context, normalizedQueries); @@ -2296,6 +2367,57 @@ class ApiGateway { return this.adapterApi(context); } + protected async resolveGlobalGranularitiesForRequest(context: RequestContext) { + return resolveGlobalGranularities(this.granularitiesOption, context); + } + + // Reconcile each time dimension's `granularitiesBlock` against the request's global config + // and emit the effective granularity array. Built-ins get tagged `built-in`, locals/globals + // become `custom`. Replaces the per-dim `granularities` array on the returned cube. + protected async applyGlobalGranularitiesToMetaCubes(context: RequestContext, cubes: any[]): Promise { + const globalConfig = await this.resolveGlobalGranularitiesForRequest(context); + const builtInsCatalog = buildBuiltInsCatalog(globalConfig); + + return cubes.map(cube => ({ + ...cube, + dimensions: cube.dimensions?.map((dim: any) => { + if (dim.type !== 'time') return dim; + // Re-key the local-custom array CubeToMetaTransformer produced so the resolver can + // merge it back into `granularitiesBlock.custom` cleanly. + const localCustom: Record = {}; + for (const g of dim.granularities || []) { + localCustom[g.name] = { + title: g.title, + interval: g.interval, + offset: g.offset, + origin: g.origin, + ...(g.format !== undefined ? { format: g.format } : {}), + }; + } + const block = dim.granularitiesBlock || normalizeGranularitiesBlock(undefined); + const blockWithLocal = { ...block, custom: { ...block.custom, ...localCustom } }; + const resolved = resolveDimensionGranularities( + blockWithLocal, + globalConfig.enabledBuiltIns, + globalConfig.customGranularities, + builtInsCatalog, + ); + const resolvedArray = Object.entries(resolved).map(([name, def]: [string, any]) => ({ + name, + type: def.type, + title: def.title, + ...(def.interval !== undefined ? { interval: def.interval } : {}), + ...(def.offset !== undefined ? { offset: def.offset } : {}), + ...(def.origin !== undefined ? { origin: def.origin } : {}), + ...(def.format !== undefined ? { format: def.format } : {}), + })); + // Strip the transport-only block; clients only see the resolved `granularities` array. + const { granularitiesBlock, ...rest } = dim; + return { ...rest, granularities: resolvedArray }; + }), + })); + } + public async contextByReq(req: Request, securityContext, requestId: string): Promise { req.securityContext = securityContext; diff --git a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts index 2f53164369ff1..92a7656740d1e 100644 --- a/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts +++ b/packages/cubejs-api-gateway/src/helpers/prepare-annotation.ts @@ -7,6 +7,7 @@ import R from 'ramda'; import { isPredefinedGranularity } from '@cubejs-backend/shared'; +import { BUILT_IN_GRANULARITIES } from '@cubejs-backend/schema-compiler'; import { MetaConfig, MetaConfigMap, toConfigMap } from './to-config-map'; import { MemberType } from '../types/strings'; import { MemberType as MemberTypeEnum } from '../types/enums'; @@ -14,7 +15,10 @@ import { MemberExpression } from '../types/query'; type GranularityMeta = { name: string; + type?: 'built-in' | 'custom'; title: string; + /** d3-time-format string for displaying bucketed timestamps. */ + format?: string; interval: string; offset?: string; origin?: string; @@ -115,14 +119,25 @@ function prepareAnnotation(metaConfig: MetaConfig[], query: any) { if (an) { let granularityMeta: GranularityMeta | undefined; if (isPredefinedGranularity(td.granularity)) { + // Prefer values the meta endpoint already attached (these honor any global + // title/format override). Fall back to BUILT_IN_GRANULARITIES, then to the bare name. + const fromMeta = an[1].granularities?.find(g => g.name === td.granularity); + const builtInDefaults = BUILT_IN_GRANULARITIES[td.granularity] || {}; granularityMeta = { name: td.granularity, - title: td.granularity, - interval: `1 ${td.granularity}`, + type: 'built-in', + title: fromMeta?.title || builtInDefaults.title || td.granularity, + interval: fromMeta?.interval || `1 ${td.granularity}`, + ...(fromMeta?.format || builtInDefaults.format + ? { format: fromMeta?.format || builtInDefaults.format } + : {}), }; } else if (an[1].granularities) { - // No need to send all the granularities defined, only those make sense for this query + // Forward only the granularity in play for this query; siblings stay in /v1/meta. granularityMeta = an[1].granularities.find(g => g.name === td.granularity); + if (granularityMeta && !granularityMeta.type) { + granularityMeta = { ...granularityMeta, type: 'custom' }; + } } const { granularities: _, ...rest } = an[1]; diff --git a/packages/cubejs-api-gateway/src/types/gateway.ts b/packages/cubejs-api-gateway/src/types/gateway.ts index e05f22d3c1e47..3b289c509062e 100644 --- a/packages/cubejs-api-gateway/src/types/gateway.ts +++ b/packages/cubejs-api-gateway/src/types/gateway.ts @@ -52,6 +52,19 @@ type ScheduledRefreshContextsFn = type ScheduledRefreshTimeZonesFn = (context: RequestContext) => string[] | Promise; +type GranularityListItem = string | { + name: string; + title?: string; + format?: string; + interval?: string; + origin?: string; + offset?: string; +}; +type GranularityList = GranularityListItem[]; +type GranularitiesOption = + | GranularityList + | ((context: RequestContext) => GranularityList | Promise); + /** * Gateway configuration options interface. */ @@ -64,6 +77,13 @@ interface ApiGatewayOptions { scheduledRefreshTimeZones?: ScheduledRefreshTimeZonesFn; basePath: string; extendContext?: ExtendContextFn; + /** + * Enabled granularities (built-in names and/or custom definitions), or a function called per + * request to produce the same. Drives /v1/granularities and the /v1/meta enrichment. + * Shape mirrors `GranularityList` in @cubejs-backend/schema-compiler; redeclared locally to + * avoid a dependency on schema-compiler from this types module. + */ + granularities?: GranularitiesOption; jwt?: JWTOptions; requestLoggerMiddleware?: RequestLoggerMiddlewareFn; queryRewrite?: QueryRewriteFn; diff --git a/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts b/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts index 2fe52c50fb52d..c87ee9291b411 100644 --- a/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts +++ b/packages/cubejs-api-gateway/test/helpers/prepare-annotation.test.ts @@ -182,6 +182,7 @@ describe('prepareAnnotation helpers', () => { }).timeDimensions ).toEqual({ 'cube_name.member': { + currency: undefined, description: undefined, format: undefined, meta: undefined, @@ -190,6 +191,7 @@ describe('prepareAnnotation helpers', () => { type: undefined, }, 'cube_name.member.day': { + currency: undefined, description: undefined, format: undefined, meta: undefined, @@ -198,8 +200,10 @@ describe('prepareAnnotation helpers', () => { type: undefined, granularity: { name: 'day', - title: 'day', + type: 'built-in', + title: 'Day', interval: '1 day', + format: '%Y-%m-%d', } }, }); diff --git a/packages/cubejs-backend-native/python/cube/src/__init__.py b/packages/cubejs-backend-native/python/cube/src/__init__.py index f779f8d13864f..af0b07735d235 100644 --- a/packages/cubejs-backend-native/python/cube/src/__init__.py +++ b/packages/cubejs-backend-native/python/cube/src/__init__.py @@ -70,6 +70,9 @@ class Configuration: check_sql_auth: Callable can_switch_sql_user: Callable extend_context: Callable + # Mirrors `granularities` in the JS config: a list of granularity names and/or custom-granularity + # definitions, or a function returning the same. Drives /v1/granularities and /v1/meta enrichment. + granularities: Union[list, Callable[[RequestContext], list]] scheduled_refresh_contexts: Callable context_to_api_scopes: Callable repository_factory: Callable @@ -120,6 +123,7 @@ def __init__(self): self.can_switch_sql_user = None self.query_rewrite = None self.extend_context = None + self.granularities = None self.scheduled_refresh_contexts = None self.scheduled_refresh_time_zones = None self.context_to_api_scopes = None diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 3cf619bd906e1..50d09ac43386a 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -2063,6 +2063,18 @@ const variables: Record any> = { .asString(), accessPolicyMaskNumber: () => get('CUBEJS_ACCESS_POLICY_MASK_NUMBER') .asString(), + // Comma-separated names (built-in or custom). Empty/unset = all 8 built-ins enabled. + granularities: () => get('CUBEJS_GRANULARITIES') + .asArray(','), + // `getEnv` forwards `opts` positionally, so callers pass `{ name }` (matches dbType: { dataSource }). + granularityCustomInterval: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_INTERVAL`) + .asString(), + granularityCustomTitle: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_TITLE`) + .asString(), + granularityCustomOffset: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_OFFSET`) + .asString(), + granularityCustomOrigin: ({ name }: { name: string }) => get(`CUBEJS_GRANULARITIES_${name.toUpperCase()}_ORIGIN`) + .asString(), }; type Vars = typeof variables; diff --git a/packages/cubejs-client-core/src/time.ts b/packages/cubejs-client-core/src/time.ts index 9143dd8bccf04..036860a649432 100644 --- a/packages/cubejs-client-core/src/time.ts +++ b/packages/cubejs-client-core/src/time.ts @@ -22,7 +22,12 @@ export type SqlInterval = string; // TODO: Define a better type as unitOfTime.DurationConstructor in moment.js export type ParsedInterval = Record; +// Runtime shape for custom-granularity time-series math. For built-ins, see +// TimeDimensionPredefinedGranularity (query value) and GranularityAnnotation (response field). export type Granularity = { + type?: 'built-in' | 'custom'; + title?: string; + format?: string; interval: SqlInterval; origin?: string; offset?: SqlInterval; diff --git a/packages/cubejs-client-core/src/types.ts b/packages/cubejs-client-core/src/types.ts index 876533d97d471..a097d436f6d3f 100644 --- a/packages/cubejs-client-core/src/types.ts +++ b/packages/cubejs-client-core/src/types.ts @@ -12,7 +12,11 @@ export type TQueryOrderArray = Array<[string, QueryOrder]>; export type GranularityAnnotation = { name: string; + type?: 'built-in' | 'custom'; title: string; + /** d3-time-format string for displaying bucketed timestamps. */ + format?: string; + /** Always present: built-ins use "1 "; customs carry the user-defined interval. */ interval: string; offset?: string; origin?: string; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 6d12a55035418..1596c76e4d500 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -5,6 +5,7 @@ import { camelize } from 'inflection'; import { UserError } from './UserError'; import { DynamicReference } from './DynamicReference'; import { camelizeCube } from './utils'; +import { normalizeGranularitiesBlock, NormalizedGranularitiesBlock } from './GranularityResolver'; import type { ErrorReporter } from './ErrorReporter'; import { TranspilerSymbolResolver } from './transpilers'; @@ -16,6 +17,8 @@ export type GranularityDefinition = { sql?: (...args: any[]) => string; name?: string; title?: string; + /** d3-time-format string used by the client to display bucketed timestamps. */ + format?: string; interval?: string; offset?: string; origin?: string; @@ -33,6 +36,7 @@ export type CubeSymbolDefinition = { sql?: (...args: any[]) => string; primaryKey?: boolean; granularities?: Record; + granularitiesBlock?: NormalizedGranularitiesBlock; timeShift?: TimeshiftDefinition[]; format?: string; currency?: string; @@ -561,6 +565,8 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface this.camelCaseTypes(cube.preAggregations); this.camelCaseTypes(cube.accessPolicy); + this.normalizeDimensionGranularities(cube.dimensions); + if (cube.preAggregations) { this.transformPreAggregations(cube.preAggregations); } @@ -579,6 +585,23 @@ export class CubeSymbols implements TranspilerSymbolResolver, CompilerInterface } as CubeSymbolsDefinition; } + // Stores the canonical `granularitiesBlock` on each time dimension and rewrites + // `granularities` to the dict of locally-defined customs only — preserving the legacy shape + // that BaseQuery, prepare-annotation, and CubeToMetaTransformer already read. + private normalizeDimensionGranularities(dimensions: Record | undefined) { + if (!dimensions) { + return; + } + + for (const dim of Object.values(dimensions)) { + if (dim && dim.type === 'time' && 'granularities' in dim) { + const block: NormalizedGranularitiesBlock = normalizeGranularitiesBlock(dim.granularities); + dim.granularitiesBlock = block; + dim.granularities = block.custom; + } + } + } + private camelCaseTypes(obj: Object | Array | undefined) { if (!obj) { return; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts index 0d8aa6551dc02..5f35b369d2b39 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.ts @@ -280,6 +280,10 @@ export class CubeToMetaTransformer implements CompilerInterface { ? this.isVisible(extendedDimDef, !extendedDimDef.primaryKey) : false; const granularitiesObj = extendedDimDef.granularities; + // The gateway reconciles `granularitiesBlock` (includes/excludes/custom) with global + // config per request. Forwarded as-is; the flat `granularities` field alone can't + // represent `includes: '*'` vs `includes: []`. + const { granularitiesBlock } = extendedDimDef as any; const dimType = this.dimensionDataType(extendedDimDef.type || 'string'); const dimFormat = this.transformDimensionFormat(extendedDimDef); const dimCurrency = extendedDimDef.currency?.toUpperCase(); @@ -306,12 +310,15 @@ export class CubeToMetaTransformer implements CompilerInterface { granularitiesObj ? Object.entries(granularitiesObj).map(([gName, gDef]: [string, any]) => ({ name: gName, + type: 'custom', title: this.title(cubeTitle, [gName, gDef], true), + ...(gDef.format !== undefined ? { format: gDef.format } : {}), interval: gDef.interval, offset: gDef.offset, origin: gDef.origin, })) : undefined, + granularitiesBlock, order: extendedDimDef.order, key: extendedDimDef.keyReference, }; diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index 17760cfcc32ee..b024749ddca6c 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -113,6 +113,74 @@ const GranularityInterval = Joi.string().pattern(/^\d+\s+(second|minute|hour|day // Do not allow negative intervals for granularities, while offsets could be negative const GranularityOffset = Joi.string().pattern(/^-?(\d+\s+)(second|minute|hour|day|week|month|quarter|year)s?(\s-?\d+\s+(second|minute|hour|day|week|month|quarter|year)s?){0,7}$/, 'granularity offset'); +// One custom granularity entry: with-origin, with-offset (interval must be aligned), or sql-defined. +// Reused by the legacy `granularities: { name: {...} }` form and the new `custom: { name: {...} }` form. +const CustomGranularityEntrySchema = Joi.alternatives([ + Joi.object().keys({ + title: Joi.string(), + format: Joi.string(), + interval: GranularityInterval.required(), + origin: Joi.string().required().custom((value, helpers) => { + const date = new Date(value); + + if (Number.isNaN(date.getTime())) { + return helpers.message({ custom: 'Origin should be valid date-only form: YYYY[-MM[-DD]] or date-time form: YYYY-MM-DD[T]HH:mm[:ss[.sss[Z]]]' }); + } + return value; + }), + }), + Joi.object().keys({ + title: Joi.string(), + format: Joi.string(), + interval: GranularityInterval.required().custom((value, helper) => { + const intParsed = value.split(' '); + const msg = { custom: 'Arbitrary intervals cannot be used without origin point specified' }; + + if (intParsed.length !== 2) { + return helper.message(msg); + } + + const v = parseInt(intParsed[0], 10); + const unit = intParsed[1]; + + const validIntervals = { + // Any number of years is valid + year: () => true, + // Only months divisible by a year with no remainder are valid + month: () => 12 % v === 0, + // Only quarters divisible by a year with no remainder are valid + quarter: () => 4 % v === 0, + // Only 1 week is valid + week: () => v === 1, + // Only 1 day is valid + day: () => v === 1, + // Only hours divisible by a day with no remainder are valid + hour: () => 24 % v === 0, + // Only minutes divisible by an hour with no remainder are valid + minute: () => 60 % v === 0, + // Only seconds divisible by a minute with no remainder are valid + second: () => 60 % v === 0, + }; + + const isValid = Object.keys(validIntervals).some(key => unit.includes(key) && validIntervals[key]()); + + return isValid ? value : helper.message(msg); + }), + offset: GranularityOffset.optional(), + }), + Joi.object().keys({ + title: Joi.string(), + format: Joi.string(), + sql: Joi.func().required() + }) +]); + +// `includes` / `excludes` accept a list of granularity names or the wildcard `'*'`. +const GranularityInclusionListSchema = Joi.alternatives([ + Joi.string().valid('*'), + Joi.array().items(Joi.string()), +]); + const formatAlternatives = [ Joi.string().valid('imageUrl', 'link', 'currency', 'percent', 'number', 'id'), Joi.object().keys({ @@ -349,65 +417,32 @@ const BaseDimensionWithoutSubQuery = { }), granularities: Joi.when('type', { is: 'time', - then: Joi.object().pattern(identifierRegex, - Joi.alternatives([ - Joi.object().keys({ - title: Joi.string(), - interval: GranularityInterval.required(), - origin: Joi.string().required().custom((value, helpers) => { - const date = new Date(value); - - if (Number.isNaN(date.getTime())) { - return helpers.message({ custom: 'Origin should be valid date-only form: YYYY[-MM[-DD]] or date-time form: YYYY-MM-DD[T]HH:mm[:ss[.sss[Z]]]' }); - } - return value; - }), - }), - Joi.object().keys({ - title: Joi.string(), - interval: GranularityInterval.required().custom((value, helper) => { - const intParsed = value.split(' '); - const msg = { custom: 'Arbitrary intervals cannot be used without origin point specified' }; - - if (intParsed.length !== 2) { - return helper.message(msg); - } - - const v = parseInt(intParsed[0], 10); - const unit = intParsed[1]; - - const validIntervals = { - // Any number of years is valid - year: () => true, - // Only months divisible by a year with no remainder are valid - month: () => 12 % v === 0, - // Only quarters divisible by a year with no remainder are valid - quarter: () => 4 % v === 0, - // Only 1 week is valid - week: () => v === 1, - // Only 1 day is valid - day: () => v === 1, - // Only hours divisible by a day with no remainder are valid - hour: () => 24 % v === 0, - // Only minutes divisible by an hour with no remainder are valid - minute: () => 60 % v === 0, - // Only seconds divisible by a minute with no remainder are valid - second: () => 60 % v === 0, - }; - - const isValid = Object.keys(validIntervals).some(key => unit.includes(key) && validIntervals[key]()); - - return isValid ? value : helper.message(msg); - }), - offset: GranularityOffset.optional(), + then: Joi.alternatives() + .conditional(Joi.ref('.'), { + // Discriminate by shape: only the new dict form has includes/excludes/custom and no other keys. + is: Joi.object().keys({ + includes: Joi.any(), + excludes: Joi.any(), + custom: Joi.any(), + }).unknown(false), + then: Joi.object().keys({ + includes: GranularityInclusionListSchema, + excludes: GranularityInclusionListSchema, + custom: Joi.object().pattern(identifierRegex, CustomGranularityEntrySchema), + }).custom((value, helper) => { + if (value && value.includes !== undefined && value.excludes !== undefined && value.includes !== '*') { + return helper.message({ custom: '"includes" and "excludes" cannot be used together unless includes is "*"' } as any); + } + return value; }), - Joi.object().keys({ - title: Joi.string(), - sql: Joi.func().required() - }) - ])).optional(), + otherwise: Joi.object().pattern(identifierRegex, CustomGranularityEntrySchema), + }) + .optional(), otherwise: Joi.forbidden() - }) + }), + // Internal field written by CubeSymbols.normalizeDimensionGranularities before the validator runs. + // Not user-facing; declared so unknown-key validation doesn't reject it. + granularitiesBlock: Joi.any() }; const BaseDimension = { diff --git a/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts new file mode 100644 index 0000000000000..d30aa08ea3bd0 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/GlobalGranularitiesConfig.ts @@ -0,0 +1,154 @@ +import { getEnv } from '@cubejs-backend/shared'; + +import type { GranularityDefinition } from './CubeSymbols'; + +// Default `title` and `format` for each built-in granularity. Overridable via `config.granularities` +// (file) or `CUBEJS_GRANULARITIES__TITLE` (env, title only). Format syntax: d3-time-format. + +export type BuiltInGranularityDefinition = { + title: string; + format: string; +}; + +export const BUILT_IN_GRANULARITIES: Readonly> = Object.freeze({ + year: { title: 'Year', format: '%Y' }, + quarter: { title: 'Quarter', format: 'Q%q %Y' }, + month: { title: 'Month', format: '%b %Y' }, + week: { title: 'Week', format: '%b %-d, %Y' }, + day: { title: 'Day', format: '%Y-%m-%d' }, + hour: { title: 'Hour', format: '%Y-%m-%d %H:00' }, + minute: { title: 'Minute', format: '%Y-%m-%d %H:%M' }, + second: { title: 'Second', format: '%Y-%m-%d %H:%M:%S' }, +}); + +export const BUILT_IN_GRANULARITY_NAMES = Object.freeze(Object.keys(BUILT_IN_GRANULARITIES)); + +export function isBuiltInGranularity(name: string): boolean { + return name in BUILT_IN_GRANULARITIES; +} + +// Item shape accepted in `config.granularities`: a built-in name, or a custom granularity object. +export type GranularityListItem = string | (GranularityDefinition & { name: string }); +export type GranularityList = GranularityListItem[]; + +export type GlobalGranularitiesConfig = { + enabledBuiltIns: ReadonlyArray; + customGranularities: Readonly>; +}; + +const DEFAULT_CONFIG: GlobalGranularitiesConfig = Object.freeze({ + enabledBuiltIns: BUILT_IN_GRANULARITY_NAMES, + customGranularities: Object.freeze({}), +}); + +// Read `CUBEJS_GRANULARITIES__{INTERVAL,TITLE,OFFSET,ORIGIN}` for a custom granularity name. +// Only consulted for names sourced from `CUBEJS_GRANULARITIES`, not for `config.granularities` entries. +function applyEnvOverrides(name: string, base?: Partial): GranularityDefinition { + // getEnv types `opts` as a Parameters<> tuple but forwards it positionally; cast to bypass that. + const opts = { name } as any; + const interval = getEnv('granularityCustomInterval', opts) ?? base?.interval; + const title = getEnv('granularityCustomTitle', opts) ?? base?.title; + const offset = getEnv('granularityCustomOffset', opts) ?? base?.offset; + const origin = getEnv('granularityCustomOrigin', opts) ?? base?.origin; + + const out: GranularityDefinition = {}; + if (interval !== undefined) out.interval = interval; + if (title !== undefined) out.title = title; + if (offset !== undefined) out.offset = offset; + if (origin !== undefined) out.origin = origin; + return out; +} + +function resolveFromEnv(): GlobalGranularitiesConfig { + const list = getEnv('granularities'); + if (!list || list.length === 0) { + return DEFAULT_CONFIG; + } + + const enabledBuiltIns: string[] = []; + const customGranularities: Record = {}; + for (const name of list) { + const trimmed = name.trim(); + if (trimmed) { + if (isBuiltInGranularity(trimmed)) { + enabledBuiltIns.push(trimmed); + } else { + // Non-built-in name: pull the definition from `CUBEJS_GRANULARITIES__*` env vars. + customGranularities[trimmed] = applyEnvOverrides(trimmed); + } + } + } + return { enabledBuiltIns, customGranularities }; +} + +function resolveFromList(list: GranularityList): GlobalGranularitiesConfig { + const enabledBuiltIns: string[] = []; + const customGranularities: Record = {}; + + for (const item of list) { + if (typeof item === 'string') { + if (isBuiltInGranularity(item)) { + enabledBuiltIns.push(item); + } + // A bare non-built-in string has no definition attached and is silently dropped: + // custom granularities in `config.granularities` must be objects. + } else if (item && typeof item === 'object' && item.name) { + const { name, ...def } = item; + if (isBuiltInGranularity(name)) { + // `{ name: 'year', title: 'Anno' }` both enables 'year' and overrides its title/format. + enabledBuiltIns.push(name); + customGranularities[name] = def; + } else { + customGranularities[name] = def; + } + } + } + return { enabledBuiltIns, customGranularities }; +} + +// `userValue` is the value of `granularities` from the cube.js / cube.py config file. +// undefined -> fall back to `CUBEJS_GRANULARITIES` env vars +// GranularityList -> use this list, replacing env vars entirely (no merge) +// function(ctx) -> called per request; same no-merge replacement as the list form +export async function resolveGlobalGranularities( + userValue: GranularityList | ((ctx: any) => GranularityList | Promise) | undefined, + ctx: any, +): Promise { + if (userValue === undefined) { + return resolveFromEnv(); + } + const resolved = typeof userValue === 'function' ? await userValue(ctx) : userValue; + if (!Array.isArray(resolved)) { + return resolveFromEnv(); + } + return resolveFromList(resolved); +} + +export function getBuiltInGranularityDefaults(name: string) { + return BUILT_IN_GRANULARITIES[name]; +} + +// Resolve title/format/interval for each enabled built-in, applying any override from +// `globalConfig.customGranularities` (e.g. `{ name: 'year', title: 'Jaar' }` localizes the title). +// `interval` is filled in as "1 " so built-ins share the same response shape as customs. +export type BuiltInCatalogEntry = { + title: string; + format: string; + interval: string; +}; + +export function buildBuiltInsCatalog(globalConfig: GlobalGranularitiesConfig): Record { + const catalog: Record = {}; + for (const name of globalConfig.enabledBuiltIns) { + const defaults = BUILT_IN_GRANULARITIES[name]; + if (defaults) { + const override = globalConfig.customGranularities[name]; + catalog[name] = { + title: override?.title || defaults.title, + format: override?.format || defaults.format, + interval: override?.interval || `1 ${name}`, + }; + } + } + return catalog; +} diff --git a/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts new file mode 100644 index 0000000000000..4f73924a6460d --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/GranularityResolver.ts @@ -0,0 +1,107 @@ +import type { GranularityDefinition } from './CubeSymbols'; + +// `'*'` is the wildcard form of an includes/excludes list. +export type GranularityInclusionList = '*' | string[]; + +// Canonical form produced by `normalizeGranularitiesBlock`. All downstream readers +// (resolver, meta transformer, pre-agg matcher) only see this shape. +export type NormalizedGranularitiesBlock = { + includes: GranularityInclusionList; + excludes: GranularityInclusionList; + custom: Record; +}; + +// Effective granularity set for one time dimension, ready to serialize into /v1/meta. +export type ResolvedGranularitySet = Record; + +// Used when a time dimension has no `granularities` block at all: take all enabled globals, no local customs. +const EMPTY_BLOCK: NormalizedGranularitiesBlock = { + includes: '*', + excludes: [], + custom: {}, +}; + +// Map any accepted input shape (omitted, legacy array, post-yaml keyed object, new dict) +// onto NormalizedGranularitiesBlock. Validator runs first, so malformed input never reaches here. +export function normalizeGranularitiesBlock(raw: any): NormalizedGranularitiesBlock { + if (raw == null) { + return EMPTY_BLOCK; + } + + if (Array.isArray(raw)) { + // Legacy form. Every array entry is a custom granularity; built-ins are inherited from globals. + return { + includes: '*', + excludes: [], + custom: Object.fromEntries(raw.filter(g => g && g.name).map(g => { + const { name, ...rest } = g; + return [name, rest]; + })), + }; + } + + if (typeof raw === 'object') { + if ('includes' in raw || 'excludes' in raw || 'custom' in raw) { + // New dict form. YamlCompiler has already keyed `custom` by name. + return { + includes: raw.includes ?? '*', + excludes: raw.excludes ?? [], + custom: raw.custom ?? {}, + }; + } + // Already-keyed legacy form (e.g. coming from JS configs, not YAML). Treat as custom-only. + return { + includes: '*', + excludes: [], + custom: raw, + }; + } + + return EMPTY_BLOCK; +} + +// Reconcile a dimension's local block against the global enabled built-ins and global customs, +// producing the effective set. Local customs always survive — even if local excludes is '*'. +export function resolveDimensionGranularities( + localBlock: NormalizedGranularitiesBlock, + globalEnabledBuiltIns: ReadonlyArray, + globalCustom: Readonly>, + allBuiltInsCatalog: Readonly>, +): ResolvedGranularitySet { + const out: ResolvedGranularitySet = {}; + + const localIncludes = localBlock.includes; + const localExcludes = localBlock.excludes; + + const includesAllowsAll = localIncludes === '*'; + const includesSet = includesAllowsAll ? null : new Set(localIncludes); + const excludesBlocksAll = localExcludes === '*'; + const excludesSet = excludesBlocksAll ? null : new Set(localExcludes); + + // Built-ins and global customs are filtered the same way: keep iff included AND not excluded. + if (!excludesBlocksAll) { + for (const builtInName of globalEnabledBuiltIns) { + const passesIncludes = includesAllowsAll || includesSet!.has(builtInName); + const blockedByExcludes = excludesSet!.has(builtInName); + const def = allBuiltInsCatalog[builtInName]; + if (passesIncludes && !blockedByExcludes && def) { + out[builtInName] = { ...def, type: 'built-in' }; + } + } + for (const [name, def] of Object.entries(globalCustom)) { + const passesIncludes = includesAllowsAll || includesSet!.has(name); + const blockedByExcludes = excludesSet!.has(name); + if (passesIncludes && !blockedByExcludes) { + out[name] = { ...def, type: 'custom' }; + } + } + } + + // Local customs are always emitted, even when `excludes: '*'` strips everything else. + // Same-named local entry replaces a global one (last-write-wins). + for (const [name, def] of Object.entries(localBlock.custom)) { + out[name] = { ...def, type: 'custom' }; + } + + return out; +} diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index 078308a6efe25..5b37821977c1a 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -416,10 +416,22 @@ export class YamlCompiler { } if (memberType === 'dimension' && granularities) { - granularities = this.yamlArrayToObj(granularities || [], 'dimension.granularity', errorsReport, { - cubeName: ctx.cubeName, - parent: { type: 'time dimension', name } - }); + if (Array.isArray(granularities)) { + // Legacy form: `granularities: [{ name, ... }]` -> `{ name: { ... } }`. + granularities = this.yamlArrayToObj(granularities || [], 'dimension.granularity', errorsReport, { + cubeName: ctx.cubeName, + parent: { type: 'time dimension', name } + }); + } else if (granularities && typeof granularities === 'object' && Array.isArray(granularities.custom)) { + // New dict form: only the inner `custom` array needs keying; includes/excludes pass through. + granularities = { + ...granularities, + custom: this.yamlArrayToObj(granularities.custom, 'dimension.granularity', errorsReport, { + cubeName: ctx.cubeName, + parent: { type: 'time dimension', name } + }), + }; + } res[name] = { granularities, ...res[name] }; } diff --git a/packages/cubejs-schema-compiler/src/compiler/index.ts b/packages/cubejs-schema-compiler/src/compiler/index.ts index ff62cfaa59ddc..86f7b4ff85114 100644 --- a/packages/cubejs-schema-compiler/src/compiler/index.ts +++ b/packages/cubejs-schema-compiler/src/compiler/index.ts @@ -11,3 +11,22 @@ export { PreAggregationInfo, EvaluatedCube, } from './CubeEvaluator'; +export { + BUILT_IN_GRANULARITIES, + BUILT_IN_GRANULARITY_NAMES, + isBuiltInGranularity, + BuiltInGranularityDefinition, + GranularityList, + GranularityListItem, + GlobalGranularitiesConfig, + BuiltInCatalogEntry, + resolveGlobalGranularities, + getBuiltInGranularityDefaults, + buildBuiltInsCatalog, +} from './GlobalGranularitiesConfig'; +export { + NormalizedGranularitiesBlock, + ResolvedGranularitySet, + normalizeGranularitiesBlock, + resolveDimensionGranularities, +} from './GranularityResolver'; diff --git a/packages/cubejs-schema-compiler/src/compiler/utils.ts b/packages/cubejs-schema-compiler/src/compiler/utils.ts index f917e6a93be06..9af688a1e92ed 100644 --- a/packages/cubejs-schema-compiler/src/compiler/utils.ts +++ b/packages/cubejs-schema-compiler/src/compiler/utils.ts @@ -4,6 +4,11 @@ import { camelize } from 'inflection'; const IGNORE_CAMELIZE = { 1: { granularities: true, + }, + // Custom granularity names live one level deeper than the legacy flat-array form + // (`granularities.custom.`), so they need their own guard. + 2: { + custom: true, } }; diff --git a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts index 1a31b7002486e..b2f1c828c08c5 100644 --- a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts @@ -1309,6 +1309,93 @@ describe('Cube Validation', () => { }); }); + describe('Granularities dict shape (includes/excludes/custom):', () => { + const newCube = (granularities: any) => ({ + name: 'Orders', + fileName: 'fileName', + sql: () => 'select * from tbl', + public: true, + dimensions: { + createdAt: { + public: true, + sql: () => 'created_at', + type: 'time', + granularities, + }, + }, + measures: { + count: { sql: () => 'count', type: 'count' }, + }, + }); + + it('accepts includes-only with built-in names', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: ['year', 'quarter'] }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts excludes-only with built-in names', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ excludes: ['day'] }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts includes "*" wildcard', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: '*' }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts custom-only', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ + custom: { fiscal_year: { interval: '1 year', origin: '2026-04-01' } }, + }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts includes "*" combined with excludes', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: '*', excludes: ['day'] }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('accepts excludes "*" combined with custom', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ + excludes: '*', + custom: { fiscal_year: { interval: '1 year', origin: '2026-04-01' } }, + }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + + it('rejects includes + excludes both as lists (mutually exclusive)', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ includes: ['year'], excludes: ['day'] }); + let captured: string | undefined; + const validationResult = cubeValidator.validate(cube, { + error: (message: any) => { captured = message; }, + } as any); + expect(validationResult.error).toBeTruthy(); + expect(captured).toContain('"includes" and "excludes" cannot be used together'); + }); + + it('legacy flat-array form (post yamlArrayToObj) still validates', () => { + const cubeValidator = new CubeValidator(new CubeSymbols()); + const cube = newCube({ + fiscal_year: { interval: '1 year', origin: '2026-04-01' }, + }); + const validationResult = cubeValidator.validate(cube, new ConsoleErrorReporter()); + expect(validationResult.error).toBeFalsy(); + }); + }); + describe('Access Policy group/groups support:', () => { const cubeValidator = new CubeValidator(new CubeSymbols()); diff --git a/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts b/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts new file mode 100644 index 0000000000000..be38fae4b89b6 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/granularities-config.test.ts @@ -0,0 +1,75 @@ +import { resolveGlobalGranularities, BUILT_IN_GRANULARITY_NAMES } from '../../src/compiler/GlobalGranularitiesConfig'; + +describe('resolveGlobalGranularities', () => { + const originalEnv = { ...process.env }; + + afterEach(() => { + for (const key of Object.keys(process.env)) { + if (key.startsWith('CUBEJS_GRANULARITIES')) delete process.env[key]; + } + Object.assign(process.env, originalEnv); + }); + + it('user value undefined + no env -> defaults to full built-in catalog', async () => { + const cfg = await resolveGlobalGranularities(undefined, {}); + expect([...cfg.enabledBuiltIns].sort()).toEqual([...BUILT_IN_GRANULARITY_NAMES].sort()); + expect(cfg.customGranularities).toEqual({}); + }); + + it('CUBEJS_GRANULARITIES restricts enabled built-ins', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,quarter,month'; + const cfg = await resolveGlobalGranularities(undefined, {}); + expect(cfg.enabledBuiltIns).toEqual(['year', 'quarter', 'month']); + expect(cfg.customGranularities).toEqual({}); + }); + + it('CUBEJS_GRANULARITIES with a custom name + companion env vars produces a custom granularity', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,fiscal_year'; + process.env.CUBEJS_GRANULARITIES_FISCAL_YEAR_INTERVAL = '1 year'; + process.env.CUBEJS_GRANULARITIES_FISCAL_YEAR_ORIGIN = '2026-04-01'; + process.env.CUBEJS_GRANULARITIES_FISCAL_YEAR_TITLE = 'Fiscal Year'; + const cfg = await resolveGlobalGranularities(undefined, {}); + expect(cfg.enabledBuiltIns).toEqual(['year']); + expect(cfg.customGranularities.fiscal_year).toEqual({ + interval: '1 year', + origin: '2026-04-01', + title: 'Fiscal Year', + }); + }); + + it('file-config replaces env vars (full replacement, not merge)', async () => { + process.env.CUBEJS_GRANULARITIES = 'year,quarter,month'; + const cfg = await resolveGlobalGranularities(['day', 'hour'], {}); + expect(cfg.enabledBuiltIns).toEqual(['day', 'hour']); + }); + + it('file-config can mix built-in names + custom objects', async () => { + const cfg = await resolveGlobalGranularities( + ['year', { name: 'fiscal_year', interval: '1 year', origin: '2026-04-01' }], + {}, + ); + expect(cfg.enabledBuiltIns).toEqual(['year']); + expect(cfg.customGranularities.fiscal_year).toEqual({ interval: '1 year', origin: '2026-04-01' }); + }); + + it('file-config function is invoked with the request context', async () => { + let receivedCtx: any; + const cfg = await resolveGlobalGranularities( + (ctx) => { + receivedCtx = ctx; + return ['year']; + }, + { securityContext: { tenant: 't1' } }, + ); + expect(receivedCtx.securityContext.tenant).toBe('t1'); + expect(cfg.enabledBuiltIns).toEqual(['year']); + }); + + it('file-config function may return a Promise', async () => { + const cfg = await resolveGlobalGranularities( + async () => ['quarter'], + {}, + ); + expect(cfg.enabledBuiltIns).toEqual(['quarter']); + }); +}); diff --git a/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts b/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts new file mode 100644 index 0000000000000..39f4924f203d3 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/granularities-shape.test.ts @@ -0,0 +1,154 @@ +import { + normalizeGranularitiesBlock, + resolveDimensionGranularities, +} from '../../src/compiler/GranularityResolver'; + +const BUILT_INS = { + year: { title: 'Year' }, + quarter: { title: 'Quarter' }, + month: { title: 'Month' }, + week: { title: 'Week' }, + day: { title: 'Day' }, + hour: { title: 'Hour' }, + minute: { title: 'Minute' }, + second: { title: 'Second' }, +}; + +const ALL_ENABLED = Object.keys(BUILT_INS); +const FISCAL_YEAR = { title: 'Fiscal Year', interval: '1 year', origin: '2026-04-01' }; +const GLOBAL_CUSTOM = { fiscal_year: FISCAL_YEAR }; + +describe('normalizeGranularitiesBlock', () => { + it('treats missing input as wide-open: includes * / no excludes / no custom', () => { + expect(normalizeGranularitiesBlock(undefined)).toEqual({ + includes: '*', + excludes: [], + custom: {}, + }); + expect(normalizeGranularitiesBlock(null)).toEqual({ + includes: '*', + excludes: [], + custom: {}, + }); + }); + + it('legacy flat-array form maps each entry into custom; includes stays *', () => { + const out = normalizeGranularitiesBlock([ + { name: 'fiscal_q', interval: '3 months', origin: '2026-04-01' }, + ]); + expect(out.includes).toBe('*'); + expect(out.excludes).toEqual([]); + expect(out.custom).toEqual({ + fiscal_q: { interval: '3 months', origin: '2026-04-01' }, + }); + }); + + it('post-yamlArrayToObj keyed object is preserved as legacy custom-only block', () => { + const out = normalizeGranularitiesBlock({ + fiscal_q: { interval: '3 months', origin: '2026-04-01' }, + }); + expect(out.includes).toBe('*'); + expect(out.custom.fiscal_q).toEqual({ interval: '3 months', origin: '2026-04-01' }); + }); + + it('new dict shape is canonicalized with defaults', () => { + const out = normalizeGranularitiesBlock({ includes: ['year'], custom: { fy: FISCAL_YEAR } }); + expect(out).toEqual({ + includes: ['year'], + excludes: [], + custom: { fy: FISCAL_YEAR }, + }); + }); +}); + +describe('resolveDimensionGranularities — spec resolution table', () => { + it('row 1: granularities omitted -> all enabled global granularities', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(Object.keys(out).sort()).toEqual([...ALL_ENABLED].sort()); + expect(out.year.type).toBe('built-in'); + }); + + it('row 2: legacy flat array -> enabled globals plus local custom', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock([{ name: 'fiscal_q', interval: '3 months', origin: '2026-04-01' }]), + ['year', 'month'], + {}, + BUILT_INS, + ); + expect(out.year.type).toBe('built-in'); + expect(out.month.type).toBe('built-in'); + expect(out.fiscal_q).toMatchObject({ interval: '3 months', origin: '2026-04-01', type: 'custom' }); + }); + + it('row 3: includes [a, b] + custom -> {a, b} plus custom', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ includes: ['year', 'quarter'], custom: { fy: FISCAL_YEAR } }), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(Object.keys(out).sort()).toEqual(['fy', 'quarter', 'year']); + expect(out.year.type).toBe('built-in'); + expect(out.fy.type).toBe('custom'); + }); + + it('row 4: excludes [x] -> all enabled globals minus x', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ excludes: ['day'] }), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(out.day).toBeUndefined(); + expect(out.year.type).toBe('built-in'); + }); + + it('row 5: excludes "*" + custom -> custom only', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ excludes: '*', custom: { fy: FISCAL_YEAR } }), + ALL_ENABLED, + {}, + BUILT_INS, + ); + expect(Object.keys(out)).toEqual(['fy']); + expect(out.fy.type).toBe('custom'); + }); + + it('row 6: includes "*" + custom -> all enabled globals plus custom', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ includes: '*', custom: { fy: FISCAL_YEAR } }), + ['year', 'month'], + {}, + BUILT_INS, + ); + expect(Object.keys(out).sort()).toEqual(['fy', 'month', 'year']); + expect(out.fy.type).toBe('custom'); + }); + + it('global custom granularities flow through unless excluded', () => { + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock(undefined), + ['year'], + GLOBAL_CUSTOM, + BUILT_INS, + ); + expect(out.fiscal_year.type).toBe('custom'); + expect(out.year.type).toBe('built-in'); + }); + + it('local custom overrides global custom of same name', () => { + const localFy = { interval: '1 year', origin: '2026-01-01' }; + const out = resolveDimensionGranularities( + normalizeGranularitiesBlock({ custom: { fiscal_year: localFy } }), + ['year'], + GLOBAL_CUSTOM, + BUILT_INS, + ); + expect(out.fiscal_year.origin).toBe('2026-01-01'); + }); +}); diff --git a/packages/cubejs-server-core/src/core/optionsValidate.ts b/packages/cubejs-server-core/src/core/optionsValidate.ts index 7a3c5b9e97027..095d2807c0682 100644 --- a/packages/cubejs-server-core/src/core/optionsValidate.ts +++ b/packages/cubejs-server-core/src/core/optionsValidate.ts @@ -91,6 +91,15 @@ const schemaOptions = Joi.object().keys({ ), schemaVersion: Joi.func(), extendContext: Joi.func(), + granularities: Joi.alternatives().try( + Joi.func(), + Joi.array().items( + Joi.alternatives().try( + Joi.string(), + Joi.object().unknown(true) + ) + ) + ), // Scheduled refresh scheduledRefreshTimer: Joi.alternatives().try( Joi.boolean(), diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index 670cf127cf6bd..22fded66fcb17 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -481,6 +481,7 @@ export class CubejsServerCore { queryRewrite: this.options.queryRewrite || this.options.queryTransformer, extendContext: this.options.extendContext, + granularities: this.options.granularities, playgroundAuthSecret: getEnv('playgroundAuthSecret'), jwt: this.options.jwt, refreshScheduler: this.getRefreshScheduler.bind(this), diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts index 10d680de3b1cd..6b26b37a5a331 100644 --- a/packages/cubejs-server-core/src/core/types.ts +++ b/packages/cubejs-server-core/src/core/types.ts @@ -10,7 +10,7 @@ import { UserBackgroundContext, } from '@cubejs-backend/api-gateway'; import { BaseDriver, CacheAndQueryDriverType } from '@cubejs-backend/query-orchestrator'; -import { BaseQuery } from '@cubejs-backend/schema-compiler'; +import { BaseQuery, GranularityList } from '@cubejs-backend/schema-compiler'; export interface QueueOptions { concurrency?: number; @@ -222,6 +222,7 @@ export interface CreateOptions { preAggregationsSchema?: string | PreAggregationsSchemaFn; schemaVersion?: (context: RequestContext) => string | Promise; extendContext?: ExtendContextFn; + granularities?: GranularityList | ((context: RequestContext) => GranularityList | Promise); scheduledRefreshTimer?: boolean | number; scheduledRefreshTimeZones?: string[] | ScheduledRefreshTimeZonesFn; scheduledRefreshContexts?: () => Promise; diff --git a/rust/cube/cubeorchestrator/src/transport.rs b/rust/cube/cubeorchestrator/src/transport.rs index f0d8e85423e95..032c726060b57 100644 --- a/rust/cube/cubeorchestrator/src/transport.rs +++ b/rust/cube/cubeorchestrator/src/transport.rs @@ -164,7 +164,13 @@ pub type MembersMap = IndexMap; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GranularityMeta { pub name: String, + /// Serialized as `type`: "built-in" or "custom". Field is named `kind` to avoid Rust's keyword. + #[serde(rename = "type", skip_serializing_if = "Option::is_none")] + pub kind: Option, pub title: String, + /// d3-time-format string used by the client to display bucketed timestamps. + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, #[serde(skip_serializing_if = "Option::is_none")] pub interval: Option, #[serde(skip_serializing_if = "Option::is_none")]