From 0523f7a823ffbd878292f6e422989e625985a31f Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 17 Jul 2025 15:47:58 +0200 Subject: [PATCH 1/3] fix(native): Handle null correctly as JsNull, not as JsString (#9717) Fixing an unexpected issue with treating null as a null string. I figured out this issue while benchmarking the cache store queue. Previously, it could lead to an issue in the production environment while someone moves between cube store versions or has an error while updating the queue_item's extra field. --- packages/cubejs-backend-native/src/orchestrator.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-backend-native/src/orchestrator.rs b/packages/cubejs-backend-native/src/orchestrator.rs index 00ab47b16be06..f2d8a133a0726 100644 --- a/packages/cubejs-backend-native/src/orchestrator.rs +++ b/packages/cubejs-backend-native/src/orchestrator.rs @@ -2,7 +2,8 @@ use crate::node_obj_deserializer::JsValueDeserializer; use crate::transport::MapCubeErrExt; use cubeorchestrator::query_message_parser::QueryResult; use cubeorchestrator::query_result_transform::{ - DBResponsePrimitive, RequestResultData, RequestResultDataMulti, TransformedData, + DBResponsePrimitive, DBResponseValue, RequestResultData, RequestResultDataMulti, + TransformedData, }; use cubeorchestrator::transport::{JsRawData, TransformDataRequest}; use cubesql::compile::engine::df::scan::{FieldValue, ValueObject}; @@ -258,7 +259,12 @@ pub fn get_cubestore_result(mut cx: FunctionContext) -> JsResult { let js_row = JsObject::new(&mut cx); for (key, value) in result.columns.iter().zip(row.iter()) { let js_key = cx.string(key); - let js_value = cx.string(value.to_string()); + let js_value: Handle<'_, JsValue> = match value { + DBResponseValue::Primitive(DBResponsePrimitive::Null) => cx.null().upcast(), + // For compatibility, we convert all primitives to strings + other => cx.string(other.to_string()).upcast(), + }; + js_row.set(&mut cx, js_key, js_value)?; } Ok(js_row) From d5172f9ac65f437962fc1076bceb31d73985b33b Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 17 Jul 2025 16:56:19 +0300 Subject: [PATCH 2/3] chore(schema-compiler): More defined types (#9785) * add @types/node-dijkstra * some ramda fixes * more types in base member classes * more types * more types and EvaluatedCube type!! * more types * more types in CubeSymbols * more types in CubeSymbols * fix types in QueryOrchestrator * more types in CubeSymbols & Evaluator * fix types in ScaffoldingSchema * move JoinGraph to ts * more types in BaseQuery * fix * fix --- .../src/orchestrator/QueryOrchestrator.ts | 26 +- packages/cubejs-schema-compiler/package.json | 1 + .../src/adapter/BaseDimension.ts | 8 +- .../src/adapter/BaseFilter.ts | 4 +- .../src/adapter/BaseMeasure.ts | 28 +- .../src/adapter/BaseQuery.js | 76 +++- .../src/adapter/BaseSegment.ts | 12 +- .../src/adapter/PreAggregations.ts | 28 +- .../src/compiler/CubeEvaluator.ts | 212 +++++----- .../src/compiler/CubeSymbols.ts | 152 ++++++-- .../src/compiler/CubeToMetaTransformer.js | 2 +- .../src/compiler/CubeValidator.ts | 8 +- .../src/compiler/JoinGraph.js | 240 ------------ .../src/compiler/JoinGraph.ts | 361 ++++++++++++++++++ .../src/scaffolding/ScaffoldingSchema.ts | 10 +- .../test/unit/cube-validator.test.ts | 12 +- yarn.lock | 5 + 17 files changed, 761 insertions(+), 424 deletions(-) delete mode 100644 packages/cubejs-schema-compiler/src/compiler/JoinGraph.js create mode 100644 packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts index 46e55da79519f..a9c9340d8bc08 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts @@ -3,7 +3,7 @@ import R from 'ramda'; import { getEnv } from '@cubejs-backend/shared'; import { CubeStoreDriver } from '@cubejs-backend/cubestore-driver'; -import { QueryCache, QueryBody, TempTable } from './QueryCache'; +import { QueryCache, QueryBody, TempTable, PreAggTableToTempTable } from './QueryCache'; import { PreAggregations, PreAggregationDescription, getLastUpdatedAtTimestamp } from './PreAggregations'; import { DriverFactory, DriverFactoryByDataSource } from './DriverFactory'; import { LocalQueueEventsBus } from './LocalQueueEventsBus'; @@ -224,28 +224,18 @@ export class QueryOrchestrator { }; } - const usedPreAggregations = R.pipe( + const usedPreAggregations = R.pipe< + PreAggTableToTempTable[], + Record, + Record + >( R.fromPairs, - R.map((pa: TempTable) => ({ + R.mapObjIndexed((pa: TempTable) => ({ targetTableName: pa.targetTableName, refreshKeyValues: pa.refreshKeyValues, lastUpdatedAt: pa.lastUpdatedAt, })), - )( - preAggregationsTablesToTempTables as unknown as [ - number, // TODO: we actually have a string here - { - buildRangeEnd: string, - lastUpdatedAt: number, - queryKey: unknown, - refreshKeyValues: [{ - 'refresh_key': string, - }][], - targetTableName: string, - type: string, - }, - ][] - ); + )(preAggregationsTablesToTempTables); if (this.rollupOnlyMode && Object.keys(usedPreAggregations).length === 0) { throw new Error( diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index c16b730803687..0982f58d4cf6b 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -68,6 +68,7 @@ "@types/inflection": "^1.5.28", "@types/jest": "^29", "@types/node": "^20", + "@types/node-dijkstra": "^2.5.6", "@types/ramda": "^0.27.34", "@types/sqlstring": "^2.3.0", "@types/syntax-error": "^1.4.1", diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts b/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts index 34e8fde294ed4..9459eece31627 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts @@ -142,6 +142,12 @@ export class BaseDimension { if (this.expression) { return `expr:${this.expressionName}`; } - return this.query.cubeEvaluator.pathFromArray(this.path() as string[]); + + const path = this.path(); + if (path === null) { + // Sanity check, this should not actually happen because we checked this.expression earlier + throw new Error('Unexpected null path'); + } + return this.query.cubeEvaluator.pathFromArray(path); } } diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts b/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts index f656b3249affd..f4fea50f91cb2 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts @@ -1,6 +1,6 @@ import inlection from 'inflection'; import moment from 'moment-timezone'; -import { contains, join, map } from 'ramda'; +import { includes, join, map } from 'ramda'; import { FROM_PARTITION_RANGE, TO_PARTITION_RANGE } from '@cubejs-backend/shared'; import { BaseDimension } from './BaseDimension'; @@ -134,7 +134,7 @@ export class BaseFilter extends BaseDimension { } public isDateOperator(): boolean { - return contains(this.camelizeOperator, DATE_OPERATORS); + return includes(this.camelizeOperator, DATE_OPERATORS); } public valuesArray() { diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts index dff05deb9ff5f..091e1dede408f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts @@ -206,32 +206,32 @@ export class BaseMeasure { return this.measureDefinition(); } - public aliasName() { + public aliasName(): string { return this.query.escapeColumnName(this.unescapedAliasName()); } - public unescapedAliasName() { + public unescapedAliasName(): string { if (this.expression) { return this.query.aliasName(this.expressionName); } return this.query.aliasName(this.measure); } - public isCumulative() { + public isCumulative(): boolean { if (this.expression) { // TODO return false; } return BaseMeasure.isCumulative(this.measureDefinition()); } - public isMultiStage() { + public isMultiStage(): boolean { if (this.expression) { // TODO return false; } return this.definition().multiStage; } - public isAdditive() { + public isAdditive(): boolean { if (this.expression) { // TODO return false; } @@ -243,7 +243,7 @@ export class BaseMeasure { definition.type === 'min' || definition.type === 'max'; } - public static isCumulative(definition) { + public static isCumulative(definition): boolean { return definition.type === 'runningTotal' || !!definition.rollingWindow; } @@ -294,7 +294,7 @@ export class BaseMeasure { return this.query.minGranularity(granularityA, granularityB); } - public granularityFromInterval(interval: string) { + public granularityFromInterval(interval: string): string | undefined { if (!interval) { return undefined; } @@ -312,7 +312,7 @@ export class BaseMeasure { return undefined; } - public shouldUngroupForCumulative() { + public shouldUngroupForCumulative(): boolean { return this.measureDefinition().rollingWindow && !this.isAdditive(); } @@ -320,17 +320,23 @@ export class BaseMeasure { return this.measureDefinition().sql; } - public path() { + public path(): string[] | null { if (this.expression) { return null; } return this.query.cubeEvaluator.parsePath('measures', this.measure); } - public expressionPath() { + public expressionPath(): string { if (this.expression) { return `expr:${this.expression.expressionName}`; } - return this.query.cubeEvaluator.pathFromArray(this.path() as string[]); + + const path = this.path(); + if (path === null) { + // Sanity check, this should not actually happen because we checked this.expression earlier + throw new Error('Unexpected null path'); + } + return this.query.cubeEvaluator.pathFromArray(path); } } diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 43b270f34b5a9..483efe6e3e44d 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -130,6 +130,11 @@ export class BaseQuery { /** @type {import('./BaseTimeDimension').BaseTimeDimension[]} */ timeDimensions; + /** + * @type {import('../compiler/JoinGraph').FinishedJoinTree} + */ + join; + /** * BaseQuery class constructor. * @param {Compilers|*} compilers @@ -2166,6 +2171,12 @@ export class BaseQuery { )); } + /** + * + * @param {string} cube + * @param {boolean} [isLeftJoinCondition] + * @returns {[string, string, string?]} + */ rewriteInlineCubeSql(cube, isLeftJoinCondition) { const sql = this.cubeSql(cube); const cubeAlias = this.cubeAlias(cube); @@ -2188,9 +2199,14 @@ export class BaseQuery { } } + /** + * @param {import('../compiler/JoinGraph').FinishedJoinTree} join + * @param {Array} subQueryDimensions + * @returns {string} + */ joinQuery(join, subQueryDimensions) { const subQueryDimensionsByCube = R.groupBy(d => this.cubeEvaluator.cubeNameFromPath(d), subQueryDimensions); - const joins = join.joins.map( + const joins = join.joins.flatMap( j => { const [cubeSql, cubeAlias, conditions] = this.rewriteInlineCubeSql(j.originalTo, true); return [{ @@ -2200,7 +2216,7 @@ export class BaseQuery { // TODO handle the case when sub query referenced by a foreign cube on other side of a join }].concat((subQueryDimensionsByCube[j.originalTo] || []).map(d => this.subQueryJoin(d))); } - ).reduce((a, b) => a.concat(b), []); + ); const [cubeSql, cubeAlias] = this.rewriteInlineCubeSql(join.root); @@ -2212,6 +2228,10 @@ export class BaseQuery { ]); } + /** + * @param {JoinChain} toJoin + * @returns {string} + */ joinSql(toJoin) { const [root, ...rest] = toJoin; const joins = rest.map( @@ -2273,6 +2293,11 @@ export class BaseQuery { return this.filtersWithoutSubQueriesValue; } + /** + * + * @param {string} dimension + * @returns {{ prefix: string, subQuery: this, cubeName: string }} + */ subQueryDescription(dimension) { const symbol = this.cubeEvaluator.dimensionByPath(dimension); const [cubeName, name] = this.cubeEvaluator.parsePath('dimensions', dimension); @@ -2317,6 +2342,12 @@ export class BaseQuery { return { prefix, subQuery, cubeName }; } + /** + * + * @param {string} cubeName + * @param {string} name + * @returns {string} + */ subQueryName(cubeName, name) { return `${cubeName}_${name}_subquery`; } @@ -2520,6 +2551,9 @@ export class BaseQuery { ); } + /** + * @param {string} cube + */ cubeSql(cube) { const foundPreAggregation = this.preAggregations.findPreAggregationToUseForCube(cube); if (foundPreAggregation && @@ -2630,6 +2664,13 @@ export class BaseQuery { ]; } + /** + * @template T + * @param {boolean} excludeTimeDimensions + * @param {(t: () => void) => T} fn + * @param {string | Array} methodName + * @returns {T} + */ collectFromMembers(excludeTimeDimensions, fn, methodName) { const membersToCollectFrom = this.allMembersConcat(excludeTimeDimensions) .concat(this.join ? this.join.joins.map(j => ({ @@ -2656,6 +2697,14 @@ export class BaseQuery { .concat(excludeTimeDimensions ? [] : this.timeDimensions); } + /** + * @template T + * @param {Array} membersToCollectFrom + * @param {(t: () => void) => T} fn + * @param {string | Array} methodName + * @param {unknown} [cache] + * @returns {T} + */ collectFrom(membersToCollectFrom, fn, methodName, cache) { const methodCacheKey = Array.isArray(methodName) ? methodName : [methodName]; return R.pipe( @@ -2677,6 +2726,11 @@ export class BaseQuery { ); } + /** + * + * @param {() => void} fn + * @returns {Array} + */ collectSubQueryDimensionsFor(fn) { const context = { subQueryDimensions: [] }; this.evaluateSymbolSqlWithContext( @@ -3239,6 +3293,11 @@ export class BaseQuery { return strings.join(' || '); } + /** + * + * @param {string} cubeName + * @returns {Array} + */ primaryKeyNames(cubeName) { const primaryKeys = this.cubeEvaluator.primaryKeys[cubeName]; if (!primaryKeys || !primaryKeys.length) { @@ -3374,6 +3433,12 @@ export class BaseQuery { )(context.leafMeasures); } + /** + * @template T + * @param {() => T} fn + * @param {unknown} context + * @returns {T} + */ evaluateSymbolSqlWithContext(fn, context) { const oldContext = this.evaluateSymbolContext; this.evaluateSymbolContext = oldContext ? Object.assign({}, oldContext, context) : context; @@ -3596,6 +3661,11 @@ export class BaseQuery { .map(s => `(${s})`).join(' AND '); } + /** + * @param {string} primaryKeyName + * @param {string} cubeName + * @returns {unknown} + */ primaryKeySql(primaryKeyName, cubeName) { const primaryKeyDimension = this.cubeEvaluator.dimensionByPath([cubeName, primaryKeyName]); return this.evaluateSymbolSql( @@ -3745,7 +3815,7 @@ export class BaseQuery { /** * * @param options - * @returns {BaseQuery} + * @returns {this} */ newSubQuery(options) { const QueryClass = this.constructor; diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts index 854eb8da1c8f2..18343505bdf08 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts @@ -81,17 +81,23 @@ export class BaseSegment { return this.segmentDefinition().sql; } - public path() { + public path(): string[] | null { if (this.expression) { return null; } return this.query.cubeEvaluator.parsePath('segments', this.segment); } - public expressionPath() { + public expressionPath(): string { if (this.expression) { return `expr:${this.expression.expressionName}`; } - return this.query.cubeEvaluator.pathFromArray(this.path() as string[]); + + const path = this.path(); + if (path === null) { + // Sanity check, this should not actually happen because we checked this.expression earlier + throw new Error('Unexpected null path'); + } + return this.query.cubeEvaluator.pathFromArray(path); } } diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index c856f675d2c83..3da6eb54d3004 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1,10 +1,10 @@ import R from 'ramda'; -import { CubeSymbols } from '../compiler/CubeSymbols'; +import { CubeSymbols, PreAggregationDefinition } from '../compiler/CubeSymbols'; import { UserError } from '../compiler/UserError'; import { BaseQuery } from './BaseQuery'; import { - PreAggregationDefinition, PreAggregationDefinitions, + PreAggregationDefinitions, PreAggregationReferences, PreAggregationTimeDimensionReference } from '../compiler/CubeEvaluator'; @@ -49,6 +49,10 @@ export type PreAggregationForQuery = { sqlAlias?: string; }; +export type PreAggregationForQueryWithTableName = PreAggregationForQuery & { + tableName: string; +}; + export type PreAggregationForCube = { preAggregationName: string; cube: string; @@ -125,7 +129,7 @@ export class PreAggregations { !isInPreAggregationQuery || isInPreAggregationQuery && this.query.options.useOriginalSqlPreAggregationsInPreAggregation) { return R.pipe( - R.map(cube => { + R.map((cube: string) => { const { preAggregations } = this.collectOriginalSqlPreAggregations(() => this.query.cubeSql(cube)); return R.unnest(preAggregations.map(p => this.preAggregationDescriptionsFor(p))); }), @@ -138,6 +142,10 @@ export class PreAggregations { private preAggregationCubes(): string[] { const { join } = this.query; + if (!join) { + // This can happen with Tesseract, or when there's no cubes to join + throw new Error('Unexpected missing join tree for query'); + } return join.joins.map(j => j.originalTo).concat([join.root]); } @@ -767,7 +775,7 @@ export class PreAggregations { if (td[1] === '*') { return R.any((tdtc: [string, string]) => tdtc[0] === td[0]); // need to match the dimension at least } else { - return R.contains(td); + return R.includes(td); } })) ) @@ -945,14 +953,12 @@ export class PreAggregations { private findRollupPreAggregationsForCube(cube: string, canUsePreAggregation: CanUsePreAggregationFn, preAggregations: PreAggregationDefinitions): PreAggregationForQuery[] { return R.pipe( R.toPairs, - // eslint-disable-next-line no-unused-vars - // eslint-disable-next-line @typescript-eslint/no-unused-vars - R.filter(([k, a]) => a.type === 'rollup' || a.type === 'rollupJoin' || a.type === 'rollupLambda'), + R.filter(([_k, a]) => a.type === 'rollup' || a.type === 'rollupJoin' || a.type === 'rollupLambda'), R.map(([preAggregationName, preAggregation]) => this.evaluatedPreAggregationObj(cube, preAggregationName, preAggregation, canUsePreAggregation)) )(preAggregations); } - public getRollupPreAggregationByName(cube, preAggregationName) { + public getRollupPreAggregationByName(cube, preAggregationName): PreAggregationForQueryWithTableName | {} { const canUsePreAggregation = () => true; const preAggregation = R.pipe( R.toPairs, @@ -1075,7 +1081,7 @@ export class PreAggregations { return this.evaluatedPreAggregationObj( joinCube, joinPreAggregationName, - this.query.cubeEvaluator.byPath('preAggregations', name), + this.query.cubeEvaluator.byPath('preAggregations', name) as PreAggregationDefinitionExtended, canUsePreAggregation ); } @@ -1093,7 +1099,7 @@ export class PreAggregations { return this.evaluatedPreAggregationObj( referencedCube, referencedPreAggregation, - this.query.cubeEvaluator.byPath('preAggregations', name), + this.query.cubeEvaluator.byPath('preAggregations', name) as PreAggregationDefinitionExtended, canUsePreAggregation ); } @@ -1311,7 +1317,7 @@ export class PreAggregations { if (aggregation.type === 'rollupLambda') { if (references.rollups.length > 0) { const [firstLambdaCube] = this.query.cubeEvaluator.parsePath('preAggregations', references.rollups[0]); - const firstLambdaPreAggregation = this.query.cubeEvaluator.byPath('preAggregations', references.rollups[0]); + const firstLambdaPreAggregation = this.query.cubeEvaluator.byPath('preAggregations', references.rollups[0]) as PreAggregationDefinitionExtended; const firstLambdaReferences = this.query.cubeEvaluator.evaluatePreAggregationReferences(firstLambdaCube, firstLambdaPreAggregation); if (references.measures.length === 0 && diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 7bc21e26bb455..3d2e2e3a993ea 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -1,30 +1,36 @@ /* eslint-disable no-restricted-syntax */ import R from 'ramda'; -import { CubeSymbols, type ToString } from './CubeSymbols'; +import { + CubeDefinitionExtended, + CubeSymbols, + HierarchyDefinition, JoinDefinition, + PreAggregationDefinition, PreAggregationDefinitionRollup, + type ToString +} from './CubeSymbols'; import { UserError } from './UserError'; -import { BaseQuery } from '../adapter'; +import { BaseQuery, PreAggregationDefinitionExtended } from '../adapter'; import type { CubeValidator } from './CubeValidator'; import type { ErrorReporter } from './ErrorReporter'; export type SegmentDefinition = { - type: string, - sql: Function, - primaryKey?: true, - ownedByCube: boolean, - fieldType?: string, + type: string; + sql(): string; + primaryKey?: true; + ownedByCube: boolean; + fieldType?: string; // TODO should we have it here? - multiStage?: boolean, + multiStage?: boolean; }; export type DimensionDefinition = { - type: string, - sql: Function, - primaryKey?: true, - ownedByCube: boolean, - fieldType?: string, - multiStage?: boolean, - shiftInterval?: string, + type: string; + sql(): string; + primaryKey?: true; + ownedByCube: boolean; + fieldType?: string; + multiStage?: boolean; + shiftInterval?: string; }; export type TimeShiftDefinition = { @@ -42,72 +48,30 @@ export type TimeShiftDefinitionReference = { }; export type MeasureDefinition = { - type: string, - sql: Function, - ownedByCube: boolean, + type: string; + sql(): string; + ownedByCube: boolean; rollingWindow?: any filters?: any - primaryKey?: true, - drillFilters?: any, - multiStage?: boolean, - groupBy?: (...args: Array) => Array, - reduceBy?: (...args: Array) => Array, - addGroupBy?: (...args: Array) => Array, - timeShift?: TimeShiftDefinition[], - groupByReferences?: string[], - reduceByReferences?: string[], - addGroupByReferences?: string[], - timeShiftReferences?: TimeShiftDefinitionReference[], - patchedFrom?: { cubeName: string, name: string }, + primaryKey?: true; + drillFilters?: any; + multiStage?: boolean; + groupBy?: (...args: Array) => Array; + reduceBy?: (...args: Array) => Array; + addGroupBy?: (...args: Array) => Array; + timeShift?: TimeShiftDefinition[]; + groupByReferences?: string[]; + reduceByReferences?: string[]; + addGroupByReferences?: string[]; + timeShiftReferences?: TimeShiftDefinitionReference[]; + patchedFrom?: { cubeName: string; name: string }; }; export type PreAggregationFilters = { - dataSources?: string[], - cubes?: string[], - preAggregationIds?: string[], - scheduled?: boolean, -}; - -export type EveryInterval = string; -type EveryCronInterval = string; -type EveryCronTimeZone = string; - -export type CubeRefreshKeySqlVariant = { - sql: () => string; - every?: EveryInterval; -}; - -export type CubeRefreshKeyEveryVariant = { - every: EveryInterval | EveryCronInterval; - timezone?: EveryCronTimeZone; - incremental?: boolean; - updateWindow?: EveryInterval; -}; - -export type CubeRefreshKeyImmutableVariant = { - immutable: true; -}; - -export type CubeRefreshKey = - | CubeRefreshKeySqlVariant - | CubeRefreshKeyEveryVariant - | CubeRefreshKeyImmutableVariant; - -export type PreAggregationDefinition = { - type: 'autoRollup' | 'originalSql' | 'rollupJoin' | 'rollupLambda' | 'rollup', - allowNonStrictDateRangeMatch?: boolean, - useOriginalSqlPreAggregations?: boolean, - timeDimensionReference?: () => ToString, - granularity: string, - timeDimensionReferences: Array<{ dimension: () => ToString, granularity: string }>, - dimensionReferences: () => Array, - segmentReferences: () => Array, - measureReferences: () => Array, - rollupReferences: () => Array, - indexes?: Record, - refreshKey?: CubeRefreshKey, - scheduledRefresh: boolean, - external: boolean, + dataSources?: string[]; + cubes?: string[]; + preAggregationIds?: string[]; + scheduled?: boolean; }; export type PreAggregationDefinitions = Record; @@ -137,12 +101,67 @@ export type PreAggregationInfo = { indexesReferences: unknown, }; +export type EvaluatedHierarchy = { + name: string; + title?: string; + public?: boolean; + levels: string[]; + aliasMember?: string; + [key: string]: any; +}; + +export type Filter = + | { + member: string; + memberReference?: string; + [key: string]: any; + } + | { + and?: Filter[]; + or?: Filter[]; + [key: string]: any; + }; + +export type AccessPolicy = { + rowLevel?: { + filters: Filter[]; + }; + memberLevel?: { + includes?: string | string[]; + excludes?: string | string[]; + includesMembers?: string[]; + excludesMembers?: string[]; + }; +}; + +export type EvaluatedFolder = { + name: string; + includes: (EvaluatedFolder | DimensionDefinition | MeasureDefinition)[]; + type: 'folder'; + [key: string]: any; +}; + +export type EvaluatedCube = { + measures: Record; + dimensions: Record; + segments: Record; + joins: Record; + hierarchies: Record; + evaluatedHierarchies: EvaluatedHierarchy[]; + preAggregations: Record; + dataSource?: string; + folders: EvaluatedFolder[]; + sql?: (...args: any[]) => string; + sqlTable?: (...args: any[]) => string; + accessPolicy?: AccessPolicy[]; +}; + export class CubeEvaluator extends CubeSymbols { - public evaluatedCubes: Record = {}; + public evaluatedCubes: Record = {}; - public primaryKeys: Record = {}; + public primaryKeys: Record = {}; - public byFileName: Record = {}; + public byFileName: Record = {}; private isRbacEnabledCache: boolean | null = null; @@ -168,7 +187,7 @@ export class CubeEvaluator extends CubeSymbols { this.evaluatedCubes[cube.name] = this.prepareCube(cube, errorReporter); } - this.byFileName = R.groupBy(v => v.fileName, validCubes); + this.byFileName = R.groupBy(v => v.fileName || v.name, validCubes); this.primaryKeys = R.fromPairs( validCubes.map((v) => { const primaryKeyNamesToSymbols = R.compose( @@ -559,7 +578,7 @@ export class CubeEvaluator extends CubeSymbols { return this.byFileName[fileName] || []; } - public timeDimensionPathsForCube(cube: any) { + public timeDimensionPathsForCube(cube: string): string[] { return R.compose( R.map(dimName => `${cube}.${dimName}`), R.keys, @@ -569,18 +588,18 @@ export class CubeEvaluator extends CubeSymbols { )(this.evaluatedCubes[cube].dimensions || {}); } - public measuresForCube(cube) { + public measuresForCube(cube: string): Record { return this.cubeFromPath(cube).measures || {}; } - public timeDimensionsForCube(cube) { + public timeDimensionsForCube(cube: string): Record { return R.filter( (d: any) => d.type === 'time', this.cubeFromPath(cube).dimensions || {} ); } - public preAggregationsForCube(path: string): Record { + public preAggregationsForCube(path: string): Record { return this.cubeFromPath(path).preAggregations || {}; } @@ -689,22 +708,22 @@ export class CubeEvaluator extends CubeSymbols { } public measureByPath(measurePath: string): MeasureDefinition { - return this.byPath('measures', measurePath); + return this.byPath('measures', measurePath) as MeasureDefinition; } public dimensionByPath(dimensionPath: string): DimensionDefinition { - return this.byPath('dimensions', dimensionPath); + return this.byPath('dimensions', dimensionPath) as DimensionDefinition; } public segmentByPath(segmentPath: string): SegmentDefinition { - return this.byPath('segments', segmentPath); + return this.byPath('segments', segmentPath) as SegmentDefinition; } - public cubeExists(cube) { + public cubeExists(cube: string): boolean { return !!this.evaluatedCubes[cube]; } - public cubeFromPath(path: string) { + public cubeFromPath(path: string): EvaluatedCube { return this.evaluatedCubes[this.cubeNameFromPath(path)]; } @@ -738,7 +757,7 @@ export class CubeEvaluator extends CubeSymbols { throw new UserError(`Can't resolve member '${Array.isArray(path) ? path.join('.') : path}'`); } - public byPath(type: 'measures' | 'dimensions' | 'segments' | 'preAggregations', path: string | string[]) { + public byPath(type: T, path: string | string[]): EvaluatedCube[T][string] { if (!type) { throw new Error(`Type can't be undefined for '${path}'`); } @@ -748,19 +767,22 @@ export class CubeEvaluator extends CubeSymbols { } const cubeAndName = Array.isArray(path) ? path : path.split('.'); - if (!this.evaluatedCubes[cubeAndName[0]]) { + const cube = this.evaluatedCubes[cubeAndName[0]]; + if (cube === undefined) { throw new UserError(`Cube '${cubeAndName[0]}' not found for path '${path}'`); } - if (!this.evaluatedCubes[cubeAndName[0]][type]) { + const typeMembers = cube[type]; + if (typeMembers === undefined) { throw new UserError(`${type} not defined for path '${path}'`); } - if (!this.evaluatedCubes[cubeAndName[0]][type][cubeAndName[1]]) { + const member = typeMembers[cubeAndName[1]]; + if (member === undefined) { throw new UserError(`'${cubeAndName[1]}' not found for path '${path}'`); } - return this.evaluatedCubes[cubeAndName[0]][type][cubeAndName[1]]; + return member as EvaluatedCube[T][string]; } public parsePath(type: 'measures' | 'dimensions' | 'segments' | 'preAggregations', path: string): string[] { @@ -829,7 +851,7 @@ export class CubeEvaluator extends CubeSymbols { return this.evaluateReferences(cube, rollupReferences, { originalSorting: true }); } - public evaluatePreAggregationReferences(cube: string, aggregation: PreAggregationDefinition): PreAggregationReferences { + public evaluatePreAggregationReferences(cube: string, aggregation: PreAggregationDefinitionRollup): PreAggregationReferences { const timeDimensions: Array = []; if (aggregation.timeDimensionReference) { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index c1f207a099216..c7acc1ab20304 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -10,21 +10,116 @@ import type { ErrorReporter } from './ErrorReporter'; export type ToString = { toString(): string }; -interface CubeDefinition { +export type GranularityDefinition = { + sql?: (...args: any[]) => string; + title?: string; + interval?: string; + offset?: string; + origin?: string; +}; + +export type TimeshiftDefinition = { + interval?: string; + type?: string; + name?: string; + timeDimension?: (...args: any[]) => string; +}; + +export type CubeSymbolDefinition = { + type?: string; + sql?: (...args: any[]) => string; + primaryKey?: boolean; + granularities?: Record; + timeShift?: TimeshiftDefinition[]; + format?: string; +}; + +export type HierarchyDefinition = { + title?: string; + public?: boolean; + levels?: (...args: any[]) => string[]; +}; + +export type EveryInterval = string; +type EveryCronInterval = string; +type EveryCronTimeZone = string; + +export type CubeRefreshKeySqlVariant = { + sql: () => string; + every?: EveryInterval; +}; + +export type CubeRefreshKeyEveryVariant = { + every: EveryInterval | EveryCronInterval; + timezone?: EveryCronTimeZone; + incremental?: boolean; + updateWindow?: EveryInterval; +}; + +export type CubeRefreshKeyImmutableVariant = { + immutable: true; +}; + +export type CubeRefreshKey = + | CubeRefreshKeySqlVariant + | CubeRefreshKeyEveryVariant + | CubeRefreshKeyImmutableVariant; + +type BasePreAggregationDefinition = { + allowNonStrictDateRangeMatch?: boolean; + useOriginalSqlPreAggregations?: boolean; + timeDimensionReference?: (...args: any[]) => ToString; + indexes?: Record; + refreshKey?: CubeRefreshKey; + ownedByCube?: boolean; +}; + +export type PreAggregationDefinitionOriginalSql = BasePreAggregationDefinition & { + type: 'originalSql'; + partitionGranularity?: string; + // eslint-disable-next-line camelcase + partition_granularity?: string; + // eslint-disable-next-line camelcase + time_dimension?: (...args: any[]) => ToString; +}; + +export type PreAggregationDefinitionRollup = BasePreAggregationDefinition & { + type: 'autoRollup' | 'rollupJoin' | 'rollupLambda' | 'rollup'; + granularity: string; + timeDimensionReferences: Array<{ dimension: () => ToString; granularity: string }>; + dimensionReferences: (...args: any[]) => ToString[]; + segmentReferences: (...args: any[]) => ToString[]; + measureReferences: (...args: any[]) => ToString[]; + rollupReferences: (...args: any[]) => ToString[]; + scheduledRefresh: boolean; + external: boolean; +}; + +// PreAggregationDefinition is widely used in the codebase, but it's assumed to be rollup, +// originalSql is not refreshed and so on. +export type PreAggregationDefinition = PreAggregationDefinitionRollup; + +export type JoinDefinition = { + relationship: string, + sql: (...args: any[]) => string, +}; + +export interface CubeDefinition { name: string; extends?: (...args: Array) => { __cubeName: string }; - sql?: string | (() => string); + sql?: string | ((...args: any[]) => string); // eslint-disable-next-line camelcase - sql_table?: string | (() => string); - sqlTable?: string | (() => string); - measures?: Record; - dimensions?: Record; - segments?: Record; - hierarchies?: Record; - preAggregations?: Record; + sql_table?: string | ((...args: any[]) => string); + sqlTable?: string | ((...args: any[]) => string); + dataSource?: string; + measures?: Record; + dimensions?: Record; + segments?: Record; + hierarchies?: Record; + preAggregations?: Record; // eslint-disable-next-line camelcase - pre_aggregations?: Record; - joins?: Record; + pre_aggregations?: Record; + joins?: Record; accessPolicy?: any[]; folders?: any[]; includes?: any; @@ -34,9 +129,10 @@ interface CubeDefinition { calendar?: boolean; isSplitView?: boolean; includedMembers?: any[]; + fileName?: string; } -interface CubeDefinitionExtended extends CubeDefinition { +export interface CubeDefinitionExtended extends CubeDefinition { allDefinitions: (type: string) => Record; rawFolders: () => any[]; rawCubes: () => any[]; @@ -46,6 +142,13 @@ interface SplitViews { [key: string]: any; } +export interface CubeSymbolsBase { + cubeName: () => string; + cubeObj: () => CubeDefinitionExtended; +} + +export type CubeSymbolsDefinition = CubeSymbolsBase & Record; + const FunctionRegex = /function\s+\w+\(([A-Za-z0-9_,]*)|\(([\s\S]*?)\)\s*=>|\(?(\w+)\)?\s*=>/; export const CONTEXT_SYMBOLS = { SECURITY_CONTEXT: 'securityContext', @@ -61,15 +164,15 @@ export const CONTEXT_SYMBOLS = { export const CURRENT_CUBE_CONSTANTS = ['CUBE', 'TABLE']; export class CubeSymbols { - public symbols: Record; + public symbols: Record; - private builtCubes: Record; + private builtCubes: Record; private cubeDefinitions: Record; private funcArgumentsValues: Record; - public cubeList: any[]; + public cubeList: CubeDefinitionExtended[]; private readonly evaluateViews: boolean; @@ -85,13 +188,10 @@ export class CubeSymbols { } public compile(cubes: CubeDefinition[], errorReporter: ErrorReporter) { - // @ts-ignore - this.cubeDefinitions = R.pipe( - // @ts-ignore - R.map((c: CubeDefinition) => [c.name, c]), - R.fromPairs - // @ts-ignore - )(cubes); + this.cubeDefinitions = Object.fromEntries( + cubes.map((c): [string, CubeDefinition] => [c.name, c]) + ); + this.cubeList = cubes.map(c => (c.name ? this.getCubeDefinition(c.name) : this.createCube(c))); // TODO support actual dependency sorting to allow using views inside views const sortedByDependency = R.pipe( @@ -815,7 +915,7 @@ export class CubeSymbols { } } - protected funcArguments(func) { + public funcArguments(func: Function): string[] { const funcDefinition = func.toString(); if (!this.funcArgumentsValues[funcDefinition]) { const match = funcDefinition.match(FunctionRegex); @@ -1017,7 +1117,7 @@ export class CubeSymbols { ), }; } - if (cube[propertyName]) { + if (cube[propertyName as string]) { return this.cubeReferenceProxy(cubeName, joinHints, propertyName); } if (self.symbols[propertyName]) { @@ -1080,9 +1180,9 @@ export class CubeSymbols { if (propertyName === '_objectWithResolvedProperties') { return true; } - if (cube[propertyName]) { + if (cube[propertyName as string]) { const index = depsResolveFn(propertyName, parentIndex); - if (cube[propertyName].type === 'time') { + if (cube[propertyName as string].type === 'time') { return this.timeDimDependenciesProxy(index); } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js index b6a706feddaa3..740da78e43576 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js +++ b/packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js @@ -172,7 +172,7 @@ export class CubeToMetaTransformer { // As for now context works on the cubes level return R.filter( - (query) => R.contains(query.config.name, context.contextMembers) + (query) => R.includes(query.config.name, context.contextMembers) )(this.queries); } diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index f9af9edab58f3..32540d4864ac6 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -1,7 +1,7 @@ import Joi from 'joi'; import cronParser from 'cron-parser'; -import type { CubeSymbols } from './CubeSymbols'; +import type { CubeSymbols, CubeDefinition } from './CubeSymbols'; import type { ErrorReporter } from './ErrorReporter'; /* ***************************** @@ -952,13 +952,13 @@ export class CubeValidator { if (result.error != null) { errorReporter.error(formatErrorMessage(result.error), result.error); } else { - this.validCubes[cube.name] = true; + this.validCubes.set(cube.name, true); } return result; } - public isCubeValid(cube) { - return this.validCubes[cube.name] || cube.isSplitView; + public isCubeValid(cube: CubeDefinition): boolean { + return this.validCubes.get(cube.name) ?? cube.isSplitView ?? false; } } diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js deleted file mode 100644 index 179ca2ac5ceba..0000000000000 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.js +++ /dev/null @@ -1,240 +0,0 @@ -import R from 'ramda'; -import Graph from 'node-dijkstra'; -import { UserError } from './UserError'; - -export class JoinGraph { - /** - * @param {import('./CubeValidator').CubeValidator} cubeValidator - * @param {import('./CubeEvaluator').CubeEvaluator} cubeEvaluator - */ - constructor(cubeValidator, cubeEvaluator) { - this.cubeValidator = cubeValidator; - this.cubeEvaluator = cubeEvaluator; - this.nodes = {}; - this.edges = {}; - this.builtJoins = {}; - } - - compile(cubes, errorReporter) { - this.edges = R.compose( - R.fromPairs, - R.unnest, - R.map(v => this.buildJoinEdges(v, errorReporter.inContext(`${v.name} cube`))), - R.filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator)) - )(this.cubeEvaluator.cubeList); - this.nodes = R.compose( - R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.to, 1]))), - R.groupBy(join => join.from), - R.map(v => v[1]), - R.toPairs - )(this.edges); - this.undirectedNodes = R.compose( - R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.from, 1]))), - R.groupBy(join => join.to), - R.unnest, - R.map(v => [v[1], { from: v[1].to, to: v[1].from }]), - R.toPairs - )(this.edges); - this.graph = new Graph(this.nodes); - } - - buildJoinEdges(cube, errorReporter) { - return R.compose( - R.filter(R.identity), - R.map(join => { - const multipliedMeasures = R.compose( - R.filter( - m => m.sql && this.cubeEvaluator.funcArguments(m.sql).length === 0 && m.sql() === 'count(*)' || - ['sum', 'avg', 'count', 'number'].indexOf(m.type) !== -1 - ), - R.values - ); - const joinRequired = - (v) => `primary key for '${v}' is required when join is defined in order to make aggregates work properly`; - if ( - !this.cubeEvaluator.primaryKeys[join[1].from].length && - multipliedMeasures(this.cubeEvaluator.measuresForCube(join[1].from)).length > 0 - ) { - errorReporter.error(joinRequired(join[1].from)); - return null; - } - if (!this.cubeEvaluator.primaryKeys[join[1].to].length && - multipliedMeasures(this.cubeEvaluator.measuresForCube(join[1].to)).length > 0) { - errorReporter.error(joinRequired(join[1].to)); - return null; - } - return join; - }), - R.unnest, - R.map(join => [ - [`${cube.name}-${join[0]}`, { - join: join[1], - from: cube.name, - to: join[0], - originalFrom: cube.name, - originalTo: join[0] - }] - ]), - R.filter(R.identity), - R.map(join => { - if (!this.cubeEvaluator.cubeExists(join[0])) { - errorReporter.error(`Cube ${join[0]} doesn't exist`); - return undefined; - } - return join; - }), - R.toPairs - )(cube.joins || {}); - } - - buildJoinNode(cube) { - return R.compose( - R.fromPairs, - R.map(v => [v[0], 1]), - R.toPairs - )(cube.joins || {}); - } - - buildJoin(cubesToJoin) { - if (!cubesToJoin.length) { - return null; - } - const key = JSON.stringify(cubesToJoin); - if (!this.builtJoins[key]) { - const join = R.pipe( - R.map( - cube => this.buildJoinTreeForRoot(cube, R.without([cube], cubesToJoin)) - ), - R.filter(R.identity), - R.sortBy(joinTree => joinTree.joins.length) - )(cubesToJoin)[0]; - if (!join) { - throw new UserError(`Can't find join path to join ${cubesToJoin.map(v => `'${v}'`).join(', ')}`); - } - this.builtJoins[key] = Object.assign(join, { - multiplicationFactor: R.compose( - R.fromPairs, - R.map(v => [this.cubeFromPath(v), this.findMultiplicationFactorFor(this.cubeFromPath(v), join.joins)]) - )(cubesToJoin) - }); - } - return this.builtJoins[key]; - } - - cubeFromPath(cubePath) { - if (Array.isArray(cubePath)) { - return cubePath[cubePath.length - 1]; - } - return cubePath; - } - - buildJoinTreeForRoot(root, cubesToJoin) { - const self = this; - if (Array.isArray(root)) { - const [newRoot, ...additionalToJoin] = root; - if (additionalToJoin.length > 0) { - cubesToJoin = [additionalToJoin].concat(cubesToJoin); - } - root = newRoot; - } - const nodesJoined = {}; - const result = cubesToJoin.map(joinHints => { - if (!Array.isArray(joinHints)) { - joinHints = [joinHints]; - } - let prevNode = root; - return joinHints.filter(toJoin => toJoin !== prevNode).map(toJoin => { - if (nodesJoined[toJoin]) { - prevNode = toJoin; - return { joins: [] }; - } - const path = this.graph.path(prevNode, toJoin); - if (!path) { - return null; - } - const foundJoins = self.joinsByPath(path); - prevNode = toJoin; - nodesJoined[toJoin] = true; - return { cubes: path, joins: foundJoins }; - }); - }).reduce((a, b) => a.concat(b), []).reduce((joined, res) => { - if (!res || !joined) { - return null; - } - const indexedPairs = R.compose( - R.addIndex(R.map)((j, i) => [i + joined.joins.length, j]) - ); - return { - joins: joined.joins.concat(indexedPairs(res.joins)) - }; - }, { joins: [] }); - - if (!result) { - return null; - } - - const pairsSortedByIndex = - R.compose(R.uniq, R.map(indexToJoin => indexToJoin[1]), R.sortBy(indexToJoin => indexToJoin[0])); - return { - joins: pairsSortedByIndex(result.joins), - root - }; - } - - findMultiplicationFactorFor(cube, joins) { - const visited = {}; - const self = this; - function findIfMultipliedRecursive(currentCube) { - if (visited[currentCube]) { - return false; - } - visited[currentCube] = true; - function nextNode(nextJoin) { - return nextJoin.from === currentCube ? nextJoin.to : nextJoin.from; - } - const nextJoins = joins.filter(j => j.from === currentCube || j.to === currentCube); - if (nextJoins.find( - nextJoin => self.checkIfCubeMultiplied(currentCube, nextJoin) && !visited[nextNode(nextJoin)] - )) { - return true; - } - return !!nextJoins.find( - nextJoin => findIfMultipliedRecursive(nextNode(nextJoin)) - ); - } - return findIfMultipliedRecursive(cube); - } - - checkIfCubeMultiplied(cube, join) { - return join.from === cube && join.join.relationship === 'hasMany' || - join.to === cube && join.join.relationship === 'belongsTo'; - } - - joinsByPath(path) { - return R.range(0, path.length - 1).map(i => this.edges[`${path[i]}-${path[i + 1]}`]); - } - - connectedComponents() { - if (!this.cachedConnectedComponents) { - let componentId = 1; - const components = {}; - R.toPairs(this.nodes).map(nameToConnection => nameToConnection[0]).forEach(node => { - this.findConnectedComponent(componentId, node, components); - componentId += 1; - }); - this.cachedConnectedComponents = components; - } - return this.cachedConnectedComponents; - } - - findConnectedComponent(componentId, node, components) { - if (!components[node]) { - components[node] = componentId; - R.toPairs(this.undirectedNodes[node]) - .map(connectedNodeNames => connectedNodeNames[0]) - .forEach(connectedNode => { - this.findConnectedComponent(componentId, connectedNode, components); - }); - } - } -} diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts new file mode 100644 index 0000000000000..179450a0cfed9 --- /dev/null +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -0,0 +1,361 @@ +import R from 'ramda'; +import Graph from 'node-dijkstra'; +import { UserError } from './UserError'; + +import type { CubeValidator } from './CubeValidator'; +import type { CubeEvaluator, MeasureDefinition } from './CubeEvaluator'; +import type { CubeDefinition, JoinDefinition } from './CubeSymbols'; +import type { ErrorReporter } from './ErrorReporter'; + +type JoinEdge = { + join: JoinDefinition, + from: string, + to: string, + originalFrom: string, + originalTo: string, +}; + +type JoinTreeJoins = JoinEdge[]; + +type JoinTree = { + root: string, + joins: JoinTreeJoins, +}; + +export type FinishedJoinTree = JoinTree & { + multiplicationFactor: Record, +}; + +export type JoinHint = string | string[]; + +export type JoinHints = JoinHint[]; + +export class JoinGraph { + private readonly cubeValidator: CubeValidator; + + private readonly cubeEvaluator: CubeEvaluator; + + // source node -> destination node -> weight + private nodes: Record>; + + // source node -> destination node -> weight + private undirectedNodes: Record>; + + private edges: Record; + + private builtJoins: Record; + + private graph: Graph | null; + + private cachedConnectedComponents: Record | null; + + public constructor(cubeValidator: CubeValidator, cubeEvaluator: CubeEvaluator) { + this.cubeValidator = cubeValidator; + this.cubeEvaluator = cubeEvaluator; + this.nodes = {}; + this.undirectedNodes = {}; + this.edges = {}; + this.builtJoins = {}; + this.cachedConnectedComponents = null; + this.graph = null; + } + + public compile(cubes: unknown, errorReporter: ErrorReporter): void { + this.edges = R.compose< + Array, + Array, + Array<[string, JoinEdge][]>, + Array<[string, JoinEdge]>, + Record + >( + R.fromPairs, + R.unnest, + R.map((v: CubeDefinition): [string, JoinEdge][] => this.buildJoinEdges(v, errorReporter.inContext(`${v.name} cube`))), + R.filter(this.cubeValidator.isCubeValid.bind(this.cubeValidator)) + )(this.cubeEvaluator.cubeList); + + // This requires @types/ramda@0.29 or newer + // @ts-ignore + this.nodes = R.compose< + Record, + Array<[string, JoinEdge]>, + Array, + Record | undefined>, + Record> + >( + // This requires @types/ramda@0.29 or newer + // @ts-ignore + R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.to, 1]))), + R.groupBy((join: JoinEdge) => join.from), + R.map(v => v[1]), + R.toPairs + // @ts-ignore + )(this.edges); + + // @ts-ignore + this.undirectedNodes = R.compose( + // @ts-ignore + R.map(groupedByFrom => R.fromPairs(groupedByFrom.map(join => [join.from, 1]))), + // @ts-ignore + R.groupBy(join => join.to), + R.unnest, + // @ts-ignore + R.map(v => [v[1], { from: v[1].to, to: v[1].from }]), + R.toPairs + // @ts-ignore + )(this.edges); + + this.graph = new Graph(this.nodes); + } + + protected buildJoinEdges(cube: CubeDefinition, errorReporter: ErrorReporter): Array<[string, JoinEdge]> { + // @ts-ignore + return R.compose( + // @ts-ignore + R.filter(R.identity), + R.map((join: [string, JoinEdge]) => { + const multipliedMeasures: ((m: Record) => MeasureDefinition[]) = R.compose( + R.filter( + (m: MeasureDefinition): boolean => m.sql && this.cubeEvaluator.funcArguments(m.sql).length === 0 && m.sql() === 'count(*)' || + ['sum', 'avg', 'count', 'number'].indexOf(m.type) !== -1 + ), + R.values as (input: Record) => MeasureDefinition[] + ); + const joinRequired = + (v) => `primary key for '${v}' is required when join is defined in order to make aggregates work properly`; + if ( + !this.cubeEvaluator.primaryKeys[join[1].from].length && + multipliedMeasures(this.cubeEvaluator.measuresForCube(join[1].from)).length > 0 + ) { + errorReporter.error(joinRequired(join[1].from)); + return null; + } + if (!this.cubeEvaluator.primaryKeys[join[1].to].length && + multipliedMeasures(this.cubeEvaluator.measuresForCube(join[1].to)).length > 0) { + errorReporter.error(joinRequired(join[1].to)); + return null; + } + return join; + }), + R.unnest, + R.map((join: [string, JoinDefinition]): [[string, JoinEdge]] => [ + [`${cube.name}-${join[0]}`, { + join: join[1], + from: cube.name, + to: join[0], + originalFrom: cube.name, + originalTo: join[0] + }] + ]), + // @ts-ignore + R.filter(R.identity), + R.map((join: [string, JoinDefinition]) => { + if (!this.cubeEvaluator.cubeExists(join[0])) { + errorReporter.error(`Cube ${join[0]} doesn't exist`); + return undefined; + } + return join; + }), + // @ts-ignore + R.toPairs + // @ts-ignore + )(cube.joins || {}); + } + + protected buildJoinNode(cube: CubeDefinition): Record { + return R.compose< + Record, + Array<[string, JoinDefinition]>, + Array<[string, 1]>, + Record + >( + R.fromPairs, + R.map(v => [v[0], 1]), + R.toPairs + )(cube.joins || {}); + } + + public buildJoin(cubesToJoin: JoinHints): FinishedJoinTree | null { + if (!cubesToJoin.length) { + return null; + } + const key = JSON.stringify(cubesToJoin); + if (!this.builtJoins[key]) { + const join = R.pipe< + JoinHints, + Array, + Array, + Array + >( + R.map( + (cube: JoinHint): JoinTree | null => this.buildJoinTreeForRoot(cube, R.without([cube], cubesToJoin)) + ), + // @ts-ignore + R.filter(R.identity), + R.sortBy((joinTree: JoinTree) => joinTree.joins.length) + // @ts-ignore + )(cubesToJoin)[0]; + + if (!join) { + throw new UserError(`Can't find join path to join ${cubesToJoin.map(v => `'${v}'`).join(', ')}`); + } + + this.builtJoins[key] = Object.assign(join, { + multiplicationFactor: R.compose< + JoinHints, + Array<[string, boolean]>, + Record + >( + R.fromPairs, + R.map(v => [this.cubeFromPath(v), this.findMultiplicationFactorFor(this.cubeFromPath(v), join.joins)]) + )(cubesToJoin) + }); + } + return this.builtJoins[key]; + } + + protected cubeFromPath(cubePath) { + if (Array.isArray(cubePath)) { + return cubePath[cubePath.length - 1]; + } + return cubePath; + } + + protected buildJoinTreeForRoot(root: JoinHint, cubesToJoin: JoinHints): JoinTree | null { + const self = this; + + const { graph } = this; + if (graph === null) { + // JoinGraph was not compiled + return null; + } + + if (Array.isArray(root)) { + const [newRoot, ...additionalToJoin] = root; + if (additionalToJoin.length > 0) { + cubesToJoin = [additionalToJoin, ...cubesToJoin]; + } + root = newRoot; + } + const nodesJoined = {}; + const result = cubesToJoin.map(joinHints => { + if (!Array.isArray(joinHints)) { + joinHints = [joinHints]; + } + let prevNode = root; + return joinHints.filter(toJoin => toJoin !== prevNode).map(toJoin => { + if (nodesJoined[toJoin]) { + prevNode = toJoin; + return { joins: [] }; + } + + const path = graph.path(prevNode, toJoin); + if (!path) { + return null; + } + if (!Array.isArray(path)) { + // Unexpected object return from graph, it should do so only when path cost was requested + return null; + } + + const foundJoins = self.joinsByPath(path); + prevNode = toJoin; + nodesJoined[toJoin] = true; + return { cubes: path, joins: foundJoins }; + }); + }).reduce((a, b) => a.concat(b), []) + // @ts-ignore + .reduce((joined, res) => { + if (!res || !joined) { + return null; + } + const indexedPairs = R.compose< + Array, + Array<[number, JoinEdge]> + >( + R.addIndex(R.map)((j, i) => [i + joined.joins.length, j]) + ); + return { + joins: [...joined.joins, ...indexedPairs(res.joins)], + }; + }, { joins: [] }); + + if (!result) { + return null; + } + + const pairsSortedByIndex: (joins: [number, JoinEdge][]) => JoinEdge[] = + R.compose< + Array<[number, JoinEdge]>, + Array<[number, JoinEdge]>, + Array, + Array + >( + R.uniq, + R.map(([_, join]: [number, JoinEdge]) => join), + R.sortBy(([index]: [number, JoinEdge]) => index) + ); + return { + // @ts-ignore + joins: pairsSortedByIndex(result.joins), + root + }; + } + + protected findMultiplicationFactorFor(cube: string, joins: JoinTreeJoins): boolean { + const visited = {}; + const self = this; + function findIfMultipliedRecursive(currentCube: string) { + if (visited[currentCube]) { + return false; + } + visited[currentCube] = true; + function nextNode(nextJoin: JoinEdge): string { + return nextJoin.from === currentCube ? nextJoin.to : nextJoin.from; + } + const nextJoins = joins.filter(j => j.from === currentCube || j.to === currentCube); + if (nextJoins.find( + nextJoin => self.checkIfCubeMultiplied(currentCube, nextJoin) && !visited[nextNode(nextJoin)] + )) { + return true; + } + return !!nextJoins.find( + nextJoin => findIfMultipliedRecursive(nextNode(nextJoin)) + ); + } + return findIfMultipliedRecursive(cube); + } + + protected checkIfCubeMultiplied(cube: string, join: JoinEdge): boolean { + return join.from === cube && join.join.relationship === 'hasMany' || + join.to === cube && join.join.relationship === 'belongsTo'; + } + + protected joinsByPath(path: string[]): JoinEdge[] { + return R.range(0, path.length - 1).map(i => this.edges[`${path[i]}-${path[i + 1]}`]); + } + + public connectedComponents(): Record { + if (!this.cachedConnectedComponents) { + let componentId = 1; + const components = {}; + R.toPairs(this.nodes).map(nameToConnection => nameToConnection[0]).forEach(node => { + this.findConnectedComponent(componentId, node, components); + componentId += 1; + }); + this.cachedConnectedComponents = components; + } + return this.cachedConnectedComponents; + } + + protected findConnectedComponent(componentId: number, node: string, components: Record): void { + if (!components[node]) { + components[node] = componentId; + R.toPairs(this.undirectedNodes[node]) + .map(connectedNodeNames => connectedNodeNames[0]) + .forEach(connectedNode => { + this.findConnectedComponent(componentId, connectedNode, components); + }); + } + } +} diff --git a/packages/cubejs-schema-compiler/src/scaffolding/ScaffoldingSchema.ts b/packages/cubejs-schema-compiler/src/scaffolding/ScaffoldingSchema.ts index 77c7fa42a0e8e..0c91881b2cc57 100644 --- a/packages/cubejs-schema-compiler/src/scaffolding/ScaffoldingSchema.ts +++ b/packages/cubejs-schema-compiler/src/scaffolding/ScaffoldingSchema.ts @@ -32,7 +32,7 @@ export type JoinRelationship = 'hasOne' | 'hasMany' | 'belongsTo'; type ColumnsToJoin = { cubeToJoin: string; columnToJoin: string; - tableName: string; + tableName: TableName; }; export type CubeDescriptorMember = { @@ -112,7 +112,7 @@ export type DatabaseSchema = Record; type TableData = { schema: string, table: string, - tableName: string; + tableName: TableName; tableDefinition: ColumnData[], }; @@ -122,7 +122,7 @@ type ScaffoldingSchemaOptions = { }; export class ScaffoldingSchema { - private tableNamesToTables: { [key: string]: TableData[] } = {}; + private tableNamesToTables: Record = {}; public constructor( private readonly dbSchema: DatabaseSchema, @@ -198,7 +198,7 @@ export class ScaffoldingSchema { tableNames.map(tableName => { const [schema, table] = this.parseTableName(tableName); const tableDefinition = this.resolveTableDefinition(tableName); - const definition = { + const definition: TableData = { schema, table, tableDefinition, tableName }; const tableizeName = inflection.tableize(this.fixCase(table)); @@ -237,7 +237,7 @@ export class ScaffoldingSchema { }; } - protected parseTableName(tableName: TableName) { + protected parseTableName(tableName: TableName): [string, string] { let schemaAndTable; if (Array.isArray(tableName)) { schemaAndTable = tableName; diff --git a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts index 19b8d84ef2c9f..8cb7943d61e12 100644 --- a/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/cube-validator.test.ts @@ -1,5 +1,9 @@ import { CubeValidator, functionFieldsPatterns } from '../../src/compiler/CubeValidator'; -import { CubeSymbols } from '../../src/compiler/CubeSymbols'; +import { + CubeRefreshKey, + CubeSymbols, + PreAggregationDefinitionOriginalSql +} from '../../src/compiler/CubeSymbols'; import { ErrorReporter } from '../../src/compiler/ErrorReporter'; describe('Cube Validation', () => { @@ -696,10 +700,10 @@ describe('Cube Validation', () => { type: 'originalSql', time_dimension: () => 'createdAt', partition_granularity: 'day', - refresh_key: { + refreshKey: { sql: () => 'SELECT MAX(created_at) FROM orders', - }, - } + } satisfies CubeRefreshKey, + } satisfies PreAggregationDefinitionOriginalSql }, data_source: 'default', rewrite_queries: true, diff --git a/yarn.lock b/yarn.lock index 4678fb93ad1e2..945c02a67e211 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7848,6 +7848,11 @@ dependencies: "@types/node" "*" +"@types/node-dijkstra@^2.5.6": + version "2.5.6" + resolved "https://registry.yarnpkg.com/@types/node-dijkstra/-/node-dijkstra-2.5.6.tgz#df4621e50df10b2e98229796ab1c2a3ca74b65b8" + integrity sha512-+n0D+FdGuCLsKoH7fwX3iWfkKSAG0e4z1F96UG5gAnlE2V/1AZO6LuPLzHQd1MC2fZJDcE2cpTS2Ln1lywgUvw== + "@types/node-fetch@^2.5.7", "@types/node-fetch@^2.5.8": version "2.6.12" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.12.tgz#8ab5c3ef8330f13100a7479e2cd56d3386830a03" From f1dff828991300c39b96a8b4a0c2b81015f65e48 Mon Sep 17 00:00:00 2001 From: Dmitry Patsura Date: Thu, 17 Jul 2025 16:17:05 +0200 Subject: [PATCH 3/3] chore: Use typings directly from generic-pool, not from DefinitelyTyped (#9712) The generic-pool package provides typings itself. Mixing types from the package and definitely typed is a bad idea :) --- packages/cubejs-cubestore-driver/package.json | 1 - packages/cubejs-databricks-jdbc-driver/package.json | 1 - packages/cubejs-dbt-schema-extension/package.json | 1 - packages/cubejs-druid-driver/package.json | 1 - packages/cubejs-jdbc-driver/package.json | 3 +-- packages/cubejs-mongobi-driver/package.json | 3 +-- packages/cubejs-mysql-driver/package.json | 3 +-- packages/cubejs-query-orchestrator/package.json | 2 -- yarn.lock | 9 +-------- 9 files changed, 4 insertions(+), 20 deletions(-) diff --git a/packages/cubejs-cubestore-driver/package.json b/packages/cubejs-cubestore-driver/package.json index 774e5c34a32de..c79ddd67cd828 100644 --- a/packages/cubejs-cubestore-driver/package.json +++ b/packages/cubejs-cubestore-driver/package.json @@ -43,7 +43,6 @@ "devDependencies": { "@cubejs-backend/linter": "1.3.39", "@types/csv-write-stream": "^2.0.0", - "@types/generic-pool": "^3.8.2", "@types/jest": "^29", "@types/node": "^20", "@types/ws": "^7.4.0", diff --git a/packages/cubejs-databricks-jdbc-driver/package.json b/packages/cubejs-databricks-jdbc-driver/package.json index e6c006e901ab6..496ab68db220d 100644 --- a/packages/cubejs-databricks-jdbc-driver/package.json +++ b/packages/cubejs-databricks-jdbc-driver/package.json @@ -41,7 +41,6 @@ }, "devDependencies": { "@cubejs-backend/linter": "1.3.39", - "@types/generic-pool": "^3.8.2", "@types/jest": "^29", "@types/node": "^20", "@types/ramda": "^0.27.34", diff --git a/packages/cubejs-dbt-schema-extension/package.json b/packages/cubejs-dbt-schema-extension/package.json index 4f585480ff39b..ecd84c2292d39 100644 --- a/packages/cubejs-dbt-schema-extension/package.json +++ b/packages/cubejs-dbt-schema-extension/package.json @@ -33,7 +33,6 @@ "devDependencies": { "@cubejs-backend/linter": "1.3.39", "@cubejs-backend/testing": "1.3.39", - "@types/generic-pool": "^3.8.2", "@types/jest": "^29", "jest": "^29", "stream-to-array": "^2.3.0", diff --git a/packages/cubejs-druid-driver/package.json b/packages/cubejs-druid-driver/package.json index 161f906f98aa6..a9002ad9995fc 100644 --- a/packages/cubejs-druid-driver/package.json +++ b/packages/cubejs-druid-driver/package.json @@ -35,7 +35,6 @@ }, "devDependencies": { "@cubejs-backend/linter": "1.3.39", - "@types/generic-pool": "^3.8.2", "@types/jest": "^29", "@types/node": "^20", "jest": "^29", diff --git a/packages/cubejs-jdbc-driver/package.json b/packages/cubejs-jdbc-driver/package.json index 969e02e92089b..7e73de02457e3 100644 --- a/packages/cubejs-jdbc-driver/package.json +++ b/packages/cubejs-jdbc-driver/package.json @@ -28,7 +28,7 @@ "@cubejs-backend/base-driver": "1.3.39", "@cubejs-backend/node-java-maven": "^0.1.3", "@cubejs-backend/shared": "1.3.39", - "generic-pool": "^3.8.2", + "generic-pool": "^3.9.0", "sqlstring": "^2.3.0" }, "optionalDependencies": { @@ -44,7 +44,6 @@ }, "devDependencies": { "@cubejs-backend/linter": "1.3.39", - "@types/generic-pool": "^3.8.2", "@types/node": "^20", "@types/sqlstring": "^2.3.0", "typescript": "~5.2.2" diff --git a/packages/cubejs-mongobi-driver/package.json b/packages/cubejs-mongobi-driver/package.json index 0b486ed68a7a1..3f7ed18d8c4a6 100644 --- a/packages/cubejs-mongobi-driver/package.json +++ b/packages/cubejs-mongobi-driver/package.json @@ -30,7 +30,7 @@ "@cubejs-backend/base-driver": "1.3.39", "@cubejs-backend/shared": "1.3.39", "@types/node": "^20", - "generic-pool": "^3.8.2", + "generic-pool": "^3.9.0", "moment": "^2.29.1", "mysql2": "^3.11.5" }, @@ -40,7 +40,6 @@ }, "devDependencies": { "@cubejs-backend/linter": "1.3.39", - "@types/generic-pool": "^3.8.2", "testcontainers": "^10.28.0", "typescript": "~5.2.2" }, diff --git a/packages/cubejs-mysql-driver/package.json b/packages/cubejs-mysql-driver/package.json index 8d4d22a301c21..0dfb37571522a 100644 --- a/packages/cubejs-mysql-driver/package.json +++ b/packages/cubejs-mysql-driver/package.json @@ -29,13 +29,12 @@ "dependencies": { "@cubejs-backend/base-driver": "1.3.39", "@cubejs-backend/shared": "1.3.39", - "generic-pool": "^3.8.2", + "generic-pool": "^3.9.0", "mysql": "^2.18.1" }, "devDependencies": { "@cubejs-backend/linter": "1.3.39", "@cubejs-backend/testing-shared": "1.3.39", - "@types/generic-pool": "^3.8.2", "@types/jest": "^29", "@types/mysql": "^2.15.21", "jest": "^29", diff --git a/packages/cubejs-query-orchestrator/package.json b/packages/cubejs-query-orchestrator/package.json index 52cc32492bd95..bafe3d40738bf 100644 --- a/packages/cubejs-query-orchestrator/package.json +++ b/packages/cubejs-query-orchestrator/package.json @@ -33,13 +33,11 @@ "@cubejs-backend/cubestore-driver": "1.3.39", "@cubejs-backend/shared": "1.3.39", "csv-write-stream": "^2.0.0", - "generic-pool": "^3.8.2", "lru-cache": "^11.1.0", "ramda": "^0.27.2" }, "devDependencies": { "@cubejs-backend/linter": "1.3.39", - "@types/generic-pool": "^3.8.2", "@types/jest": "^29", "@types/node": "^20", "@types/ramda": "^0.27.32", diff --git a/yarn.lock b/yarn.lock index 945c02a67e211..9cee7240fcc71 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7676,13 +7676,6 @@ dependencies: "@types/node" "*" -"@types/generic-pool@^3.8.2": - version "3.8.3" - resolved "https://registry.yarnpkg.com/@types/generic-pool/-/generic-pool-3.8.3.tgz#7e542afd682b18fd1e8d9e251ac94037c7a48004" - integrity sha512-ZGnwaX+JGSCMnjf4/CmLkTjJwE5bnT45Z3y+3HvoYymKU5vPMCEZAFtrl3sBpg+rjFpEvhSBTJX7jfJArBKjYQ== - dependencies: - generic-pool "*" - "@types/glob@*", "@types/glob@^7.1.1": version "7.2.0" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb" @@ -14379,7 +14372,7 @@ generate-object-property@^1.0.0: dependencies: is-property "^1.0.0" -generic-pool@*, generic-pool@^3.8.2: +generic-pool@^3.8.2, generic-pool@^3.9.0: version "3.9.0" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==