From 69904fd438a2a4ce45b50e395dc4440de6674dbf Mon Sep 17 00:00:00 2001 From: Pavel Tiunov Date: Thu, 28 Mar 2024 22:10:45 -0700 Subject: [PATCH] feat: Ungrouped query pre-aggregation support --- .../src/adapter/BaseQuery.js | 8 +- .../src/adapter/PreAggregations.js | 40 +++- .../fixtures/_schemas.json | 28 ++- .../src/tests/testQueries.ts | 12 ++ .../__snapshots__/postgres-full.test.ts.snap | 181 ++++++++++++++++++ .../__snapshots__/snowflake-full.test.ts.snap | 181 ++++++++++++++++++ 6 files changed, 436 insertions(+), 14 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 0b040e831b4eb..977bffaf0f530 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -463,7 +463,7 @@ export class BaseQuery { buildParamAnnotatedSql() { let sql; let preAggForQuery; - if (!this.options.preAggregationQuery && !this.ungrouped) { + if (!this.options.preAggregationQuery) { preAggForQuery = this.preAggregations.findPreAggregationForQuery(); if (this.options.disableExternalPreAggregations && preAggForQuery && preAggForQuery.preAggregation.external) { @@ -539,7 +539,7 @@ export class BaseQuery { } externalPreAggregationQuery() { - if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && !this.ungrouped && this.externalQueryClass) { + if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) { const preAggregationForQuery = this.preAggregations.findPreAggregationForQuery(); if (preAggregationForQuery && preAggregationForQuery.preAggregation.external) { return true; @@ -558,7 +558,7 @@ export class BaseQuery { * @returns {Array} */ buildSqlAndParams(exportAnnotatedSql) { - if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && !this.ungrouped && this.externalQueryClass) { + if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) { if (this.externalPreAggregationQuery()) { // TODO performance return this.externalQuery().buildSqlAndParams(exportAnnotatedSql); } @@ -1848,7 +1848,7 @@ export class BaseQuery { primaryKeyNames(cubeName) { const primaryKeys = this.cubeEvaluator.primaryKeys[cubeName]; if (!primaryKeys || !primaryKeys.length) { - throw new UserError(`One or more Primary key is required for '${cubeName}`); + throw new UserError(`One or more Primary key is required for '${cubeName}' cube`); } return primaryKeys.map((pk) => this.primaryKeyName(cubeName, pk)); } diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js index 5b36775f64303..09d810a835c83 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.js @@ -314,6 +314,17 @@ export class PreAggregations { }); }).map(d => query.newTimeDimension(d)); + let sortedAllCubeNames; + let sortedUsedCubePrimaryKeys; + + if (query.ungrouped) { + const { allCubeNames } = query; + sortedAllCubeNames = allCubeNames.concat([]); + sortedAllCubeNames.sort(); + sortedUsedCubePrimaryKeys = query.allCubeNames.flatMap(c => query.primaryKeyNames(c)); + sortedUsedCubePrimaryKeys.sort(); + } + const measureToLeafMeasures = {}; const leafMeasurePaths = @@ -398,7 +409,10 @@ export class PreAggregations { ownedDimensions, ownedTimeDimensionsWithRollupGranularity, ownedTimeDimensionsAsIs, - allBackAliasMembers + allBackAliasMembers, + ungrouped: query.ungrouped, + sortedUsedCubePrimaryKeys, + sortedAllCubeNames }; } @@ -665,6 +679,20 @@ export class PreAggregations { (references.sortedTimeDimensions || sortTimeDimensions(references.timeDimensions)) ); + if (transformedQuery.ungrouped) { + const allReferenceCubes = R.pipe(R.map(m => (m.dimension || m).split('.')[0]), R.uniq, R.sortBy(R.identity))( + references.measures.concat(references.dimensions).concat(references.timeDimensions) + ); + if ( + !R.equals(transformedQuery.sortedAllCubeNames, allReferenceCubes) || + !( + dimensionsMatch(transformedQuery.sortedUsedCubePrimaryKeys, true) || dimensionsMatch(transformedQuery.sortedUsedCubePrimaryKeys, false) + ) + ) { + return false; + } + } + const backAliasMeasures = backAlias(references.measures); return (( windowGranularityMatches(references) @@ -687,9 +715,9 @@ export class PreAggregations { * @returns {boolean} */ const canUseFn = - transformedQuery.leafMeasureAdditive && - !transformedQuery.hasMultipliedMeasures - ? (r) => canUsePreAggregationLeafMeasureAdditive(r) || + ( + transformedQuery.leafMeasureAdditive && !transformedQuery.hasMultipliedMeasures || transformedQuery.ungrouped + ) ? (r) => canUsePreAggregationLeafMeasureAdditive(r) || canUsePreAggregationNotAdditive(r) : canUsePreAggregationNotAdditive; @@ -1293,12 +1321,12 @@ export class PreAggregations { const measure = this.query.newMeasure(path); return [ path, - this.query.aggregateOnGroupedColumn( + this.query.ungrouped ? measure.aliasName() : (this.query.aggregateOnGroupedColumn( measure.measureDefinition(), measure.aliasName(), !this.query.safeEvaluateSymbolContext().overTimeSeriesAggregate, path, - ) || `sum(${measure.aliasName()})`, + ) || `sum(${measure.aliasName()})`), ]; }), R.fromPairs, diff --git a/packages/cubejs-testing-drivers/fixtures/_schemas.json b/packages/cubejs-testing-drivers/fixtures/_schemas.json index fa3d8c3b1f3a8..16a48195fe40b 100644 --- a/packages/cubejs-testing-drivers/fixtures/_schemas.json +++ b/packages/cubejs-testing-drivers/fixtures/_schemas.json @@ -6,17 +6,23 @@ { "name": "category", "type": "string", - "sql": "category" + "sql": "category", + "primary_key": true, + "shown": true }, { "name": "subCategory", "type": "string", - "sql": "sub_category" + "sql": "sub_category", + "primary_key": true, + "shown": true }, { "name": "productName", "type": "string", - "sql": "product_name" + "sql": "product_name", + "primary_key": true, + "shown": true } ] }, @@ -151,7 +157,6 @@ }, { "name": "BigECommerce", - "extends": "Products", "joins": [ { "name": "Customers", @@ -167,6 +172,21 @@ "primary_key": true, "shown": true }, + { + "name": "category", + "type": "string", + "sql": "category" + }, + { + "name": "subCategory", + "type": "string", + "sql": "sub_category" + }, + { + "name": "productName", + "type": "string", + "sql": "product_name" + }, { "name": "orderId", "sql": "order_id", diff --git a/packages/cubejs-testing-drivers/src/tests/testQueries.ts b/packages/cubejs-testing-drivers/src/tests/testQueries.ts index ccf3bc1d377fa..2a5a6beabe985 100644 --- a/packages/cubejs-testing-drivers/src/tests/testQueries.ts +++ b/packages/cubejs-testing-drivers/src/tests/testQueries.ts @@ -1522,5 +1522,17 @@ from `); expect(res.rows).toMatchSnapshot('powerbi_min_max_ungrouped_flag'); }); + + executePg('SQL API: ungrouped pre-agg', async () => { + const res = await connection.query(` + select + "productName", + "totalSales" + from + "public"."BigECommerce" "$Table" + order by 2 desc, 1 asc + `); + expect(res.rows).toMatchSnapshot('ungrouped_pre_agg'); + }); }); } diff --git a/packages/cubejs-testing-drivers/test/__snapshots__/postgres-full.test.ts.snap b/packages/cubejs-testing-drivers/test/__snapshots__/postgres-full.test.ts.snap index dc335464cbd07..715b041b47939 100644 --- a/packages/cubejs-testing-drivers/test/__snapshots__/postgres-full.test.ts.snap +++ b/packages/cubejs-testing-drivers/test/__snapshots__/postgres-full.test.ts.snap @@ -19,6 +19,187 @@ Array [ ] `; +exports[`Queries with the @cubejs-backend/postgres-driver SQL API: ungrouped pre-agg: ungrouped_pre_agg 1`] = ` +Array [ + Object { + "productName": "Canon PC1080F Personal Copier", + "totalSales": 2399.96, + }, + Object { + "productName": "Logitech diNovo Edge Keyboard", + "totalSales": 2249.91, + }, + Object { + "productName": "Global Adaptabilites Bookcase, Cherry/Storm Gray Finish", + "totalSales": 2154.9, + }, + Object { + "productName": "Google Nexus 5", + "totalSales": 1979.89, + }, + Object { + "productName": "Global Adaptabilites Bookcase, Cherry/Storm Gray Finish", + "totalSales": 1292.94, + }, + Object { + "productName": "Canon PC1080F Personal Copier", + "totalSales": 1199.98, + }, + Object { + "productName": "Hewlett Packard 610 Color Digital Copier / Printer", + "totalSales": 899.982, + }, + Object { + "productName": "Okidata C610n Printer", + "totalSales": 649, + }, + Object { + "productName": "Lexmark 20R1285 X6650 Wireless All-in-One Printer", + "totalSales": 600, + }, + Object { + "productName": "Google Nexus 6", + "totalSales": 539.97, + }, + Object { + "productName": "Google Nexus 7", + "totalSales": 539.97, + }, + Object { + "productName": "Harbour Creations 67200 Series Stacking Chairs", + "totalSales": 498.26, + }, + Object { + "productName": "DMI Eclipse Executive Suite Bookcases", + "totalSales": 400.784, + }, + Object { + "productName": "HTC One", + "totalSales": 239.976, + }, + Object { + "productName": "Iceberg Nesting Folding Chair, 19w x 6d x 43h", + "totalSales": 232.88, + }, + Object { + "productName": "Balt Solid Wood Rectangular Table", + "totalSales": 210.98, + }, + Object { + "productName": "Tyvek Side-Opening Peel & Seel Expanding Envelopes", + "totalSales": 180.96, + }, + Object { + "productName": "Panasonic KP-380BK Classic Electric Pencil Sharpener", + "totalSales": 179.9, + }, + Object { + "productName": "Harbour Creations 67200 Series Stacking Chairs", + "totalSales": 128.124, + }, + Object { + "productName": "Harbour Creations 67200 Series Stacking Chairs", + "totalSales": 113.888, + }, + Object { + "productName": "Panasonic KP-380BK Classic Electric Pencil Sharpener", + "totalSales": 86.352, + }, + Object { + "productName": "Kingston Digital DataTraveler 16GB USB 2.2", + "totalSales": 71.6, + }, + Object { + "productName": "Linden 10 Round Wall Clock, Black", + "totalSales": 48.896, + }, + Object { + "productName": "Vinyl Coated Wire Paper Clips in Organizer Box, 800/Box", + "totalSales": 45.92, + }, + Object { + "productName": "Vinyl Coated Wire Paper Clips in Organizer Box, 800/Box", + "totalSales": 45.92, + }, + Object { + "productName": "Kingston Digital DataTraveler 16GB USB 2.0", + "totalSales": 44.75, + }, + Object { + "productName": "Kingston Digital DataTraveler 16GB USB 2.1", + "totalSales": 44.75, + }, + Object { + "productName": "Recycled Eldon Regeneration Jumbo File", + "totalSales": 39.296, + }, + Object { + "productName": "Linden 10 Round Wall Clock, Black", + "totalSales": 36.672, + }, + Object { + "productName": "Linden 10 Round Wall Clock, Black", + "totalSales": 30.56, + }, + Object { + "productName": "Linden 10 Round Wall Clock, Black", + "totalSales": 30.56, + }, + Object { + "productName": "Anderson Hickey Conga Table Tops & Accessories", + "totalSales": 24.368, + }, + Object { + "productName": "Plymouth Boxed Rubber Bands by Plymouth", + "totalSales": 23.55, + }, + Object { + "productName": "Vinyl Coated Wire Paper Clips in Organizer Box, 800/Box", + "totalSales": 18.368, + }, + Object { + "productName": "Wausau Papers Astrobrights Colored Envelopes", + "totalSales": 14.352, + }, + Object { + "productName": "Plymouth Boxed Rubber Bands by Plymouth", + "totalSales": 14.13, + }, + Object { + "productName": "Project Tote Personal File", + "totalSales": 14.03, + }, + Object { + "productName": "Plymouth Boxed Rubber Bands by Plymouth", + "totalSales": 11.304, + }, + Object { + "productName": "Magna Visual Magnetic Picture Hangers", + "totalSales": 9.64, + }, + Object { + "productName": "OIC #2 Pencils, Medium Soft", + "totalSales": 9.4, + }, + Object { + "productName": "Magna Visual Magnetic Picture Hangers", + "totalSales": 7.712, + }, + Object { + "productName": "OIC #2 Pencils, Medium Soft", + "totalSales": 3.76, + }, + Object { + "productName": "OIC #2 Pencils, Medium Soft", + "totalSales": 3.76, + }, + Object { + "productName": "Lexmark 20R1285 X6650 Wireless All-in-One Printer", + "totalSales": null, + }, +] +`; + exports[`Queries with the @cubejs-backend/postgres-driver filtering Customers: contains + dimensions, first 1`] = ` Array [ Object { diff --git a/packages/cubejs-testing-drivers/test/__snapshots__/snowflake-full.test.ts.snap b/packages/cubejs-testing-drivers/test/__snapshots__/snowflake-full.test.ts.snap index 08554c6734e00..71dae27edb221 100644 --- a/packages/cubejs-testing-drivers/test/__snapshots__/snowflake-full.test.ts.snap +++ b/packages/cubejs-testing-drivers/test/__snapshots__/snowflake-full.test.ts.snap @@ -19,6 +19,187 @@ Array [ ] `; +exports[`Queries with the @cubejs-backend/snowflake-driver SQL API: ungrouped pre-agg: ungrouped_pre_agg 1`] = ` +Array [ + Object { + "productName": "Canon PC1080F Personal Copier", + "totalSales": 2399.96, + }, + Object { + "productName": "Logitech diNovo Edge Keyboard", + "totalSales": 2249.91, + }, + Object { + "productName": "Global Adaptabilites Bookcase, Cherry/Storm Gray Finish", + "totalSales": 2154.9, + }, + Object { + "productName": "Google Nexus 5", + "totalSales": 1979.89, + }, + Object { + "productName": "Global Adaptabilites Bookcase, Cherry/Storm Gray Finish", + "totalSales": 1292.94, + }, + Object { + "productName": "Canon PC1080F Personal Copier", + "totalSales": 1199.98, + }, + Object { + "productName": "Hewlett Packard 610 Color Digital Copier / Printer", + "totalSales": 899.982, + }, + Object { + "productName": "Okidata C610n Printer", + "totalSales": 649, + }, + Object { + "productName": "Lexmark 20R1285 X6650 Wireless All-in-One Printer", + "totalSales": 600, + }, + Object { + "productName": "Google Nexus 6", + "totalSales": 539.97, + }, + Object { + "productName": "Google Nexus 7", + "totalSales": 539.97, + }, + Object { + "productName": "Harbour Creations 67200 Series Stacking Chairs", + "totalSales": 498.26, + }, + Object { + "productName": "DMI Eclipse Executive Suite Bookcases", + "totalSales": 400.784, + }, + Object { + "productName": "HTC One", + "totalSales": 239.976, + }, + Object { + "productName": "Iceberg Nesting Folding Chair, 19w x 6d x 43h", + "totalSales": 232.88, + }, + Object { + "productName": "Balt Solid Wood Rectangular Table", + "totalSales": 210.98, + }, + Object { + "productName": "Tyvek Side-Opening Peel & Seel Expanding Envelopes", + "totalSales": 180.96, + }, + Object { + "productName": "Panasonic KP-380BK Classic Electric Pencil Sharpener", + "totalSales": 179.9, + }, + Object { + "productName": "Harbour Creations 67200 Series Stacking Chairs", + "totalSales": 128.124, + }, + Object { + "productName": "Harbour Creations 67200 Series Stacking Chairs", + "totalSales": 113.888, + }, + Object { + "productName": "Panasonic KP-380BK Classic Electric Pencil Sharpener", + "totalSales": 86.352, + }, + Object { + "productName": "Kingston Digital DataTraveler 16GB USB 2.2", + "totalSales": 71.6, + }, + Object { + "productName": "Linden 10 Round Wall Clock, Black", + "totalSales": 48.896, + }, + Object { + "productName": "Vinyl Coated Wire Paper Clips in Organizer Box, 800/Box", + "totalSales": 45.92, + }, + Object { + "productName": "Vinyl Coated Wire Paper Clips in Organizer Box, 800/Box", + "totalSales": 45.92, + }, + Object { + "productName": "Kingston Digital DataTraveler 16GB USB 2.0", + "totalSales": 44.75, + }, + Object { + "productName": "Kingston Digital DataTraveler 16GB USB 2.1", + "totalSales": 44.75, + }, + Object { + "productName": "Recycled Eldon Regeneration Jumbo File", + "totalSales": 39.296, + }, + Object { + "productName": "Linden 10 Round Wall Clock, Black", + "totalSales": 36.672, + }, + Object { + "productName": "Linden 10 Round Wall Clock, Black", + "totalSales": 30.56, + }, + Object { + "productName": "Linden 10 Round Wall Clock, Black", + "totalSales": 30.56, + }, + Object { + "productName": "Anderson Hickey Conga Table Tops & Accessories", + "totalSales": 24.368, + }, + Object { + "productName": "Plymouth Boxed Rubber Bands by Plymouth", + "totalSales": 23.55, + }, + Object { + "productName": "Vinyl Coated Wire Paper Clips in Organizer Box, 800/Box", + "totalSales": 18.368, + }, + Object { + "productName": "Wausau Papers Astrobrights Colored Envelopes", + "totalSales": 14.352, + }, + Object { + "productName": "Plymouth Boxed Rubber Bands by Plymouth", + "totalSales": 14.13, + }, + Object { + "productName": "Project Tote Personal File", + "totalSales": 14.03, + }, + Object { + "productName": "Plymouth Boxed Rubber Bands by Plymouth", + "totalSales": 11.304, + }, + Object { + "productName": "Magna Visual Magnetic Picture Hangers", + "totalSales": 9.64, + }, + Object { + "productName": "OIC #2 Pencils, Medium Soft", + "totalSales": 9.4, + }, + Object { + "productName": "Magna Visual Magnetic Picture Hangers", + "totalSales": 7.712, + }, + Object { + "productName": "OIC #2 Pencils, Medium Soft", + "totalSales": 3.76, + }, + Object { + "productName": "OIC #2 Pencils, Medium Soft", + "totalSales": 3.76, + }, + Object { + "productName": "Lexmark 20R1285 X6650 Wireless All-in-One Printer", + "totalSales": null, + }, +] +`; + exports[`Queries with the @cubejs-backend/snowflake-driver filtering Customers: contains + dimensions, first 1`] = ` Array [ Object {