From a9dd95e8072fb41ac17ae8f404f94230965ac880 Mon Sep 17 00:00:00 2001 From: Marc Selwan Date: Thu, 4 Dec 2025 14:30:45 -0800 Subject: [PATCH 1/3] init commit with new aggregations --- .../reference/limitations-best-practices.mdx | 148 +++++++++++++-- src/content/docs/r2-sql/sql-reference.mdx | 171 +++++++++++++++++- 2 files changed, 299 insertions(+), 20 deletions(-) diff --git a/src/content/docs/r2-sql/reference/limitations-best-practices.mdx b/src/content/docs/r2-sql/reference/limitations-best-practices.mdx index ef6c506750b..b459cf63204 100644 --- a/src/content/docs/r2-sql/reference/limitations-best-practices.mdx +++ b/src/content/docs/r2-sql/reference/limitations-best-practices.mdx @@ -15,22 +15,23 @@ R2 SQL is designed for querying **partitioned** Apache Iceberg tables in your R2 ## Quick Reference -| Feature | Supported | Notes | -| :-------------------- | :-------- | :------------------------------------ | -| Basic SELECT | Yes | Columns, \* | -| Aggregation functions | No | No COUNT, AVG, etc. | -| Single table FROM | Yes | Note, aliasing not supported | -| WHERE clause | Yes | Filters, comparisons, equality, etc | -| JOINs | No | No table joins | -| Array filtering | No | No array type support | -| JSON filtering | No | No nested object queries | -| Simple LIMIT | Yes | 1-10,000 range, no pagination support | -| ORDER BY | Yes | Any columns of the partition key only | -| GROUP BY | No | Not supported | +| Feature | Supported | Notes | +| :-------------------- | :-------- | :----------------------------------------------- | +| Basic SELECT | Yes | Columns, \* | +| Aggregation functions | Yes | COUNT(\*), SUM, AVG, MIN, MAX with limitations | +| Single table FROM | Yes | Note, aliasing not supported | +| WHERE clause | Yes | Filters, comparisons, equality, etc | +| JOINs | No | No table joins | +| Array filtering | No | No array type support | +| JSON filtering | No | No nested object queries | +| Simple LIMIT | Yes | 1-10,000 range, no pagination support | +| ORDER BY | Yes | Partition key only, or COUNT(\*) with GROUP BY | +| GROUP BY | Yes | Supported with limitations | +| HAVING | Yes | Only with COUNT(\*) | ## Supported SQL Clauses -R2 SQL supports a limited set of SQL clauses: `SELECT`, `FROM`, `WHERE`, `ORDER BY`, and `LIMIT`. All other SQL clauses are not supported at the moment. New features will be released in the future, keep an eye on this page for the latest. +R2 SQL supports: `SELECT`, `FROM`, `WHERE`, `GROUP BY`, `HAVING`, `ORDER BY`, and `LIMIT`. New features will be released in the future, keep an eye on this page for the latest. --- @@ -44,9 +45,9 @@ R2 SQL supports a limited set of SQL clauses: `SELECT`, `FROM`, `WHERE`, `ORDER ### Limitations - **No JSON field querying**: Cannot query individual fields from JSON objects -- **No SQL functions**: Functions like `AVG()`, `COUNT()`, `MAX()`, `MIN()`, quantiles are not supported +- **Limited aggregation functions**: See Aggregation Functions section below for details - **No synthetic data**: Cannot create synthetic columns like `SELECT 1 AS what, "hello" AS greeting` -- **No field aliasing**: `SELECT field AS another_name` +- **No field aliasing**: `SELECT field AS another_name` (applies to both regular columns and aggregations) ### Examples @@ -64,6 +65,103 @@ SELECT 1 AS synthetic_column FROM my_table; --- +## Aggregation Functions + +### Supported Features + +- **COUNT(*)**: Count total rows **note**: only `*` is supported +- **SUM(column)**: Sum numeric values +- **AVG(column)**: Calculate average of numeric values +- **MIN(column)**: Find minimum value +- **MAX(column)**: Find maximum value +- **With GROUP BY**: All aggregations work with `GROUP BY` + +### Limitations + +- **No aliases**: `AS` keyword not supported (`SELECT COUNT(*) AS total` fails) +- **COUNT(*) only**: `COUNT(column_name)` or `COUNT(DISTINCT column)` is not supported +- **No aggregation in WHERE**: Cannot use aggregations in WHERE clause + +### Examples + +```sql +-- Valid +SELECT department, COUNT(*) FROM sales GROUP BY department +SELECT region, AVG(amount) FROM sales GROUP BY region +SELECT category, MIN(price), MAX(price) FROM products GROUP BY category + +-- Invalid +SELECT COUNT(*) AS total FROM sales GROUP BY department -- No aliases +SELECT COUNT(department) FROM sales -- Must use COUNT(*) +SELECT COUNT(DISTINCT region) FROM sales -- No DISTINCT +SELECT SUM(quantity) FROM sales -- SUM on INTEGER fails +SELECT department, SUM(amount) +FROM sales +GROUP BY department +ORDER BY SUM(amount) DESC -- ORDER BY SUM not supported +``` + +--- + +## GROUP BY Clause + +### Supported Features + +- **Single column grouping**: `GROUP BY column` +- **Multiple column grouping**: `GROUP BY column1, column2` +- **With WHERE**: Filter before grouping +- **With HAVING**: Filter grouped results (COUNT(*) only) +- **With LIMIT**: Limit grouped results + +### Limitations + +- **HAVING COUNT(*) only**: Cannot use HAVING with SUM/AVG +- **No expressions**: Cannot use expressions in GROUP BY (e.g., `GROUP BY YEAR(date)`) +- **No BOOLEAN grouping**: Cannot group by BOOLEAN columns + +### Examples + +```sql +-- Valid +SELECT region, COUNT(*) FROM sales GROUP BY region +SELECT dept, category, COUNT(*) FROM sales GROUP BY dept, category +SELECT region, COUNT(*) FROM sales WHERE status = 'completed' GROUP BY region +SELECT dept, COUNT(*) FROM sales GROUP BY dept ORDER BY COUNT(*) DESC LIMIT 10 + +-- Invalid +SELECT is_active, SUM(amount) FROM sales GROUP BY is_active -- BOOLEAN grouping +SELECT dept, SUM(amount) FROM sales GROUP BY dept ORDER BY SUM(amount) DESC -- ORDER BY SUM +``` + +--- + +## HAVING Clause + +### Supported Features + +- **With COUNT(*)**: Filter groups by count +- **Comparison operators**: `>`, `>=`, `=`, `<`, `<=`, `!=` +- **With GROUP BY**: Must be used with GROUP BY + +### Limitations + +- **COUNT(*) only**: `HAVING SUM(column)` or `HAVING AVG(column)` not supported +- **No complex expressions**: Simple comparisons only +- **No AND/OR**: Multiple HAVING conditions not supported + +### Examples + +```sql +-- Valid +SELECT region, COUNT(*) FROM sales GROUP BY region HAVING COUNT(*) > 1000 + +-- Invalid +SELECT dept, SUM(amount) FROM sales GROUP BY dept HAVING SUM(amount) > 100000 -- HAVING with SUM +SELECT region, COUNT(*) FROM sales GROUP BY region HAVING COUNT(*) > 100 AND COUNT(*) < 1000 -- Multiple conditions +``` + +--- + ## FROM Clause ### Supported Features @@ -138,16 +236,25 @@ SELECT * FROM logs WHERE response_time + latency > 5000 -- Arithmetic - **ASC**: Ascending order - **DESC**: Descending order (Default, on full partition key) +- **With partition key**: Order by partition key columns +- **With COUNT(*)**: Order by COUNT(*) when using GROUP BY ### Limitations -- **Non-partition keys not supported**: `ORDER BY` on columns other than the partition key is not supported +- **Non-partition keys not supported**: `ORDER BY` on columns other than the partition key is not supported (except with aggregations) +- **COUNT(*) only**: `ORDER BY SUM(column)`, `ORDER BY AVG(column)`, `ORDER BY MIN(column)`, or `ORDER BY MAX(column)` not supported ### Examples ```sql +-- Valid SELECT * FROM table_name WHERE ... ORDER BY partitionKey SELECT * FROM table_name WHERE ... ORDER BY partitionKey DESC +SELECT dept, COUNT(*) FROM table_name GROUP BY dept ORDER BY COUNT(*) DESC + +-- Invalid +SELECT dept, SUM(amount) FROM table_name GROUP BY dept ORDER BY SUM(amount) DESC +SELECT region, AVG(price) FROM table_name GROUP BY region ORDER BY AVG(price) ``` --- @@ -184,14 +291,14 @@ SELECT * FROM events LIMIT 10 * 10 -- Arithmetic The following SQL clauses are **not supported**: -- `GROUP BY` -- `HAVING` - `UNION`/`INTERSECT`/`EXCEPT` - `WITH` (Common Table Expressions) - `WINDOW` functions - `INSERT`/`UPDATE`/`DELETE` - `CREATE`/`ALTER`/`DROP` +Note: `GROUP BY` and `HAVING` are partially supported with significant limitations (see sections above). + --- ## Best Practices @@ -199,5 +306,10 @@ The following SQL clauses are **not supported**: 1. Always include time filters in your WHERE clause to ensure efficient queries. 2. Use specific column selection instead of `SELECT *` when possible for better performance. 3. Structure your data to avoid nested JSON objects if you need to filter on those fields. +4. When using aggregations, always include `GROUP BY` for reliable results. +5. Use `COUNT(*)` exclusively - avoid `COUNT(column_name)` or `COUNT(DISTINCT column)`. +6. For ordering aggregated results, use `ORDER BY COUNT(*)` rather than `ORDER BY SUM/AVG/MIN/MAX`. +7. Use `SUM()` only on DECIMAL or FLOAT columns, not INTEGER columns. +8. Avoid grouping by BOOLEAN columns when using aggregation functions. --- diff --git a/src/content/docs/r2-sql/sql-reference.mdx b/src/content/docs/r2-sql/sql-reference.mdx index 48e9888132e..e826d31bcbd 100644 --- a/src/content/docs/r2-sql/sql-reference.mdx +++ b/src/content/docs/r2-sql/sql-reference.mdx @@ -16,12 +16,14 @@ This page documents the R2 SQL syntax based on the currently supported grammar i --- -## Complete Query Syntax +## Query Syntax ```sql -SELECT column_list +SELECT column_list | aggregation_function FROM table_name WHERE conditions --optional +[GROUP BY column_list] +[HAVING conditions] [ORDER BY column_name [DESC | ASC]] [LIMIT number] ``` @@ -52,6 +54,59 @@ SELECT timestamp, user_id, response_code FROM table_name --- +## Aggregation Functions + +### Syntax + +```sql +SELECT aggregation_function(column_name) +FROM table_name +GROUP BY column_list +``` + +### Supported Functions + +- **COUNT(*)**: Counts total rows **note**: only `*` is supported +- **SUM(column)**: Sums numeric values +- **AVG(column)**: Calculates average of numeric values +- **MIN(column)**: Finds minimum value +- **MAX(column)**: Finds maximum value + +### Examples + +```sql +-- Count rows by department +SELECT department, COUNT(*) +FROM sales_data +GROUP BY department + +-- Sum decimal values +SELECT region, SUM(total_amount) +FROM sales_data +GROUP BY region + +-- Average by category +SELECT category, AVG(price) +FROM products +GROUP BY category + +-- Min and Max +SELECT department, MIN(salary), MAX(salary) +FROM employees +GROUP BY department + +-- Invalid: No aliases +SELECT department, COUNT(*) AS total FROM sales_data GROUP BY department + +-- Invalid: COUNT column name +SELECT COUNT(department) FROM sales_data + +-- Invalid: SUM on integer +SELECT SUM(quantity) FROM sales_data +``` + +--- + ## FROM Clause ### Syntax @@ -115,6 +170,87 @@ SELECT * FROM table_name WHERE (status = 404 OR status = 500) AND timestamp > '2 --- +## GROUP BY Clause + +### Syntax + +```sql +SELECT column_list, aggregation_function +FROM table_name +[WHERE conditions] +GROUP BY column_list +``` + +### Examples + +```sql +-- Single column grouping +SELECT department, COUNT(*) +FROM sales_data +GROUP BY department + +-- Multiple column grouping +SELECT department, category, COUNT(*) +FROM sales_data +GROUP BY department, category + +-- With WHERE filter +SELECT region, COUNT(*) +FROM sales_data +WHERE status = 'completed' +GROUP BY region + +-- With ORDER BY (COUNT only) +SELECT region, COUNT(*) +FROM sales_data +GROUP BY region +ORDER BY COUNT(*) DESC +LIMIT 10 + +-- Invalid: ORDER BY SUM +SELECT department, SUM(amount) +FROM sales_data +GROUP BY department +ORDER BY SUM(amount) DESC +``` + +--- + +## HAVING Clause + +### Syntax + +```sql +SELECT column_list, COUNT(*) +FROM table_name +GROUP BY column_list +HAVING COUNT(*) comparison_operator value +``` + +### Examples + +```sql +-- Filter by count threshold +SELECT department, COUNT(*) +FROM sales_data +GROUP BY department +HAVING COUNT(*) > 1000 + +-- Multiple conditions +SELECT region, COUNT(*) +FROM sales_data +GROUP BY region +HAVING COUNT(*) >= 100 + +-- Invalid: HAVING with SUM +SELECT department, SUM(amount) +FROM sales_data +GROUP BY department +HAVING SUM(amount) > 1000000 +``` + +--- + ## ORDER BY Clause ### Syntax @@ -204,6 +340,37 @@ ORDER BY timestamp LIMIT 500 ``` +### Aggregation Query + +```sql +SELECT department, COUNT(*) +FROM sales_data +WHERE sale_date >= '2024-01-01' +GROUP BY department +ORDER BY COUNT(*) DESC +LIMIT 10 +``` + +### Aggregation with HAVING + +```sql +SELECT region, COUNT(*) +FROM sales_data +WHERE status = 'completed' +GROUP BY region +HAVING COUNT(*) > 1000 +LIMIT 20 +``` + +### Multiple Column Grouping + +```sql +SELECT department, category, MIN(price), MAX(price) +FROM products +GROUP BY department, category +LIMIT 100 +``` + --- ## Data Type Reference From d900824cb2d624384e6dd79517e7a4b0572d8fc4 Mon Sep 17 00:00:00 2001 From: Marc Selwan Date: Thu, 11 Dec 2025 10:37:02 -0800 Subject: [PATCH 2/3] addressed comments, added changelog --- ...025-12-12-aggregation-support-and-more.mdx | 86 +++++++++++ .../reference/limitations-best-practices.mdx | 55 +++----- src/content/docs/r2-sql/sql-reference.mdx | 133 +++++++++++------- 3 files changed, 186 insertions(+), 88 deletions(-) create mode 100644 src/content/changelog/r2-sql/2025-12-12-aggregation-support-and-more.mdx diff --git a/src/content/changelog/r2-sql/2025-12-12-aggregation-support-and-more.mdx b/src/content/changelog/r2-sql/2025-12-12-aggregation-support-and-more.mdx new file mode 100644 index 00000000000..1591ffa9c1a --- /dev/null +++ b/src/content/changelog/r2-sql/2025-12-12-aggregation-support-and-more.mdx @@ -0,0 +1,86 @@ +--- +title: R2 SQL now supports aggregations and schema discovery +description: Perform aggregations, grouping, and filtering on Apache Iceberg tables stored in R2 Data Catalog +date: 2025-12-12 +products: + - r2-sql +hidden: false +--- + +R2 SQL now supports aggregation functions, `GROUP BY`, `HAVING`, along with schema discovery commands to make it easy to explore your data catalog. + +## Aggregation Functions + +You can now perform aggregations on Apache Iceberg tables in [R2 Data Catalog](/r2/data-catalog/) using standard SQL functions including `COUNT(*)`, `SUM()`, `AVG()`, `MIN()`, and `MAX()`. Combine these with `GROUP BY` to analyze data across dimensions, and use `HAVING` to filter aggregated results. + +```sql +-- Calculate average transaction amounts by department +SELECT department, COUNT(*), AVG(total_amount) +FROM my_namespace.sales_data +WHERE region = 'North' +GROUP BY department +HAVING COUNT(*) > 50 +ORDER BY AVG(total_amount) DESC +``` + +```sql +-- Find high-value departments +SELECT department, SUM(total_amount) +FROM my_namespace.sales_data +GROUP BY department +HAVING SUM(total_amount) > 50000 +``` + +## Schema Discovery + +New metadata commands make it easy to explore your data catalog and understand table structures: + +- `SHOW DATABASES` or `SHOW NAMESPACES` - List all available namespaces +- `SHOW TABLES IN namespace_name` - List tables within a namespace +- `DESCRIBE namespace_name.table_name` - View table schema and column types + +```bash +❯ npx wrangler r2 sql query "{ACCOUNT_ID}_{BUCKET_NAME}" "DESCRIBE default.sales_data;" + + ⛅️ wrangler 4.54.0 +───────────────────────────────────────────── + +┌──────────────────┬────────────────┬──────────┬─────────────────┬───────────────┬───────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ column_name │ type │ required │ initial_default │ write_default │ doc │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ sale_id │ BIGINT │ false │ │ │ Unique identifier for each sales transaction │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ sale_timestamp │ TIMESTAMPTZ │ false │ │ │ Exact date and time when the sale occurred (used for partitioning) │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ department │ TEXT │ false │ │ │ Product department (8 categories: Electronics, Beauty, Home, Toys, Sports, Food, Clothing, Books) │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ category │ TEXT │ false │ │ │ Product category grouping (4 categories: Premium, Standard, Budget, Clearance) │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ region │ TEXT │ false │ │ │ Geographic sales region (5 regions: North, South, East, West, Central) │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ product_id │ INT │ false │ │ │ Unique identifier for the product sold │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ quantity │ INT │ false │ │ │ Number of units sold in this transaction (range: 1-50) │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ unit_price │ DECIMAL(10, 2) │ false │ │ │ Price per unit in dollars (range: $5.00-$500.00) │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ total_amount │ DECIMAL(10, 2) │ false │ │ │ Total sale amount before tax (quantity × unit_price with discounts applied) │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ discount_percent │ INT │ false │ │ │ Discount percentage applied to this sale (0-50%) │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ tax_amount │ DECIMAL(10, 2) │ false │ │ │ Tax amount collected on this sale │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ profit_margin │ DECIMAL(10, 2) │ false │ │ │ Profit margin on this sale as a decimal percentage │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ customer_id │ INT │ false │ │ │ Unique identifier for the customer who made the purchase │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ is_online_sale │ BOOLEAN │ false │ │ │ Boolean flag indicating if sale was made online (true) or in-store (false) │ +├──────────────────┼────────────────┼──────────┼─────────────────┼───────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ sale_date │ DATE │ false │ │ │ Calendar date of the sale (extracted from sale_timestamp) │ +└──────────────────┴────────────────┴──────────┴─────────────────┴───────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────┘ +Read 0 B across 0 files from R2 +On average, 0 B / s + +``` + +To learn more about the new aggregation capabilities and schema discovery commands, check out the [SQL reference](/r2-sql/sql-reference/). If you're new to R2 SQL, visit our [getting started guide](/r2-sql/get-started/) to begin querying your data. diff --git a/src/content/docs/r2-sql/reference/limitations-best-practices.mdx b/src/content/docs/r2-sql/reference/limitations-best-practices.mdx index b459cf63204..11197749c6c 100644 --- a/src/content/docs/r2-sql/reference/limitations-best-practices.mdx +++ b/src/content/docs/r2-sql/reference/limitations-best-practices.mdx @@ -18,20 +18,20 @@ R2 SQL is designed for querying **partitioned** Apache Iceberg tables in your R2 | Feature | Supported | Notes | | :-------------------- | :-------- | :----------------------------------------------- | | Basic SELECT | Yes | Columns, \* | -| Aggregation functions | Yes | COUNT(\*), SUM, AVG, MIN, MAX with limitations | +| Aggregation functions | Yes | COUNT(\*), SUM, AVG, MIN, MAX | | Single table FROM | Yes | Note, aliasing not supported | | WHERE clause | Yes | Filters, comparisons, equality, etc | | JOINs | No | No table joins | | Array filtering | No | No array type support | | JSON filtering | No | No nested object queries | | Simple LIMIT | Yes | 1-10,000 range, no pagination support | -| ORDER BY | Yes | Partition key only, or COUNT(\*) with GROUP BY | -| GROUP BY | Yes | Supported with limitations | -| HAVING | Yes | Only with COUNT(\*) | +| ORDER BY | Yes | Partition key or with GROUP BY columns | +| GROUP BY | Yes | Supported | +| HAVING | Yes | Supported | ## Supported SQL Clauses -R2 SQL supports: `SELECT`, `FROM`, `WHERE`, `GROUP BY`, `HAVING`, `ORDER BY`, and `LIMIT`. New features will be released in the future, keep an eye on this page for the latest. +R2 SQL supports: `DESCRIBE`, `SHOW`, `SELECT`, `FROM`, `WHERE`, `GROUP BY`, `HAVING`, `ORDER BY`, and `LIMIT`. New features will be released in the future, keep an eye on this page for the latest. --- @@ -80,7 +80,6 @@ SELECT 1 AS synthetic_column FROM my_table; - **No aliases**: `AS` keyword not supported (`SELECT COUNT(*) AS total` fails) - **COUNT(*) only**: `COUNT(column_name)` or `COUNT(DISTINCT column)` is not supported -- **No aggregation in WHERE**: Cannot use aggregations in WHERE clause ### Examples @@ -89,16 +88,15 @@ SELECT 1 AS synthetic_column FROM my_table; SELECT department, COUNT(*) FROM sales GROUP BY department SELECT region, AVG(amount) FROM sales GROUP BY region SELECT category, MIN(price), MAX(price) FROM products GROUP BY category +SELECT SUM(quantity) FROM sales +FROM sales +GROUP BY department +ORDER BY SUM(amount) DESC -- Invalid SELECT COUNT(*) AS total FROM sales GROUP BY department -- No aliases SELECT COUNT(department) FROM sales -- Must use COUNT(*) SELECT COUNT(DISTINCT region) FROM sales -- No DISTINCT -SELECT SUM(quantity) FROM sales -- SUM on INTEGER fails -SELECT department, SUM(amount) -FROM sales -GROUP BY department -ORDER BY SUM(amount) DESC -- ORDER BY SUM not supported ``` --- @@ -110,27 +108,21 @@ ORDER BY SUM(amount) DESC -- ORDER BY SUM not supported - **Single column grouping**: `GROUP BY column` - **Multiple column grouping**: `GROUP BY column1, column2` - **With WHERE**: Filter before grouping -- **With HAVING**: Filter grouped results (COUNT(*) only) - **With LIMIT**: Limit grouped results ### Limitations -- **HAVING COUNT(*) only**: Cannot use HAVING with SUM/AVG - **No expressions**: Cannot use expressions in GROUP BY (e.g., `GROUP BY YEAR(date)`) -- **No BOOLEAN grouping**: Cannot group by BOOLEAN columns ### Examples ```sql --- Valid SELECT region, COUNT(*) FROM sales GROUP BY region SELECT dept, category, COUNT(*) FROM sales GROUP BY dept, category SELECT region, COUNT(*) FROM sales WHERE status = 'completed' GROUP BY region SELECT dept, COUNT(*) FROM sales GROUP BY dept ORDER BY COUNT(*) DESC LIMIT 10 - --- Invalid -SELECT is_active, SUM(amount) FROM sales GROUP BY is_active -- BOOLEAN grouping -SELECT dept, SUM(amount) FROM sales GROUP BY dept ORDER BY SUM(amount) DESC -- ORDER BY SUM +SELECT is_active, SUM(amount) FROM sales GROUP BY is_active +SELECT dept, SUM(amount) FROM sales GROUP BY dept ORDER BY SUM(amount) DESC ``` --- @@ -143,21 +135,12 @@ SELECT dept, SUM(amount) FROM sales GROUP BY dept ORDER BY SUM(amount) DESC -- O - **Comparison operators**: `>`, `>=`, `=`, `<`, `<=`, `!=` - **With GROUP BY**: Must be used with GROUP BY -### Limitations - -- **COUNT(*) only**: `HAVING SUM(column)` or `HAVING AVG(column)` not supported -- **No complex expressions**: Simple comparisons only -- **No AND/OR**: Multiple HAVING conditions not supported - ### Examples ```sql --- Valid SELECT region, COUNT(*) FROM sales GROUP BY region HAVING COUNT(*) > 1000 - --- Invalid SELECT dept, SUM(amount) FROM sales GROUP BY dept HAVING SUM(amount) > 100000 -- HAVING with SUM -SELECT region, COUNT(*) FROM sales GROUP BY region HAVING COUNT(*) > 100 AND COUNT(*) < 1000 -- Multiple conditions +SELECT region, COUNT(*) FROM sales GROUP BY region HAVING COUNT(*) > 100 AND COUNT(*) < 1000 ``` --- @@ -242,7 +225,6 @@ SELECT * FROM logs WHERE response_time + latency > 5000 -- Arithmetic ### Limitations - **Non-partition keys not supported**: `ORDER BY` on columns other than the partition key is not supported (except with aggregations) -- **COUNT(*) only**: `ORDER BY SUM(column)`, `ORDER BY AVG(column)`, `ORDER BY MIN(column)`, or `ORDER BY MAX(column)` not supported ### Examples @@ -253,8 +235,7 @@ SELECT * FROM table_name WHERE ... ORDER BY partitionKey DESC SELECT dept, COUNT(*) FROM table_name GROUP BY dept ORDER BY COUNT(*) DESC -- Invalid -SELECT dept, SUM(amount) FROM table_name GROUP BY dept ORDER BY SUM(amount) DESC -SELECT region, AVG(price) FROM table_name GROUP BY region ORDER BY AVG(price) +SELECT * FROM table_name GROUP BY dept ORDER BY nonPartitionKey DESC --ORDER BY on non-partition key ``` --- @@ -305,11 +286,7 @@ Note: `GROUP BY` and `HAVING` are partially supported with significant limitatio 1. Always include time filters in your WHERE clause to ensure efficient queries. 2. Use specific column selection instead of `SELECT *` when possible for better performance. -3. Structure your data to avoid nested JSON objects if you need to filter on those fields. -4. When using aggregations, always include `GROUP BY` for reliable results. -5. Use `COUNT(*)` exclusively - avoid `COUNT(column_name)` or `COUNT(DISTINCT column)`. -6. For ordering aggregated results, use `ORDER BY COUNT(*)` rather than `ORDER BY SUM/AVG/MIN/MAX`. -7. Use `SUM()` only on DECIMAL or FLOAT columns, not INTEGER columns. -8. Avoid grouping by BOOLEAN columns when using aggregation functions. - +3. Flatten your data to avoid nested JSON objects if you need to filter on those fields. +4. Use `COUNT(*)` exclusively - avoid `COUNT(column_name)` or `COUNT(DISTINCT column)`. +5. Enable compaction in R2 Data Catalog to reduce the number of data files needed to be scanned. --- diff --git a/src/content/docs/r2-sql/sql-reference.mdx b/src/content/docs/r2-sql/sql-reference.mdx index e826d31bcbd..5981087478d 100644 --- a/src/content/docs/r2-sql/sql-reference.mdx +++ b/src/content/docs/r2-sql/sql-reference.mdx @@ -30,6 +30,44 @@ WHERE conditions --optional --- +## Schema Discovery Commands + +R2 SQL supports metadata queries to explore available namespaces and tables. + +### SHOW DATABASES + +Lists all available namespaces. + +```sql +SHOW DATABASES; +``` + +### SHOW NAMESPACES + +Alias for `SHOW DATABASES`. Lists all available namespaces. + +```sql +SHOW NAMESPACES; +``` + +### SHOW TABLES + +Lists all tables within a specific namespace. + +```sql +SHOW TABLES IN namespace_name; +``` + +### DESCRIBE + +Describes the structure of a table, showing column names and data types. + +```sql +DESCRIBE namespace_name.table_name; +``` + +--- + ## SELECT Clause ### Syntax @@ -46,10 +84,10 @@ SELECT column_specification [, column_specification, ...] ### Examples ```sql -SELECT * FROM table_name -SELECT user_id FROM table_name -SELECT user_id, timestamp, status FROM table_name -SELECT timestamp, user_id, response_code FROM table_name +SELECT * FROM namespace_name.table_name +SELECT user_id FROM namespace_name.table_name +SELECT user_id, timestamp, status FROM namespace_name.table_name +SELECT timestamp, user_id, response_code FROM namespace_name.table_name ``` --- @@ -77,32 +115,29 @@ GROUP BY column_list ```sql -- Count rows by department SELECT department, COUNT(*) -FROM sales_data +FROM my_namespace.sales_data GROUP BY department -- Sum decimal values SELECT region, SUM(total_amount) -FROM sales_data +FROM my_namespace.sales_data GROUP BY region -- Average by category SELECT category, AVG(price) -FROM products +FROM my_namespace.products GROUP BY category -- Min and Max SELECT department, MIN(salary), MAX(salary) -FROM employees +FROM my_namespace.employees GROUP BY department -- Invalid: No aliases -SELECT department, COUNT(*) AS total FROM sales_data GROUP BY department +SELECT department, COUNT(*) AS total FROM my_namespace.sales_data GROUP BY department -- Invalid: COUNT column name -SELECT COUNT(department) FROM sales_data - --- Invalid: SUM on integer -SELECT SUM(quantity) FROM sales_data +SELECT COUNT(department) FROM my_namespace.sales_data ``` --- @@ -160,12 +195,12 @@ SELECT * WHERE condition [AND|OR condition ...] ### Examples ```sql -SELECT * FROM table_name WHERE timestamp BETWEEN '2025-09-24T01:00:00Z' AND '2025-09-25T01:00:00Z' -SELECT * FROM table_name WHERE status = 200 -SELECT * FROM table_name WHERE response_time > 1000 -SELECT * FROM table_name WHERE user_id IS NOT NULL -SELECT * FROM table_name WHERE method = 'GET' AND status >= 200 AND status < 300 -SELECT * FROM table_name WHERE (status = 404 OR status = 500) AND timestamp > '2024-01-01' +SELECT * FROM namespace_name.table_name WHERE timestamp BETWEEN '2025-09-24T01:00:00Z' AND '2025-09-25T01:00:00Z' +SELECT * FROM namespace_name.table_name WHERE status = 200 +SELECT * FROM namespace_name.table_name WHERE response_time > 1000 +SELECT * FROM namespace_name.table_name WHERE user_id IS NOT NULL +SELECT * FROM namespace_name.table_name WHERE method = 'GET' AND status >= 200 AND status < 300 +SELECT * FROM namespace_name.table_name WHERE (status = 404 OR status = 500) AND timestamp > '2024-01-01' ``` --- @@ -186,30 +221,30 @@ GROUP BY column_list ```sql -- Single column grouping SELECT department, COUNT(*) -FROM sales_data +FROM my_namespace.sales_data GROUP BY department -- Multiple column grouping SELECT department, category, COUNT(*) -FROM sales_data +FROM my_namespace.sales_data GROUP BY department, category -- With WHERE filter SELECT region, COUNT(*) -FROM sales_data +FROM my_namespace.sales_data WHERE status = 'completed' GROUP BY region -- With ORDER BY (COUNT only) SELECT region, COUNT(*) -FROM sales_data +FROM my_namespace.sales_data GROUP BY region ORDER BY COUNT(*) DESC LIMIT 10 --- Invalid: ORDER BY SUM +-- ORDER BY SUM SELECT department, SUM(amount) -FROM sales_data +FROM my_namespace.sales_data GROUP BY department ORDER BY SUM(amount) DESC ``` @@ -224,7 +259,7 @@ ORDER BY SUM(amount) DESC SELECT column_list, COUNT(*) FROM table_name GROUP BY column_list -HAVING COUNT(*) comparison_operator value +HAVING SUM/COUNT/MIN/MAX/AVG(column_name) comparison_operator value ``` ### Examples @@ -232,19 +267,19 @@ HAVING COUNT(*) comparison_operator value ```sql -- Filter by count threshold SELECT department, COUNT(*) -FROM sales_data +FROM my_namespace.sales_data GROUP BY department HAVING COUNT(*) > 1000 -- Multiple conditions SELECT region, COUNT(*) -FROM sales_data +FROM my_namespace.sales_data GROUP BY region HAVING COUNT(*) >= 100 --- Invalid: HAVING with SUM +-- HAVING with SUM SELECT department, SUM(amount) -FROM sales_data +FROM my_namespace.sales_data GROUP BY department HAVING SUM(amount) > 1000000 ``` @@ -268,9 +303,9 @@ ORDER BY partition_key [DESC] ### Examples ```sql -SELECT * FROM table_name WHERE ... ORDER BY paetition_key_A -SELECT * FROM table_name WHERE ... ORDER BY partition_key_B DESC -SELECT * FROM table_name WHERE ... ORDER BY partitionKey_A ASC +SELECT * FROM namespace_name.table_name WHERE ... ORDER BY partition_key_A +SELECT * FROM namespace_name.table_name WHERE ... ORDER BY partition_key_B DESC +SELECT * FROM namespace_name.table_name WHERE ... ORDER BY partition_key_A ASC ``` @@ -291,7 +326,7 @@ LIMIT number ### Examples ```sql -SELECT * FROM table_name WHERE ... LIMIT 100 +SELECT * FROM namespace_name.table_name WHERE ... LIMIT 100 ``` --- @@ -302,7 +337,7 @@ SELECT * FROM table_name WHERE ... LIMIT 100 ```sql SELECT * -FROM http_requests +FROM my_namespace.http_requests WHERE timestamp BETWEEN '2025-09-24T01:00:00Z' AND '2025-09-25T01:00:00Z' LIMIT 100 ``` @@ -311,7 +346,7 @@ LIMIT 100 ```sql SELECT user_id, timestamp, status, response_time -FROM access_logs +FROM my_namespace.access_logs WHERE status >= 400 AND response_time > 5000 ORDER BY response_time DESC LIMIT 50 @@ -321,7 +356,7 @@ LIMIT 50 ```sql SELECT timestamp, method, status, user_agent -FROM http_requests +FROM my_namespace.http_requests WHERE (method = 'POST' OR method = 'PUT') AND status BETWEEN 200 AND 299 AND user_agent IS NOT NULL @@ -333,7 +368,7 @@ LIMIT 1000 ```sql SELECT user_id, session_id, date_column -FROM user_events +FROM my_namespace.user_events WHERE session_id IS NOT NULL AND date_column >= '2024-01-01' ORDER BY timestamp @@ -344,7 +379,7 @@ LIMIT 500 ```sql SELECT department, COUNT(*) -FROM sales_data +FROM my_namespace.sales_data WHERE sale_date >= '2024-01-01' GROUP BY department ORDER BY COUNT(*) DESC @@ -355,7 +390,7 @@ LIMIT 10 ```sql SELECT region, COUNT(*) -FROM sales_data +FROM my_namespace.sales_data WHERE status = 'completed' GROUP BY region HAVING COUNT(*) > 1000 @@ -366,7 +401,7 @@ LIMIT 20 ```sql SELECT department, category, MIN(price), MAX(price) -FROM products +FROM my_namespace.products GROUP BY department, category LIMIT 100 ``` @@ -390,17 +425,17 @@ LIMIT 100 ```sql -- Integer comparisons -SELECT * FROM table_name WHERE status = 200 -SELECT * FROM table_name WHERE response_time > 1000 +SELECT * FROM namespace_name.table_name WHERE status = 200 +SELECT * FROM namespace_name.table_name WHERE response_time > 1000 -- Float comparisons -SELECT * FROM table_name WHERE cpu_usage >= 85.5 -SELECT * FROM table_name WHERE memory_ratio < 0.8 +SELECT * FROM namespace_name.table_name WHERE cpu_usage >= 85.5 +SELECT * FROM namespace_name.table_name WHERE memory_ratio < 0.8 -- String comparisons -SELECT * FROM table_name WHERE method = 'POST' -SELECT * FROM table_name WHERE user_agent != 'bot' -SELECT * FROM table_name WHERE country_code = 'US' +SELECT * FROM namespace_name.table_name WHERE method = 'POST' +SELECT * FROM namespace_name.table_name WHERE user_agent != 'bot' +SELECT * FROM namespace_name.table_name WHERE country_code = 'US' ``` --- @@ -414,7 +449,7 @@ SELECT * FROM table_name WHERE country_code = 'US' Use parentheses to override default precedence: ```sql -SELECT * FROM table_name WHERE (status = 404 OR status = 500) AND method = 'GET' +SELECT * FROM namespace_name.table_name WHERE (status = 404 OR status = 500) AND method = 'GET' ``` --- From affc94daed923116d41e7697354f717dd938271d Mon Sep 17 00:00:00 2001 From: Marc Selwan Date: Fri, 12 Dec 2025 07:48:09 -0800 Subject: [PATCH 3/3] addressed feedback/comments added `;` to the end of all queries, formatted some things better, corrected some inaccuracies. --- .../reference/limitations-best-practices.mdx | 78 +++++++++---------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/src/content/docs/r2-sql/reference/limitations-best-practices.mdx b/src/content/docs/r2-sql/reference/limitations-best-practices.mdx index 11197749c6c..edffb2dea6d 100644 --- a/src/content/docs/r2-sql/reference/limitations-best-practices.mdx +++ b/src/content/docs/r2-sql/reference/limitations-best-practices.mdx @@ -85,18 +85,15 @@ SELECT 1 AS synthetic_column FROM my_table; ```sql -- Valid -SELECT department, COUNT(*) FROM sales GROUP BY department -SELECT region, AVG(amount) FROM sales GROUP BY region -SELECT category, MIN(price), MAX(price) FROM products GROUP BY category -SELECT SUM(quantity) FROM sales -FROM sales -GROUP BY department -ORDER BY SUM(amount) DESC +SELECT department, COUNT(*) FROM sales GROUP BY department; +SELECT region, AVG(amount) FROM sales GROUP BY region; +SELECT category, MIN(price), MAX(price) FROM products GROUP BY category; +SELECT SUM(quantity) FROM sales GROUP BY department ORDER BY SUM(amount) DESC; -- Invalid -SELECT COUNT(*) AS total FROM sales GROUP BY department -- No aliases -SELECT COUNT(department) FROM sales -- Must use COUNT(*) -SELECT COUNT(DISTINCT region) FROM sales -- No DISTINCT +SELECT COUNT(*) AS total FROM sales GROUP BY department; -- No aliases +SELECT COUNT(department) FROM sales; -- Must use COUNT(*) +SELECT COUNT(DISTINCT region) FROM sales; -- No DISTINCT support ``` --- @@ -117,12 +114,12 @@ SELECT COUNT(DISTINCT region) FROM sales -- No DISTINCT ### Examples ```sql -SELECT region, COUNT(*) FROM sales GROUP BY region -SELECT dept, category, COUNT(*) FROM sales GROUP BY dept, category -SELECT region, COUNT(*) FROM sales WHERE status = 'completed' GROUP BY region -SELECT dept, COUNT(*) FROM sales GROUP BY dept ORDER BY COUNT(*) DESC LIMIT 10 -SELECT is_active, SUM(amount) FROM sales GROUP BY is_active -SELECT dept, SUM(amount) FROM sales GROUP BY dept ORDER BY SUM(amount) DESC +SELECT region, COUNT(*) FROM sales GROUP BY region; +SELECT dept, category, COUNT(*) FROM sales GROUP BY dept, category; +SELECT region, COUNT(*) FROM sales WHERE status = 'completed' GROUP BY region; +SELECT dept, COUNT(*) FROM sales GROUP BY dept ORDER BY COUNT(*) DESC LIMIT 10; +SELECT is_active, SUM(amount) FROM sales GROUP BY is_active; +SELECT dept, SUM(amount) FROM sales GROUP BY dept ORDER BY SUM(amount) DESC; ``` --- @@ -132,15 +129,15 @@ SELECT dept, SUM(amount) FROM sales GROUP BY dept ORDER BY SUM(amount) DESC ### Supported Features - **With COUNT(*)**: Filter groups by count -- **Comparison operators**: `>`, `>=`, `=`, `<`, `<=`, `!=` +- **Comparison operators**: `>`, `>=`, `=`, `<`, `<=`, `!=`, `BETWEEN`, `AND`, `IS NOT NULL` - **With GROUP BY**: Must be used with GROUP BY ### Examples ```sql -SELECT region, COUNT(*) FROM sales GROUP BY region HAVING COUNT(*) > 1000 -SELECT dept, SUM(amount) FROM sales GROUP BY dept HAVING SUM(amount) > 100000 -- HAVING with SUM -SELECT region, COUNT(*) FROM sales GROUP BY region HAVING COUNT(*) > 100 AND COUNT(*) < 1000 +SELECT region, COUNT(*) FROM sales GROUP BY region HAVING COUNT(*) > 1000; +SELECT dept, SUM(amount) FROM sales GROUP BY dept HAVING SUM(amount) > 100000; -- HAVING with SUM +SELECT region, COUNT(*) FROM sales GROUP BY region HAVING COUNT(*) > 100 AND COUNT(*) < 1000; ``` --- @@ -167,12 +164,12 @@ SELECT region, COUNT(*) FROM sales GROUP BY region HAVING COUNT(*) > 100 AND COU ```sql --Valid -SELECT * FROM http_requests +SELECT * FROM http_requests; --Invalid -SELECT * FROM table1, table2 -SELECT * FROM table1 JOIN table2 ON table1.id = table2.id -SELECT * FROM (SELECT * FROM events WHERE status = 200) +SELECT * FROM table1, table2; +SELECT * FROM table1 JOIN table2 ON table1.id = table2.id; +SELECT * FROM (SELECT * FROM events WHERE status = 200); ``` --- @@ -200,15 +197,15 @@ SELECT * FROM (SELECT * FROM events WHERE status = 200) ```sql --Valid -SELECT * FROM events WHERE timestamp BETWEEN '2024-01-01' AND '2024-01-02' -SELECT * FROM logs WHERE status = 200 AND user_type = 'premium' -SELECT * FROM requests WHERE (method = 'GET' OR method = 'POST') AND response_time < 1000 +SELECT * FROM events WHERE timestamp BETWEEN '2024-01-01' AND '2024-01-02'; +SELECT * FROM logs WHERE status = 200 AND user_type = 'premium'; +SELECT * FROM requests WHERE (method = 'GET' OR method = 'POST') AND response_time < 1000; --Invalid -SELECT * FROM logs WHERE tags[0] = 'error' -- Array filtering -SELECT * FROM requests WHERE metadata.user_id = '123' -- JSON field filtering -SELECT * FROM events WHERE col_a = col_b -- Column comparison -SELECT * FROM logs WHERE response_time + latency > 5000 -- Arithmetic +SELECT * FROM logs WHERE tags[0] = 'error'; -- Array filtering +SELECT * FROM requests WHERE metadata.user_id = '123'; -- JSON field filtering +SELECT * FROM events WHERE col_a = col_b; -- Column comparison +SELECT * FROM logs WHERE response_time + latency > 5000; -- Arithmetic ``` --- @@ -220,7 +217,7 @@ SELECT * FROM logs WHERE response_time + latency > 5000 -- Arithmetic - **ASC**: Ascending order - **DESC**: Descending order (Default, on full partition key) - **With partition key**: Order by partition key columns -- **With COUNT(*)**: Order by COUNT(*) when using GROUP BY +- **With GROUP BY**: Can order by all aggregation columns ### Limitations @@ -230,12 +227,12 @@ SELECT * FROM logs WHERE response_time + latency > 5000 -- Arithmetic ```sql -- Valid -SELECT * FROM table_name WHERE ... ORDER BY partitionKey -SELECT * FROM table_name WHERE ... ORDER BY partitionKey DESC -SELECT dept, COUNT(*) FROM table_name GROUP BY dept ORDER BY COUNT(*) DESC +SELECT * FROM table_name WHERE ... ORDER BY partitionKey; +SELECT * FROM table_name WHERE ... ORDER BY partitionKey DESC; +SELECT dept, COUNT(*) FROM table_name GROUP BY dept ORDER BY COUNT(*) DESC; -- Invalid -SELECT * FROM table_name GROUP BY dept ORDER BY nonPartitionKey DESC --ORDER BY on non-partition key +SELECT * FROM table_name GROUP BY dept ORDER BY nonPartitionKey DESC --ORDER BY a non-grouped column ``` --- @@ -261,9 +258,9 @@ SELECT * FROM events LIMIT 100 SELECT * FROM logs WHERE ... LIMIT 10000 -- Invalid -SELECT * FROM events LIMIT 100, 50 -- Pagination -SELECT * FROM logs LIMIT COUNT(*) / 2 -- Functions -SELECT * FROM events LIMIT 10 * 10 -- Arithmetic +SELECT * FROM events LIMIT 100, 50; -- Pagination +SELECT * FROM logs LIMIT COUNT(*); / 2 -- Functions +SELECT * FROM events LIMIT 10 * 10; -- Arithmetic ``` --- @@ -277,9 +274,6 @@ The following SQL clauses are **not supported**: - `WINDOW` functions - `INSERT`/`UPDATE`/`DELETE` - `CREATE`/`ALTER`/`DROP` - -Note: `GROUP BY` and `HAVING` are partially supported with significant limitations (see sections above). - --- ## Best Practices