diff --git a/mysql-test/main/statistics_json.result b/mysql-test/main/statistics_json.result new file mode 100644 index 0000000000000..e53a6f72f12b5 --- /dev/null +++ b/mysql-test/main/statistics_json.result @@ -0,0 +1,300 @@ +# +# Test that we can store JSON arrays in histogram field mysql.column_stats when histogram_type=JSON +# +drop table if exists t1; +set @save_histogram_type=@@histogram_type; +set @save_histogram_size=@@histogram_size; +CREATE TABLE t1 ( +a int, +b varchar(32), +c char(2), +d double +); +INSERT INTO t1 SELECT seq, seq, seq, seq from seq_1_to_25; +SET histogram_type='JSON'; +set histogram_size=10; +ANALYZE TABLE t1 PERSISTENT FOR ALL; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * FROM mysql.column_stats WHERE table_name='t1'; +db_name table_name column_name min_value max_value nulls_ratio avg_length avg_frequency hist_size hist_type histogram +test t1 a 1 25 0.0000 4.0000 1.0000 10 JSON [ + "3", + "5", + "7", + "10", + "12", + "14", + "16", + "19", + "21", + "23" +] +test t1 b 1 9 0.0000 1.6400 1.0000 10 JSON [ + "11", + "13", + "15", + "18", + "2", + "21", + "23", + "3", + "5", + "7" +] +test t1 c 1 9 0.0000 2.0000 1.0000 10 JSON [ + "11", + "13", + "15", + "18", + "2", + "21", + "23", + "3", + "5", + "7" +] +test t1 d 1 25 0.0000 8.0000 1.0000 10 JSON [ + "3", + "5", + "7", + "10", + "12", + "14", + "16", + "19", + "21", + "23" +] +DELETE FROM mysql.column_stats; +DROP TABLE t1; +create schema world; +use world; +set histogram_type='JSON'; +set histogram_size=25; +ANALYZE TABLE Country PERSISTENT FOR ALL; +SELECT column_name, min_value, max_value, hist_size, hist_type, histogram FROM mysql.column_stats; +column_name min_value max_value hist_size hist_type histogram +Code ABW ZWE 25 JSON [ + "ARM", + "BEL", + "BLZ", + "CAF", + "COG", + "CZE", + "ERI", + "FRO", + "GMB", + "GUY", + "IRL", + "KAZ", + "LBN", + "LVA", + "MKD", + "MUS", + "NIC", + "PAN", + "PRT", + "SAU", + "SMR", + "SYC", + "TMP", + "UKR", + "VIR" +] +Name Afghanistan Zimbabwe 25 JSON [ + "Argentina", + "Barbados", + "Botswana", + "Cameroon", + "Cocos (Keeling) Islands", + "Czech Republic", + "El Salvador", + "France", + "Gibraltar", + "Guyana", + "Iran", + "Kenya", + "Libyan Arab Jamahiriya", + "Maldives", + "Micronesia, Federated States of", + "Nepal", + "Norfolk Island", + "Papua New Guinea", + "Romania", + "Samoa", + "Solomon Islands", + "Svalbard and Jan Mayen", + "Togo", + "Uganda", + "Venezuela" +] +SurfaceArea 0.40 17075400.00 25 JSON [ + "36.00", + "151.00", + "261.00", + "388.00", + "618.00", + "1102.00", + "4000.00", + "10990.00", + "18274.00", + "27750.00", + "36188.00", + "51197.00", + "71740.00", + "93030.00", + "112622.00", + "163610.00", + "238391.00", + "283561.00", + "338145.00", + "449964.00", + "587041.00", + "796095.00", + "1104300.00", + "1648195.00", + "2724900.00" +] +Population 0 1277558000 25 JSON [ + "1000", + "8000", + "25000", + "65000", + "94000", + "181000", + "293000", + "453000", + "817000", + "1439200", + "2542000", + "3337000", + "3869000", + "4807000", + "5496000", + "7430000", + "8861400", + "10278100", + "11669000", + "16125000", + "22256000", + "28351000", + "40377000", + "61399000", + "111506000" +] +Capital 1 4074 25 JSON [ + "62", + "149", + "201", + "553", + "645", + "902", + "922", + "1109", + "1530", + "1859", + "2296", + "2430", + "2452", + "2482", + "2689", + "2729", + "2831", + "2919", + "3047", + "3162", + "3209", + "3250", + "3336", + "3426", + "3538" +] +ID 1 4079 10 JSON [ + "371", + "742", + "1113", + "1484", + "1855", + "2225", + "2596", + "2967", + "3338", + "3709" +] +Name A Coruña (La Coruña) Århus 10 JSON [ + "Berdytšiv", + "Clermont-Ferrand", + "Gatineau", + "Itapecerica da Serra", + "Kuytun", + "Milano", + "Ota", + "Roanoke", + "Split", + "Ulsan" +] +Country ABW ZWE 10 JSON [ + "BRA", + "CHN", + "DZA", + "IDN", + "IND", + "JPN", + "MEX", + "PHL", + "RUS", + "USA" +] +Population 42 10500000 10 JSON [ + "96002", + "105819", + "118326", + "132318", + "152397", + "181900", + "221400", + "288173", + "398300", + "670208" +] +Country ABW ZWE 10 JSON [ + "BGD", + "CMR", + "ESP", + "GUF", + "KAZ", + "MDA", + "NAM", + "PNG", + "SVK", + "UGA" +] +Language Abhyasi [South]Mande 10 JSON [ + "Bariba", + "Creole English", + "English", + "Georgiana", + "Italian", + "Lomwe", + "Moravian", + "Punjabi", + "Soga", + "Tigrinja" +] +Percentage 0.0 99.9 10 JSON [ + "0.2", + "0.6", + "1.4", + "2.4", + "3.9", + "6.5", + "9.6", + "17.7", + "45.5", + "87.5" +] +set histogram_type=@save_histogram_type; +set histogram_size=@save_histogram_size; +DROP SCHEMA world; diff --git a/mysql-test/main/statistics_json.test b/mysql-test/main/statistics_json.test new file mode 100644 index 0000000000000..acc44456d8f23 --- /dev/null +++ b/mysql-test/main/statistics_json.test @@ -0,0 +1,56 @@ +--source include/have_stat_tables.inc +--source include/have_sequence.inc +--source include/analyze-format.inc +--echo # +--echo # Test that we can store JSON arrays in histogram field mysql.column_stats when histogram_type=JSON +--echo # +--disable_warnings +drop table if exists t1; +--enable_warnings + +set @save_histogram_type=@@histogram_type; +set @save_histogram_size=@@histogram_size; + +CREATE TABLE t1 ( + a int, + b varchar(32), + c char(2), + d double +); + +--disable_result_log +INSERT INTO t1 SELECT seq, seq, seq, seq from seq_1_to_25; +--enable_result_log + +SET histogram_type='JSON'; +# set histogram size to be < row count (25 in this case) to see how histogram behaves +set histogram_size=10; + +ANALYZE TABLE t1 PERSISTENT FOR ALL; +SELECT * FROM mysql.column_stats WHERE table_name='t1'; +DELETE FROM mysql.column_stats; +DROP TABLE t1; + +create schema world; +use world; +--disable_query_log +--disable_result_log +--disable_warnings +--source include/world_schema_utf8.inc +--source include/world.inc +--enable_warnings +--enable_result_log +--enable_query_log + +set histogram_type='JSON'; +set histogram_size=25; +--disable_result_log +ANALYZE TABLE Country PERSISTENT FOR ALL; +--enable_result_log + +SELECT column_name, min_value, max_value, hist_size, hist_type, histogram FROM mysql.column_stats; + +set histogram_type=@save_histogram_type; +set histogram_size=@save_histogram_size; + +DROP SCHEMA world; \ No newline at end of file diff --git a/mysql-test/main/system_mysql_db.result b/mysql-test/main/system_mysql_db.result index d0d1e7c11757b..b756dfcf45e43 100644 --- a/mysql-test/main/system_mysql_db.result +++ b/mysql-test/main/system_mysql_db.result @@ -234,8 +234,8 @@ column_stats CREATE TABLE `column_stats` ( `avg_length` decimal(12,4) DEFAULT NULL, `avg_frequency` decimal(12,4) DEFAULT NULL, `hist_size` tinyint(3) unsigned DEFAULT NULL, - `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB') COLLATE utf8mb3_bin DEFAULT NULL, - `histogram` varbinary(255) DEFAULT NULL, + `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON') COLLATE utf8mb3_bin DEFAULT NULL, + `histogram` blob DEFAULT NULL, PRIMARY KEY (`db_name`,`table_name`,`column_name`) ) ENGINE=Aria DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_bin PAGE_CHECKSUM=1 TRANSACTIONAL=0 COMMENT='Statistics on Columns' show create table index_stats; diff --git a/mysql-test/main/system_mysql_db_fix40123.result b/mysql-test/main/system_mysql_db_fix40123.result index 127e2c49642f5..ec972058d54e0 100644 --- a/mysql-test/main/system_mysql_db_fix40123.result +++ b/mysql-test/main/system_mysql_db_fix40123.result @@ -272,8 +272,8 @@ column_stats CREATE TABLE `column_stats` ( `avg_length` decimal(12,4) DEFAULT NULL, `avg_frequency` decimal(12,4) DEFAULT NULL, `hist_size` tinyint(3) unsigned DEFAULT NULL, - `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB') COLLATE utf8mb3_bin DEFAULT NULL, - `histogram` varbinary(255) DEFAULT NULL, + `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON') COLLATE utf8mb3_bin DEFAULT NULL, + `histogram` blob DEFAULT NULL, PRIMARY KEY (`db_name`,`table_name`,`column_name`) ) ENGINE=Aria DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_bin PAGE_CHECKSUM=1 TRANSACTIONAL=0 COMMENT='Statistics on Columns' show create table index_stats; diff --git a/mysql-test/main/system_mysql_db_fix50030.result b/mysql-test/main/system_mysql_db_fix50030.result index fd0074c7f54f5..4e038849cf672 100644 --- a/mysql-test/main/system_mysql_db_fix50030.result +++ b/mysql-test/main/system_mysql_db_fix50030.result @@ -276,8 +276,8 @@ column_stats CREATE TABLE `column_stats` ( `avg_length` decimal(12,4) DEFAULT NULL, `avg_frequency` decimal(12,4) DEFAULT NULL, `hist_size` tinyint(3) unsigned DEFAULT NULL, - `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB') COLLATE utf8mb3_bin DEFAULT NULL, - `histogram` varbinary(255) DEFAULT NULL, + `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON') COLLATE utf8mb3_bin DEFAULT NULL, + `histogram` blob DEFAULT NULL, PRIMARY KEY (`db_name`,`table_name`,`column_name`) ) ENGINE=Aria DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_bin PAGE_CHECKSUM=1 TRANSACTIONAL=0 COMMENT='Statistics on Columns' show create table index_stats; diff --git a/mysql-test/main/system_mysql_db_fix50117.result b/mysql-test/main/system_mysql_db_fix50117.result index a7d705c4158c1..7d540477d51ac 100644 --- a/mysql-test/main/system_mysql_db_fix50117.result +++ b/mysql-test/main/system_mysql_db_fix50117.result @@ -256,8 +256,8 @@ column_stats CREATE TABLE `column_stats` ( `avg_length` decimal(12,4) DEFAULT NULL, `avg_frequency` decimal(12,4) DEFAULT NULL, `hist_size` tinyint(3) unsigned DEFAULT NULL, - `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB') COLLATE utf8mb3_bin DEFAULT NULL, - `histogram` varbinary(255) DEFAULT NULL, + `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON') COLLATE utf8mb3_bin DEFAULT NULL, + `histogram` blob DEFAULT NULL, PRIMARY KEY (`db_name`,`table_name`,`column_name`) ) ENGINE=Aria DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_bin PAGE_CHECKSUM=1 TRANSACTIONAL=0 COMMENT='Statistics on Columns' show create table index_stats; diff --git a/scripts/mysql_system_tables.sql b/scripts/mysql_system_tables.sql index cee47a0e5d656..e31f3372b5f7e 100644 --- a/scripts/mysql_system_tables.sql +++ b/scripts/mysql_system_tables.sql @@ -314,7 +314,7 @@ DROP TABLE tmp_proxies_priv; CREATE TABLE IF NOT EXISTS table_stats (db_name varchar(64) NOT NULL, table_name varchar(64) NOT NULL, cardinality bigint(21) unsigned DEFAULT NULL, PRIMARY KEY (db_name,table_name) ) engine=Aria transactional=0 CHARACTER SET utf8 COLLATE utf8_bin comment='Statistics on Tables'; -CREATE TABLE IF NOT EXISTS column_stats (db_name varchar(64) NOT NULL, table_name varchar(64) NOT NULL, column_name varchar(64) NOT NULL, min_value varbinary(255) DEFAULT NULL, max_value varbinary(255) DEFAULT NULL, nulls_ratio decimal(12,4) DEFAULT NULL, avg_length decimal(12,4) DEFAULT NULL, avg_frequency decimal(12,4) DEFAULT NULL, hist_size tinyint unsigned, hist_type enum('SINGLE_PREC_HB','DOUBLE_PREC_HB'), histogram varbinary(255), PRIMARY KEY (db_name,table_name,column_name) ) engine=Aria transactional=0 CHARACTER SET utf8 COLLATE utf8_bin comment='Statistics on Columns'; +CREATE TABLE IF NOT EXISTS column_stats (db_name varchar(64) NOT NULL, table_name varchar(64) NOT NULL, column_name varchar(64) NOT NULL, min_value varbinary(255) DEFAULT NULL, max_value varbinary(255) DEFAULT NULL, nulls_ratio decimal(12,4) DEFAULT NULL, avg_length decimal(12,4) DEFAULT NULL, avg_frequency decimal(12,4) DEFAULT NULL, hist_size tinyint unsigned, hist_type enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON'), histogram blob, PRIMARY KEY (db_name,table_name,column_name) ) engine=Aria transactional=0 CHARACTER SET utf8 COLLATE utf8_bin comment='Statistics on Columns'; CREATE TABLE IF NOT EXISTS index_stats (db_name varchar(64) NOT NULL, table_name varchar(64) NOT NULL, index_name varchar(64) NOT NULL, prefix_arity int(11) unsigned NOT NULL, avg_frequency decimal(12,4) DEFAULT NULL, PRIMARY KEY (db_name,table_name,index_name,prefix_arity) ) engine=Aria transactional=0 CHARACTER SET utf8 COLLATE utf8_bin comment='Statistics on Indexes'; diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 7e8ff667e75a6..072d775dcaa7d 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -496,7 +496,7 @@ String *Item_func_from_base64::val_str(String *str) const char *histogram_types[] = - {"SINGLE_PREC_HB", "DOUBLE_PREC_HB", 0}; + {"SINGLE_PREC_HB", "DOUBLE_PREC_HB", "JSON", 0}; static TYPELIB hystorgam_types_typelib= { array_elements(histogram_types), "histogram_types", diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 1f034f490c8e2..56e20ecf48e08 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -32,6 +32,10 @@ #include "uniques.h" #include "sql_show.h" #include "sql_partition.h" +#include "my_json_writer.h" + +#include +#include /* The system variable 'use_stat_tables' can take one of the @@ -178,12 +182,12 @@ TABLE_FIELD_TYPE column_stat_fields[COLUMN_STAT_N_FIELDS] = }, { { STRING_WITH_LEN("hist_type") }, - { STRING_WITH_LEN("enum('SINGLE_PREC_HB','DOUBLE_PREC_HB')") }, + { STRING_WITH_LEN("enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON')") }, { STRING_WITH_LEN("utf8mb3") } }, { { STRING_WITH_LEN("histogram") }, - { STRING_WITH_LEN("varbinary(255)") }, + { STRING_WITH_LEN("blob") }, { NULL, 0 } } }; @@ -1070,9 +1074,16 @@ class Column_stat: public Stat_table stat_field->store(stats->histogram.get_type() + 1); break; case COLUMN_STAT_HISTOGRAM: - stat_field->store((char *)stats->histogram.get_values(), - stats->histogram.get_size(), &my_charset_bin); - break; + if (stats->histogram.get_type() == JSON) + { + stat_field->store((char *) stats->histogram.get_values(), + strlen((char *) stats->histogram.get_values()), &my_charset_bin); + } else + { + stat_field->store((char *) stats->histogram.get_values(), + stats->histogram.get_size(), &my_charset_bin); + } + break; } } } @@ -1519,6 +1530,7 @@ class Stat_table_write_iter class Histogram_builder { +protected: Field *column; /* table field for which the histogram is built */ uint col_length; /* size of this field */ ha_rows records; /* number of records the histogram is built for */ @@ -1549,13 +1561,15 @@ class Histogram_builder count_distinct_single_occurence= 0; } + virtual ~Histogram_builder() = default; + ulonglong get_count_distinct() const { return count_distinct; } ulonglong get_count_single_occurence() const { return count_distinct_single_occurence; } - int next(void *elem, element_count elem_cnt) + virtual int next(void *elem, element_count elem_cnt) { count_distinct++; if (elem_cnt == 1) @@ -1567,7 +1581,7 @@ class Histogram_builder { column->store_field_value((uchar *) elem, col_length); histogram->set_value(curr_bucket, - column->pos_in_interval(min_value, max_value)); + column->pos_in_interval(min_value, max_value)); curr_bucket++; while (curr_bucket != hist_width && count > bucket_capacity * (curr_bucket + 1)) @@ -1580,6 +1594,59 @@ class Histogram_builder } }; +class Histogram_builder_json : public Histogram_builder +{ +std::vector bucket_bounds = {}; + +public: + Histogram_builder_json(Field *col, uint col_len, ha_rows rows) + : Histogram_builder(col, col_len, rows) + { + Column_statistics *col_stats= col->collected_stats; + min_value= col_stats->min_value; + max_value= col_stats->max_value; + histogram= &col_stats->histogram; + hist_width= histogram->get_width(); + bucket_capacity= (double) records / (hist_width + 1); + curr_bucket= 0; + count= 0; + count_distinct= 0; + count_distinct_single_occurence= 0; + } + + ~Histogram_builder_json() override = default; + + int next(void *elem, element_count elem_cnt) override + { + count_distinct++; + if (elem_cnt == 1) + count_distinct_single_occurence++; + count+= elem_cnt; + if (curr_bucket == hist_width) + return 0; + if (count > bucket_capacity * (curr_bucket + 1)) + { + column->store_field_value((uchar *) elem, col_length); + StringBuffer val; + column->val_str(&val); + bucket_bounds.emplace_back(val.c_ptr()); + curr_bucket++; + } + return 0; + } + + void build() { + Json_writer *writer = new Json_writer(); + writer->start_array(); + for(auto& value: bucket_bounds) { + writer->add_str(value.c_str()); + } + writer->end_array(); + histogram->set_size(bucket_bounds.size()); + Binary_string *json_string = (Binary_string *) writer->output.get_string(); + histogram->set_values((uchar *) json_string->c_ptr()); + } +}; C_MODE_START @@ -1589,6 +1656,12 @@ int histogram_build_walk(void *elem, element_count elem_cnt, void *arg) return hist_builder->next(elem, elem_cnt); } +int json_histogram_build_walk(void *elem, element_count elem_cnt, void *arg) +{ + Histogram_builder_json *hist_builder= (Histogram_builder_json *) arg; + return hist_builder->next(elem, elem_cnt); +} + static int count_distinct_single_occurence_walk(void *elem, @@ -1694,10 +1767,22 @@ class Count_distinct_field: public Sql_alloc */ void walk_tree_with_histogram(ha_rows rows) { - Histogram_builder hist_builder(table_field, tree_key_length, rows); - tree->walk(table_field->table, histogram_build_walk, (void *) &hist_builder); - distincts= hist_builder.get_count_distinct(); - distincts_single_occurence= hist_builder.get_count_single_occurence(); + if(table_field->collected_stats->histogram.get_type() == JSON) + { + Histogram_builder_json hist_builder(table_field, tree_key_length, rows); + tree->walk(table_field->table, json_histogram_build_walk, + (void *) &hist_builder); + hist_builder.build(); + distincts= hist_builder.get_count_distinct(); + distincts_single_occurence= hist_builder.get_count_single_occurence(); + } else + { + Histogram_builder hist_builder(table_field, tree_key_length, rows); + tree->walk(table_field->table, histogram_build_walk, + (void *) &hist_builder); + distincts= hist_builder.get_count_distinct(); + distincts_single_occurence= hist_builder.get_count_single_occurence(); + } } ulonglong get_count_distinct() diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 35b3aa33accf5..a554721d50b13 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -42,7 +42,8 @@ typedef enum enum_histogram_type { SINGLE_PREC_HB, - DOUBLE_PREC_HB + DOUBLE_PREC_HB, + JSON } Histogram_type; enum enum_stat_tables @@ -152,6 +153,7 @@ class Histogram { switch (type) { case SINGLE_PREC_HB: + case JSON: return ((uint) (1 << 8) - 1); case DOUBLE_PREC_HB: return ((uint) (1 << 16) - 1); @@ -164,6 +166,7 @@ class Histogram { switch (type) { case SINGLE_PREC_HB: + case JSON: return size; case DOUBLE_PREC_HB: return size / 2; @@ -177,6 +180,7 @@ class Histogram DBUG_ASSERT(i < get_width()); switch (type) { case SINGLE_PREC_HB: + case JSON: return (uint) (((uint8 *) values)[i]); case DOUBLE_PREC_HB: return (uint) uint2korr(values + i * 2); @@ -253,7 +257,8 @@ class Histogram void set_value(uint i, double val) { switch (type) { - case SINGLE_PREC_HB: + case SINGLE_PREC_HB: + case JSON: ((uint8 *) values)[i]= (uint8) (val * prec_factor()); return; case DOUBLE_PREC_HB: @@ -265,7 +270,8 @@ class Histogram void set_prev_value(uint i) { switch (type) { - case SINGLE_PREC_HB: + case SINGLE_PREC_HB: + case JSON: ((uint8 *) values)[i]= ((uint8 *) values)[i-1]; return; case DOUBLE_PREC_HB: