diff --git a/include/json_lib.h b/include/json_lib.h index bb649928eaaa4..8f5e2776242b1 100644 --- a/include/json_lib.h +++ b/include/json_lib.h @@ -280,6 +280,14 @@ int json_key_matches(json_engine_t *je, json_string_t *k); */ int json_read_value(json_engine_t *j); +/* + * json_smart_read_value() reads parses a scalar value and value length from the json engine, + * and copies them into `value` and `value_length` respectively. + * It should only be called when the json_engine state is JST_VALUE. + * If it encounters a non-scalar value (say object or array) before getting to value_len, + * such value is also read and copied into value. + */ +enum json_types json_smart_read_value(json_engine_t *je, const char **value, int *value_len); /* json_skip_key() makes parser skip the content of the current diff --git a/mysql-test/main/statistics_json.result b/mysql-test/main/statistics_json.result new file mode 100644 index 0000000000000..e53a6f72f12b5 --- /dev/null +++ b/mysql-test/main/statistics_json.result @@ -0,0 +1,300 @@ +# +# Test that we can store JSON arrays in histogram field mysql.column_stats when histogram_type=JSON +# +drop table if exists t1; +set @save_histogram_type=@@histogram_type; +set @save_histogram_size=@@histogram_size; +CREATE TABLE t1 ( +a int, +b varchar(32), +c char(2), +d double +); +INSERT INTO t1 SELECT seq, seq, seq, seq from seq_1_to_25; +SET histogram_type='JSON'; +set histogram_size=10; +ANALYZE TABLE t1 PERSISTENT FOR ALL; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * FROM mysql.column_stats WHERE table_name='t1'; +db_name table_name column_name min_value max_value nulls_ratio avg_length avg_frequency hist_size hist_type histogram +test t1 a 1 25 0.0000 4.0000 1.0000 10 JSON [ + "3", + "5", + "7", + "10", + "12", + "14", + "16", + "19", + "21", + "23" +] +test t1 b 1 9 0.0000 1.6400 1.0000 10 JSON [ + "11", + "13", + "15", + "18", + "2", + "21", + "23", + "3", + "5", + "7" +] +test t1 c 1 9 0.0000 2.0000 1.0000 10 JSON [ + "11", + "13", + "15", + "18", + "2", + "21", + "23", + "3", + "5", + "7" +] +test t1 d 1 25 0.0000 8.0000 1.0000 10 JSON [ + "3", + "5", + "7", + "10", + "12", + "14", + "16", + "19", + "21", + "23" +] +DELETE FROM mysql.column_stats; +DROP TABLE t1; +create schema world; +use world; +set histogram_type='JSON'; +set histogram_size=25; +ANALYZE TABLE Country PERSISTENT FOR ALL; +SELECT column_name, min_value, max_value, hist_size, hist_type, histogram FROM mysql.column_stats; +column_name min_value max_value hist_size hist_type histogram +Code ABW ZWE 25 JSON [ + "ARM", + "BEL", + "BLZ", + "CAF", + "COG", + "CZE", + "ERI", + "FRO", + "GMB", + "GUY", + "IRL", + "KAZ", + "LBN", + "LVA", + "MKD", + "MUS", + "NIC", + "PAN", + "PRT", + "SAU", + "SMR", + "SYC", + "TMP", + "UKR", + "VIR" +] +Name Afghanistan Zimbabwe 25 JSON [ + "Argentina", + "Barbados", + "Botswana", + "Cameroon", + "Cocos (Keeling) Islands", + "Czech Republic", + "El Salvador", + "France", + "Gibraltar", + "Guyana", + "Iran", + "Kenya", + "Libyan Arab Jamahiriya", + "Maldives", + "Micronesia, Federated States of", + "Nepal", + "Norfolk Island", + "Papua New Guinea", + "Romania", + "Samoa", + "Solomon Islands", + "Svalbard and Jan Mayen", + "Togo", + "Uganda", + "Venezuela" +] +SurfaceArea 0.40 17075400.00 25 JSON [ + "36.00", + "151.00", + "261.00", + "388.00", + "618.00", + "1102.00", + "4000.00", + "10990.00", + "18274.00", + "27750.00", + "36188.00", + "51197.00", + "71740.00", + "93030.00", + "112622.00", + "163610.00", + "238391.00", + "283561.00", + "338145.00", + "449964.00", + "587041.00", + "796095.00", + "1104300.00", + "1648195.00", + "2724900.00" +] +Population 0 1277558000 25 JSON [ + "1000", + "8000", + "25000", + "65000", + "94000", + "181000", + "293000", + "453000", + "817000", + "1439200", + "2542000", + "3337000", + "3869000", + "4807000", + "5496000", + "7430000", + "8861400", + "10278100", + "11669000", + "16125000", + "22256000", + "28351000", + "40377000", + "61399000", + "111506000" +] +Capital 1 4074 25 JSON [ + "62", + "149", + "201", + "553", + "645", + "902", + "922", + "1109", + "1530", + "1859", + "2296", + "2430", + "2452", + "2482", + "2689", + "2729", + "2831", + "2919", + "3047", + "3162", + "3209", + "3250", + "3336", + "3426", + "3538" +] +ID 1 4079 10 JSON [ + "371", + "742", + "1113", + "1484", + "1855", + "2225", + "2596", + "2967", + "3338", + "3709" +] +Name A Coruña (La Coruña) Århus 10 JSON [ + "Berdytšiv", + "Clermont-Ferrand", + "Gatineau", + "Itapecerica da Serra", + "Kuytun", + "Milano", + "Ota", + "Roanoke", + "Split", + "Ulsan" +] +Country ABW ZWE 10 JSON [ + "BRA", + "CHN", + "DZA", + "IDN", + "IND", + "JPN", + "MEX", + "PHL", + "RUS", + "USA" +] +Population 42 10500000 10 JSON [ + "96002", + "105819", + "118326", + "132318", + "152397", + "181900", + "221400", + "288173", + "398300", + "670208" +] +Country ABW ZWE 10 JSON [ + "BGD", + "CMR", + "ESP", + "GUF", + "KAZ", + "MDA", + "NAM", + "PNG", + "SVK", + "UGA" +] +Language Abhyasi [South]Mande 10 JSON [ + "Bariba", + "Creole English", + "English", + "Georgiana", + "Italian", + "Lomwe", + "Moravian", + "Punjabi", + "Soga", + "Tigrinja" +] +Percentage 0.0 99.9 10 JSON [ + "0.2", + "0.6", + "1.4", + "2.4", + "3.9", + "6.5", + "9.6", + "17.7", + "45.5", + "87.5" +] +set histogram_type=@save_histogram_type; +set histogram_size=@save_histogram_size; +DROP SCHEMA world; diff --git a/mysql-test/main/statistics_json.test b/mysql-test/main/statistics_json.test new file mode 100644 index 0000000000000..acc44456d8f23 --- /dev/null +++ b/mysql-test/main/statistics_json.test @@ -0,0 +1,56 @@ +--source include/have_stat_tables.inc +--source include/have_sequence.inc +--source include/analyze-format.inc +--echo # +--echo # Test that we can store JSON arrays in histogram field mysql.column_stats when histogram_type=JSON +--echo # +--disable_warnings +drop table if exists t1; +--enable_warnings + +set @save_histogram_type=@@histogram_type; +set @save_histogram_size=@@histogram_size; + +CREATE TABLE t1 ( + a int, + b varchar(32), + c char(2), + d double +); + +--disable_result_log +INSERT INTO t1 SELECT seq, seq, seq, seq from seq_1_to_25; +--enable_result_log + +SET histogram_type='JSON'; +# set histogram size to be < row count (25 in this case) to see how histogram behaves +set histogram_size=10; + +ANALYZE TABLE t1 PERSISTENT FOR ALL; +SELECT * FROM mysql.column_stats WHERE table_name='t1'; +DELETE FROM mysql.column_stats; +DROP TABLE t1; + +create schema world; +use world; +--disable_query_log +--disable_result_log +--disable_warnings +--source include/world_schema_utf8.inc +--source include/world.inc +--enable_warnings +--enable_result_log +--enable_query_log + +set histogram_type='JSON'; +set histogram_size=25; +--disable_result_log +ANALYZE TABLE Country PERSISTENT FOR ALL; +--enable_result_log + +SELECT column_name, min_value, max_value, hist_size, hist_type, histogram FROM mysql.column_stats; + +set histogram_type=@save_histogram_type; +set histogram_size=@save_histogram_size; + +DROP SCHEMA world; \ No newline at end of file diff --git a/mysql-test/main/system_mysql_db.result b/mysql-test/main/system_mysql_db.result index d0d1e7c11757b..b756dfcf45e43 100644 --- a/mysql-test/main/system_mysql_db.result +++ b/mysql-test/main/system_mysql_db.result @@ -234,8 +234,8 @@ column_stats CREATE TABLE `column_stats` ( `avg_length` decimal(12,4) DEFAULT NULL, `avg_frequency` decimal(12,4) DEFAULT NULL, `hist_size` tinyint(3) unsigned DEFAULT NULL, - `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB') COLLATE utf8mb3_bin DEFAULT NULL, - `histogram` varbinary(255) DEFAULT NULL, + `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON') COLLATE utf8mb3_bin DEFAULT NULL, + `histogram` blob DEFAULT NULL, PRIMARY KEY (`db_name`,`table_name`,`column_name`) ) ENGINE=Aria DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_bin PAGE_CHECKSUM=1 TRANSACTIONAL=0 COMMENT='Statistics on Columns' show create table index_stats; diff --git a/mysql-test/main/system_mysql_db_fix40123.result b/mysql-test/main/system_mysql_db_fix40123.result index 127e2c49642f5..ec972058d54e0 100644 --- a/mysql-test/main/system_mysql_db_fix40123.result +++ b/mysql-test/main/system_mysql_db_fix40123.result @@ -272,8 +272,8 @@ column_stats CREATE TABLE `column_stats` ( `avg_length` decimal(12,4) DEFAULT NULL, `avg_frequency` decimal(12,4) DEFAULT NULL, `hist_size` tinyint(3) unsigned DEFAULT NULL, - `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB') COLLATE utf8mb3_bin DEFAULT NULL, - `histogram` varbinary(255) DEFAULT NULL, + `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON') COLLATE utf8mb3_bin DEFAULT NULL, + `histogram` blob DEFAULT NULL, PRIMARY KEY (`db_name`,`table_name`,`column_name`) ) ENGINE=Aria DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_bin PAGE_CHECKSUM=1 TRANSACTIONAL=0 COMMENT='Statistics on Columns' show create table index_stats; diff --git a/mysql-test/main/system_mysql_db_fix50030.result b/mysql-test/main/system_mysql_db_fix50030.result index fd0074c7f54f5..4e038849cf672 100644 --- a/mysql-test/main/system_mysql_db_fix50030.result +++ b/mysql-test/main/system_mysql_db_fix50030.result @@ -276,8 +276,8 @@ column_stats CREATE TABLE `column_stats` ( `avg_length` decimal(12,4) DEFAULT NULL, `avg_frequency` decimal(12,4) DEFAULT NULL, `hist_size` tinyint(3) unsigned DEFAULT NULL, - `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB') COLLATE utf8mb3_bin DEFAULT NULL, - `histogram` varbinary(255) DEFAULT NULL, + `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON') COLLATE utf8mb3_bin DEFAULT NULL, + `histogram` blob DEFAULT NULL, PRIMARY KEY (`db_name`,`table_name`,`column_name`) ) ENGINE=Aria DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_bin PAGE_CHECKSUM=1 TRANSACTIONAL=0 COMMENT='Statistics on Columns' show create table index_stats; diff --git a/mysql-test/main/system_mysql_db_fix50117.result b/mysql-test/main/system_mysql_db_fix50117.result index a7d705c4158c1..7d540477d51ac 100644 --- a/mysql-test/main/system_mysql_db_fix50117.result +++ b/mysql-test/main/system_mysql_db_fix50117.result @@ -256,8 +256,8 @@ column_stats CREATE TABLE `column_stats` ( `avg_length` decimal(12,4) DEFAULT NULL, `avg_frequency` decimal(12,4) DEFAULT NULL, `hist_size` tinyint(3) unsigned DEFAULT NULL, - `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB') COLLATE utf8mb3_bin DEFAULT NULL, - `histogram` varbinary(255) DEFAULT NULL, + `hist_type` enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON') COLLATE utf8mb3_bin DEFAULT NULL, + `histogram` blob DEFAULT NULL, PRIMARY KEY (`db_name`,`table_name`,`column_name`) ) ENGINE=Aria DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_bin PAGE_CHECKSUM=1 TRANSACTIONAL=0 COMMENT='Statistics on Columns' show create table index_stats; diff --git a/scripts/mysql_system_tables.sql b/scripts/mysql_system_tables.sql index cee47a0e5d656..e31f3372b5f7e 100644 --- a/scripts/mysql_system_tables.sql +++ b/scripts/mysql_system_tables.sql @@ -314,7 +314,7 @@ DROP TABLE tmp_proxies_priv; CREATE TABLE IF NOT EXISTS table_stats (db_name varchar(64) NOT NULL, table_name varchar(64) NOT NULL, cardinality bigint(21) unsigned DEFAULT NULL, PRIMARY KEY (db_name,table_name) ) engine=Aria transactional=0 CHARACTER SET utf8 COLLATE utf8_bin comment='Statistics on Tables'; -CREATE TABLE IF NOT EXISTS column_stats (db_name varchar(64) NOT NULL, table_name varchar(64) NOT NULL, column_name varchar(64) NOT NULL, min_value varbinary(255) DEFAULT NULL, max_value varbinary(255) DEFAULT NULL, nulls_ratio decimal(12,4) DEFAULT NULL, avg_length decimal(12,4) DEFAULT NULL, avg_frequency decimal(12,4) DEFAULT NULL, hist_size tinyint unsigned, hist_type enum('SINGLE_PREC_HB','DOUBLE_PREC_HB'), histogram varbinary(255), PRIMARY KEY (db_name,table_name,column_name) ) engine=Aria transactional=0 CHARACTER SET utf8 COLLATE utf8_bin comment='Statistics on Columns'; +CREATE TABLE IF NOT EXISTS column_stats (db_name varchar(64) NOT NULL, table_name varchar(64) NOT NULL, column_name varchar(64) NOT NULL, min_value varbinary(255) DEFAULT NULL, max_value varbinary(255) DEFAULT NULL, nulls_ratio decimal(12,4) DEFAULT NULL, avg_length decimal(12,4) DEFAULT NULL, avg_frequency decimal(12,4) DEFAULT NULL, hist_size tinyint unsigned, hist_type enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON'), histogram blob, PRIMARY KEY (db_name,table_name,column_name) ) engine=Aria transactional=0 CHARACTER SET utf8 COLLATE utf8_bin comment='Statistics on Columns'; CREATE TABLE IF NOT EXISTS index_stats (db_name varchar(64) NOT NULL, table_name varchar(64) NOT NULL, index_name varchar(64) NOT NULL, prefix_arity int(11) unsigned NOT NULL, avg_frequency decimal(12,4) DEFAULT NULL, PRIMARY KEY (db_name,table_name,index_name,prefix_arity) ) engine=Aria transactional=0 CHARACTER SET utf8 COLLATE utf8_bin comment='Statistics on Indexes'; diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 7e8ff667e75a6..072d775dcaa7d 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -496,7 +496,7 @@ String *Item_func_from_base64::val_str(String *str) const char *histogram_types[] = - {"SINGLE_PREC_HB", "DOUBLE_PREC_HB", 0}; + {"SINGLE_PREC_HB", "DOUBLE_PREC_HB", "JSON", 0}; static TYPELIB hystorgam_types_typelib= { array_elements(histogram_types), "histogram_types", diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 1f034f490c8e2..e4fee5ae7cd08 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -32,6 +32,10 @@ #include "uniques.h" #include "sql_show.h" #include "sql_partition.h" +#include "my_json_writer.h" + +#include +#include /* The system variable 'use_stat_tables' can take one of the @@ -57,8 +61,21 @@ the collected statistics in the persistent statistical tables only when the value of the variable 'use_stat_tables' is not equal to "never". -*/ - +*/ + +/* + * json_get_array_items expects a JSON array as argument, + * and pushes the elements of the array into the `container` vector. + * It only works if all the elements in the original JSON array + * are scalar values (i.e., strings, numbers, true or false), and returns JSV_BAD_JSON if: + * the original JSON is not an array OR the JSON array contains non-scalar elements. + */ +bool json_get_array_items(const char *json, const char *json_end, int *value_type, std::vector &container); + +std::vector parse_histogram_from_json(const char *json); + +void test_parse_histogram_from_json(); + /* Currently there are only 3 persistent statistical tables */ static const uint STATISTICS_TABLES= 3; @@ -178,12 +195,12 @@ TABLE_FIELD_TYPE column_stat_fields[COLUMN_STAT_N_FIELDS] = }, { { STRING_WITH_LEN("hist_type") }, - { STRING_WITH_LEN("enum('SINGLE_PREC_HB','DOUBLE_PREC_HB')") }, + { STRING_WITH_LEN("enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON')") }, { STRING_WITH_LEN("utf8mb3") } }, { { STRING_WITH_LEN("histogram") }, - { STRING_WITH_LEN("varbinary(255)") }, + { STRING_WITH_LEN("blob") }, { NULL, 0 } } }; @@ -307,7 +324,7 @@ class Column_statistics_collected :public Column_statistics inline void init(THD *thd, Field * table_field); inline bool add(); - inline void finish(ha_rows rows, double sample_fraction); + inline void finish(MEM_ROOT *mem_root, ha_rows rows, double sample_fraction); inline void cleanup(); }; @@ -1064,15 +1081,23 @@ class Column_stat: public Stat_table stat_field->store(stats->get_avg_frequency()); break; case COLUMN_STAT_HIST_SIZE: - stat_field->store(stats->histogram.get_size()); + // Note: this is dumb. the histogram size is stored with the + // histogram! + stat_field->store(stats->histogram_? + stats->histogram_->get_size() : 0); break; case COLUMN_STAT_HIST_TYPE: - stat_field->store(stats->histogram.get_type() + 1); + if (stats->histogram_) + stat_field->store(stats->histogram_->get_type() + 1); + else + stat_field->set_null(); break; case COLUMN_STAT_HISTOGRAM: - stat_field->store((char *)stats->histogram.get_values(), - stats->histogram.get_size(), &my_charset_bin); - break; + if (stats->histogram_) + stats->histogram_->serialize(stat_field); + else + stat_field->set_null(); + break; } } } @@ -1100,6 +1125,7 @@ class Column_stat: public Stat_table void get_stat_values() { table_field->read_stats->set_all_nulls(); + table_field->read_stats->histogram_type_on_disk= INVALID_HISTOGRAM; if (table_field->read_stats->min_value) table_field->read_stats->min_value->set_null(); @@ -1111,7 +1137,7 @@ class Column_stat: public Stat_table char buff[MAX_FIELD_WIDTH]; String val(buff, sizeof(buff), &my_charset_bin); - for (uint i= COLUMN_STAT_MIN_VALUE; i <= COLUMN_STAT_HIST_TYPE; i++) + for (uint i= COLUMN_STAT_MIN_VALUE; i <= COLUMN_STAT_HISTOGRAM; i++) { Field *stat_field= stat_table->field[i]; @@ -1155,13 +1181,22 @@ class Column_stat: public Stat_table table_field->read_stats->set_avg_frequency(stat_field->val_real()); break; case COLUMN_STAT_HIST_SIZE: - table_field->read_stats->histogram.set_size(stat_field->val_int()); + //TODO: ignore this. The size is a part of histogram! + //table_field->read_stats->histogram.set_size(stat_field->val_int()); break; case COLUMN_STAT_HIST_TYPE: - Histogram_type hist_type= (Histogram_type) (stat_field->val_int() - - 1); - table_field->read_stats->histogram.set_type(hist_type); - break; + // TODO: save this next to histogram. + // For some reason, the histogram itself is read in + // read_histograms_for_table + { + Histogram_type hist_type= (Histogram_type) (stat_field->val_int() - + 1); + table_field->read_stats->histogram_type_on_disk= hist_type; + break; + } + case COLUMN_STAT_HISTOGRAM: + //TODO: if stat_field->length() == 0 then histogram_type_on_disk is set to INVALID_HISTOGRAM + break; } } } @@ -1184,7 +1219,7 @@ class Column_stat: public Stat_table of read_stats->histogram. */ - void get_histogram_value() + Histogram * load_histogram(MEM_ROOT *mem_root) { if (find_stat()) { @@ -1194,13 +1229,54 @@ class Column_stat: public Stat_table Field *stat_field= stat_table->field[fldno]; table_field->read_stats->set_not_null(fldno); stat_field->val_str(&val); - memcpy(table_field->read_stats->histogram.get_values(), - val.ptr(), table_field->read_stats->histogram.get_size()); + // histogram-todo: here, create the histogram of appropriate type. + Histogram *hist= new (mem_root) Histogram(); + if (!hist->parse(mem_root, table_field->read_stats->histogram_type_on_disk, + (const uchar*)val.ptr(), val.length())) + { + table_field->read_stats->histogram_= hist; + return hist; + } + //memcpy(table_field->read_stats->histogram_.get_values(), + // val.ptr(), table_field->read_stats->histogram.get_size()); } + return NULL; } - }; +bool Histogram::parse(MEM_ROOT *mem_root, Histogram_type type_arg, const uchar *ptr_arg, uint size_arg) +{ + // Just copy the data + size = (uint8) size_arg; + type = type_arg; + values = (uchar*)alloc_root(mem_root, size_arg); + memcpy(values, ptr_arg, size_arg); + return false; +} + + +/* + Save the histogram data info a table field. +*/ +void Histogram::serialize(Field *field) +{ + if (get_type() == JSON) + { + field->store((char*)get_values(), strlen((char*)get_values()), + &my_charset_bin); + } + else + field->store((char*)get_values(), get_size(), &my_charset_bin); +} + +void Histogram::init_for_collection(MEM_ROOT *mem_root, + Histogram_type htype_arg, + ulonglong size_arg) +{ + type= htype_arg; + values = (uchar*)alloc_root(mem_root, size_arg); + size= (uint8) size_arg; +} /* An object of the class Index_stat is created to read statistical @@ -1519,6 +1595,7 @@ class Stat_table_write_iter class Histogram_builder { +protected: Field *column; /* table field for which the histogram is built */ uint col_length; /* size of this field */ ha_rows records; /* number of records the histogram is built for */ @@ -1540,7 +1617,7 @@ class Histogram_builder Column_statistics *col_stats= col->collected_stats; min_value= col_stats->min_value; max_value= col_stats->max_value; - histogram= &col_stats->histogram; + histogram= col_stats->histogram_; hist_width= histogram->get_width(); bucket_capacity= (double) records / (hist_width + 1); curr_bucket= 0; @@ -1549,13 +1626,15 @@ class Histogram_builder count_distinct_single_occurence= 0; } + virtual ~Histogram_builder() = default; + ulonglong get_count_distinct() const { return count_distinct; } ulonglong get_count_single_occurence() const { return count_distinct_single_occurence; } - int next(void *elem, element_count elem_cnt) + virtual int next(void *elem, element_count elem_cnt) { count_distinct++; if (elem_cnt == 1) @@ -1567,7 +1646,7 @@ class Histogram_builder { column->store_field_value((uchar *) elem, col_length); histogram->set_value(curr_bucket, - column->pos_in_interval(min_value, max_value)); + column->pos_in_interval(min_value, max_value)); curr_bucket++; while (curr_bucket != hist_width && count > bucket_capacity * (curr_bucket + 1)) @@ -1580,6 +1659,138 @@ class Histogram_builder } }; +class Histogram_builder_json : public Histogram_builder +{ +std::vector bucket_bounds = {}; + +public: + Histogram_builder_json(Field *col, uint col_len, ha_rows rows) + : Histogram_builder(col, col_len, rows) + { + Column_statistics *col_stats= col->collected_stats; + min_value= col_stats->min_value; + max_value= col_stats->max_value; + histogram= col_stats->histogram_; + hist_width= histogram->get_width(); + bucket_capacity= (double) records / (hist_width + 1); + curr_bucket= 0; + count= 0; + count_distinct= 0; + count_distinct_single_occurence= 0; + } + + ~Histogram_builder_json() override = default; + + int next(void *elem, element_count elem_cnt) override + { + count_distinct++; + if (elem_cnt == 1) + count_distinct_single_occurence++; + count+= elem_cnt; + if (curr_bucket == hist_width) + return 0; + if (count > bucket_capacity * (curr_bucket + 1)) + { + column->store_field_value((uchar *) elem, col_length); + StringBuffer val; + column->val_str(&val); + bucket_bounds.emplace_back(val.c_ptr()); + curr_bucket++; + } + return 0; + } + + void build_json_from_histogram() { + Json_writer *writer = new Json_writer(); + writer->start_array(); + for(auto& value: bucket_bounds) { + writer->add_str(value.c_str()); + } + writer->end_array(); + histogram->set_size(bucket_bounds.size()); + Binary_string *json_string = (Binary_string *) writer->output.get_string(); + histogram->set_values((uchar *) json_string->c_ptr()); + + std::vector buckets = parse_histogram_from_json(json_string->c_ptr()); + printf("%zu", buckets.size()); + + test_parse_histogram_from_json(); + } +}; + +void test_parse_histogram_from_json() +{ + std::vector bucket = {}; + std::string json; + std::string tests[7] = { + R"(["aabbb", "ccccdd", "eeefff"])", + R"(["aabbb", "ccc{}dd", "eeefff"])", + R"(["aabbb", {"a": "b"}, "eeefff"])", + R"({})", + R"([1,2,3, null])", + R"([null])", + R"([])" + }; + + for(const auto& test : tests) { + json = test; + bucket = parse_histogram_from_json(json.c_str()); + } +} + +std::vector parse_histogram_from_json(const char *json) +{ + std::vector hist_buckets= {}; + int vt; + bool result = json_get_array_items(json, json + strlen(json), &vt, hist_buckets); + fprintf(stderr,"==============\n"); + fprintf(stderr,"histogram: %s\n", json); + fprintf(stderr, "json_get_array_items() returned %s\n", result ? "true" : "false"); + fprintf(stderr, "value type after json_get_array_items() is %d\n", vt); + fprintf(stderr, " JSV_BAD_JSON=%d, JSON_VALUE_ARRAY=%d\n", (int)JSV_BAD_JSON, (int)JSON_VALUE_ARRAY); + fprintf(stderr, "hist_buckets.size()=%zu\n", hist_buckets.size()); + + return hist_buckets; +} + +bool json_get_array_items(const char *json, const char *json_end, int *value_type, std::vector &container) { + json_engine_t je; + int vl; + const char *v; + + json_scan_start(&je, &my_charset_utf8mb4_bin, (const uchar *)json, (const uchar *)json_end); + + if (json_read_value(&je) || je.value_type != JSON_VALUE_ARRAY) + { + *value_type = JSV_BAD_JSON; + return false; + } + *value_type = je.value_type; + + std::string val; + while(!json_scan_next(&je)) + { + switch(je.state) + { + case JST_VALUE: + *value_type = json_smart_read_value(&je, &v, &vl); + if (je.value_type != JSON_VALUE_STRING && + je.value_type != JSON_VALUE_NUMBER && + je.value_type != JSON_VALUE_TRUE && + je.value_type != JSON_VALUE_FALSE) + { + *value_type = JSV_BAD_JSON; + return false; + } + val = std::string(v, vl); + container.emplace_back(val); + case JST_ARRAY_END: + break; + } + } + + return true; +} C_MODE_START @@ -1589,6 +1800,12 @@ int histogram_build_walk(void *elem, element_count elem_cnt, void *arg) return hist_builder->next(elem, elem_cnt); } +int json_histogram_build_walk(void *elem, element_count elem_cnt, void *arg) +{ + Histogram_builder_json *hist_builder= (Histogram_builder_json *) arg; + return hist_builder->next(elem, elem_cnt); +} + static int count_distinct_single_occurence_walk(void *elem, @@ -1692,12 +1909,25 @@ class Count_distinct_field: public Sql_alloc @brief Calculate a histogram of the tree */ - void walk_tree_with_histogram(ha_rows rows) + void walk_tree_with_histogram(ha_rows rows) { - Histogram_builder hist_builder(table_field, tree_key_length, rows); - tree->walk(table_field->table, histogram_build_walk, (void *) &hist_builder); - distincts= hist_builder.get_count_distinct(); - distincts_single_occurence= hist_builder.get_count_single_occurence(); + if (table_field->collected_stats->histogram_->get_type() == JSON) + { + Histogram_builder_json hist_builder(table_field, tree_key_length, rows); + tree->walk(table_field->table, json_histogram_build_walk, + (void *) &hist_builder); + hist_builder.build_json_from_histogram(); + distincts= hist_builder.get_count_distinct(); + distincts_single_occurence= hist_builder.get_count_single_occurence(); + } + else + { + Histogram_builder hist_builder(table_field, tree_key_length, rows); + tree->walk(table_field->table, histogram_build_walk, + (void *) &hist_builder); + distincts= hist_builder.get_count_distinct(); + distincts_single_occurence= hist_builder.get_count_single_occurence(); + } } ulonglong get_count_distinct() @@ -1714,18 +1944,19 @@ class Count_distinct_field: public Sql_alloc @brief Get the size of the histogram in bytes built for table_field */ + /* uint get_hist_size() { return table_field->collected_stats->histogram.get_size(); - } + }*/ /* @brief Get the pointer to the histogram built for table_field */ - uchar *get_histogram() + Histogram *get_histogram() { - return table_field->collected_stats->histogram.get_values(); + return table_field->collected_stats->histogram_; } }; @@ -2124,7 +2355,7 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) uint key_parts= table->s->ext_key_parts; ulonglong *idx_avg_frequency= (ulonglong*) alloc_root(&table->mem_root, sizeof(ulonglong) * key_parts); - +/* uint hist_size= thd->variables.histogram_size; Histogram_type hist_type= (Histogram_type) (thd->variables.histogram_type); uchar *histogram= NULL; @@ -2135,16 +2366,16 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) bzero(histogram, hist_size * columns); } - - if (!table_stats || !column_stats || !index_stats || !idx_avg_frequency || - (hist_size && !histogram)) +*/ + if (!table_stats || !column_stats || !index_stats || !idx_avg_frequency) + //|| (hist_size && !histogram)) DBUG_RETURN(1); table->collected_stats= table_stats; table_stats->column_stats= column_stats; table_stats->index_stats= index_stats; table_stats->idx_avg_frequency= idx_avg_frequency; - table_stats->histograms= histogram; + //table_stats->histograms= histogram; memset(column_stats, 0, sizeof(Column_statistics) * columns); @@ -2152,10 +2383,12 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) { if (bitmap_is_set(table->read_set, (*field_ptr)->field_index)) { + column_stats->histogram_ = NULL; + /* column_stats->histogram.set_size(hist_size); column_stats->histogram.set_type(hist_type); column_stats->histogram.set_values(histogram); - histogram+= hist_size; + histogram+= hist_size;*/ (*field_ptr)->collected_stats= column_stats++; } } @@ -2374,6 +2607,25 @@ bool Column_statistics_collected::add() } +/* + Create an empty Histogram object from histogram_type. + + Note: it is not yet clear whether collection-time histogram should be the same + as lookup-time histogram. At the moment, they are. +*/ + +Histogram* get_histogram_by_type(MEM_ROOT *mem_root, Histogram_type hist_type) { + switch (hist_type) { + case SINGLE_PREC_HB: + case DOUBLE_PREC_HB: + case JSON: + return new Histogram(); + default: + DBUG_ASSERT(0); + } + return NULL; +}; + /** @brief Get the results of aggregation when collecting the statistics on a column @@ -2383,7 +2635,7 @@ bool Column_statistics_collected::add() */ inline -void Column_statistics_collected::finish(ha_rows rows, double sample_fraction) +void Column_statistics_collected::finish(MEM_ROOT *mem_root, ha_rows rows, double sample_fraction) { double val; @@ -2401,10 +2653,19 @@ void Column_statistics_collected::finish(ha_rows rows, double sample_fraction) } if (count_distinct) { - uint hist_size= count_distinct->get_hist_size(); + //uint hist_size= count_distinct->get_hist_size(); + uint hist_size= current_thd->variables.histogram_size; + Histogram_type hist_type= (Histogram_type) (current_thd->variables.histogram_type); + bool have_histogram= false; + if (hist_size != 0 && hist_type != INVALID_HISTOGRAM) + { + have_histogram= true; + histogram_= new Histogram; + histogram_->init_for_collection(mem_root, hist_type, hist_size); + } /* Compute cardinality statistics and optionally histogram. */ - if (hist_size == 0) + if (!have_histogram) count_distinct->walk_tree(); else count_distinct->walk_tree_with_histogram(rows - nulls); @@ -2442,13 +2703,14 @@ void Column_statistics_collected::finish(ha_rows rows, double sample_fraction) set_not_null(COLUMN_STAT_AVG_FREQUENCY); } else - hist_size= 0; - histogram.set_size(hist_size); + have_histogram= false ; // TODO: need this? + //histogram.set_size(hist_size); set_not_null(COLUMN_STAT_HIST_SIZE); - if (hist_size && distincts) + if (have_histogram && distincts) { set_not_null(COLUMN_STAT_HIST_TYPE); - histogram.set_values(count_distinct->get_histogram()); + //histogram.set_values(count_distinct->get_histogram()); + histogram_= count_distinct->get_histogram(); set_not_null(COLUMN_STAT_HISTOGRAM); } delete count_distinct; @@ -2710,7 +2972,7 @@ int collect_statistics_for_table(THD *thd, TABLE *table) continue; bitmap_set_bit(table->write_set, table_field->field_index); if (!rc) - table_field->collected_stats->finish(rows, sample_fraction); + table_field->collected_stats->finish(&table->mem_root, rows, sample_fraction); else table_field->collected_stats->cleanup(); } @@ -2916,16 +3178,19 @@ int read_statistics_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) /* Read statistics from the statistical table column_stats */ stat_table= stat_tables[COLUMN_STAT].table; - ulong total_hist_size= 0; + //ulong total_hist_size= 0; + bool have_histograms= false; Column_stat column_stat(stat_table, table); for (field_ptr= table_share->field; *field_ptr; field_ptr++) { table_field= *field_ptr; column_stat.set_key_fields(table_field); column_stat.get_stat_values(); - total_hist_size+= table_field->read_stats->histogram.get_size(); + //total_hist_size+= table_field->read_stats->histogram.get_size(); + if (table_field->read_stats->histogram_type_on_disk != INVALID_HISTOGRAM) + have_histograms= true; } - table_share->stats_cb.total_hist_size= total_hist_size; + table_share->stats_cb.total_hist_size= have_histograms? 1:0; // total_hist_size /* Read statistics from the statistical table index_stats */ stat_table= stat_tables[INDEX_STAT].table; @@ -3062,28 +3327,36 @@ int read_histograms_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) { TABLE_STATISTICS_CB *stats_cb= &table->s->stats_cb; DBUG_ENTER("read_histograms_for_table"); - + + // histograms-todo: why do we use synchronization here, when we load + // histogram for the TABLE object, not TABLE_SHARE? + // is it because of the use of stats_cb->mem_root? if (stats_cb->start_histograms_load()) { - uchar *histogram= (uchar *) alloc_root(&stats_cb->mem_root, - stats_cb->total_hist_size); + //uchar *histogram= (uchar *) alloc_root(&stats_cb->mem_root, + // stats_cb->total_hist_size); + /* if (!histogram) { stats_cb->abort_histograms_load(); DBUG_RETURN(1); } - memset(histogram, 0, stats_cb->total_hist_size); + */ + //memset(histogram, 0, stats_cb->total_hist_size); Column_stat column_stat(stat_tables[COLUMN_STAT].table, table); for (Field **field_ptr= table->s->field; *field_ptr; field_ptr++) { Field *table_field= *field_ptr; - if (uint hist_size= table_field->read_stats->histogram.get_size()) + //if (uint hist_size= table_field->read_stats->histogram.get_size()) + if (table_field->read_stats->histogram_type_on_disk != INVALID_HISTOGRAM) { column_stat.set_key_fields(table_field); - table_field->read_stats->histogram.set_values(histogram); - column_stat.get_histogram_value(); - histogram+= hist_size; + //table_field->read_stats->histogram.set_values(histogram); + + table_field->read_stats->histogram_= + column_stat.load_histogram(&stats_cb->mem_root); + //histogram+= hist_size; } } stats_cb->end_histograms_load(); @@ -3775,8 +4048,8 @@ double get_column_range_cardinality(Field *field, if (avg_frequency > 1.0 + 0.000001 && col_stats->min_max_values_are_provided()) { - Histogram *hist= &col_stats->histogram; - if (hist->is_usable(thd)) + Histogram *hist= col_stats->histogram_; + if (hist && hist->is_usable(thd)) { store_key_image_to_rec(field, (uchar *) min_endp->key, field->key_length()); @@ -3819,8 +4092,8 @@ double get_column_range_cardinality(Field *field, else max_mp_pos= 1.0; - Histogram *hist= &col_stats->histogram; - if (hist->is_usable(thd)) + Histogram *hist= col_stats->histogram_; + if (hist && hist->is_usable(thd)) sel= hist->range_selectivity(min_mp_pos, max_mp_pos); else sel= (max_mp_pos - min_mp_pos); diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 35b3aa33accf5..178bc11a2784f 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -42,7 +42,9 @@ typedef enum enum_histogram_type { SINGLE_PREC_HB, - DOUBLE_PREC_HB + DOUBLE_PREC_HB, + JSON, + INVALID_HISTOGRAM } Histogram_type; enum enum_stat_tables @@ -140,8 +142,50 @@ double get_column_range_cardinality(Field *field, bool is_stat_table(const LEX_CSTRING *db, LEX_CSTRING *table); bool is_eits_usable(Field* field); -class Histogram +/* + Common base for all histograms +*/ +class Histogram_base : public Sql_alloc +{ +public: + virtual bool parse(MEM_ROOT *mem_root, Histogram_type type_arg, + const uchar *ptr, uint size)= 0; + virtual void serialize(Field *to_field)= 0; + + virtual Histogram_type get_type()=0; + + // Legacy: return the size of the histogram on disk. + // This will be stored in mysql.column_stats.hist_size column. + // Newer, JSON-based histograms may return 0. + virtual uint get_size()=0; + + virtual ~Histogram_base(){} +}; + +class Histogram : public Histogram_base { +public: + bool parse(MEM_ROOT *mem_root, Histogram_type type_arg, + const uchar *ptr_arg, uint size_arg) override; + void serialize(Field *to_field) override; + Histogram_type get_type() override { return type; } + + uint get_size() override { return (uint) size; } + + // returns number of buckets in the histogram + uint get_width() + { + switch (type) { + case SINGLE_PREC_HB: + case JSON: + return size; + case DOUBLE_PREC_HB: + return size / 2; + default: + DBUG_ASSERT(0); + } + return 0; + } private: Histogram_type type; @@ -152,34 +196,27 @@ class Histogram { switch (type) { case SINGLE_PREC_HB: + case JSON: return ((uint) (1 << 8) - 1); case DOUBLE_PREC_HB: return ((uint) (1 << 16) - 1); + default: + DBUG_ASSERT(0); } return 1; } -public: - uint get_width() - { - switch (type) { - case SINGLE_PREC_HB: - return size; - case DOUBLE_PREC_HB: - return size / 2; - } - return 0; - } - -private: uint get_value(uint i) { DBUG_ASSERT(i < get_width()); switch (type) { case SINGLE_PREC_HB: + case JSON: return (uint) (((uint8 *) values)[i]); case DOUBLE_PREC_HB: return (uint) uint2korr(values + i * 2); + default: + DBUG_ASSERT(0); } return 0; } @@ -223,19 +260,13 @@ class Histogram return i; } -public: - - uint get_size() { return (uint) size; } - - Histogram_type get_type() { return type; } - uchar *get_values() { return (uchar *) values; } +public: + void init_for_collection(MEM_ROOT *mem_root, Histogram_type htype_arg, ulonglong size); - void set_size (ulonglong sz) { size= (uint8) sz; } - - void set_type (Histogram_type t) { type= t; } - + // Note: these two are used only for saving the JSON text: void set_values (uchar *vals) { values= (uchar *) vals; } + void set_size (ulonglong sz) { size= (uint8) sz; } bool is_available() { return get_size() > 0 && get_values(); } @@ -253,24 +284,32 @@ class Histogram void set_value(uint i, double val) { switch (type) { - case SINGLE_PREC_HB: + case SINGLE_PREC_HB: + case JSON: ((uint8 *) values)[i]= (uint8) (val * prec_factor()); return; case DOUBLE_PREC_HB: int2store(values + i * 2, val * prec_factor()); return; + default: + DBUG_ASSERT(0); + return; } } void set_prev_value(uint i) { switch (type) { - case SINGLE_PREC_HB: + case SINGLE_PREC_HB: + case JSON: ((uint8 *) values)[i]= ((uint8 *) values)[i-1]; return; case DOUBLE_PREC_HB: int2store(values + i * 2, uint2korr(values + i * 2 - 2)); return; + default: + DBUG_ASSERT(0); + return; } } @@ -308,7 +347,7 @@ class Table_statistics /* Array of records per key for index prefixes */ ulonglong *idx_avg_frequency; - uchar *histograms; /* Sequence of histograms */ + //uchar *histograms; /* Sequence of histograms */ }; @@ -371,7 +410,8 @@ class Column_statistics public: - Histogram histogram; + Histogram_type histogram_type_on_disk; + Histogram *histogram_; uint32 no_values_provided_bitmap() { diff --git a/sql/table.h b/sql/table.h index 2e074abcea0ef..f557f4ca59e9b 100644 --- a/sql/table.h +++ b/sql/table.h @@ -679,7 +679,15 @@ class TABLE_STATISTICS_CB public: MEM_ROOT mem_root; /* MEM_ROOT to allocate statistical data for the table */ Table_statistics *table_stats; /* Structure to access the statistical data */ - ulong total_hist_size; /* Total size of all histograms */ + + /* + Total size of all histograms. A value of 0 means historams are not present, + and histograms_are_ready() can finish sooner. + + Currently we just set it to 1 when we expect to load histograms. + histogram-todo: rename this or even remove? + */ + ulong total_hist_size; bool histograms_are_ready() const { diff --git a/strings/json_lib.c b/strings/json_lib.c index 49f29903ed518..efa716461d453 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -1868,7 +1868,7 @@ int json_path_compare(const json_path_t *a, const json_path_t *b, } -static enum json_types smart_read_value(json_engine_t *je, +enum json_types json_smart_read_value(json_engine_t *je, const char **value, int *value_len) { if (json_read_value(je)) @@ -1909,7 +1909,7 @@ enum json_types json_type(const char *js, const char *js_end, json_scan_start(&je, &my_charset_utf8mb4_bin,(const uchar *) js, (const uchar *) js_end); - return smart_read_value(&je, value, value_len); + return json_smart_read_value(&je, value, value_len); } @@ -1933,7 +1933,7 @@ enum json_types json_get_array_item(const char *js, const char *js_end, { case JST_VALUE: if (c_item == n_item) - return smart_read_value(&je, value, value_len); + return json_smart_read_value(&je, value, value_len); if (json_skip_key(&je)) goto err_return; @@ -1952,7 +1952,6 @@ enum json_types json_get_array_item(const char *js, const char *js_end, return JSV_BAD_JSON; } - /** Simple json lookup for a value by the key. Expects JSON object. @@ -1998,7 +1997,7 @@ enum json_types json_get_object_key(const char *js, const char *js_end, json_string_set_str(&key_name, (const uchar *) key, (const uchar *) key_end); if (json_key_matches(&je, &key_name)) - return smart_read_value(&je, value, value_len); + return json_smart_read_value(&je, value, value_len); if (json_skip_key(&je)) goto err_return; @@ -2029,6 +2028,7 @@ enum json_types json_get_object_nkey(const char *js __attribute__((unused)), } + /** Check if json is valid (well-formed) @retval 0 - success, json is well-formed