MDEV-29445: Reimplement SET GLOBAL innodb_buffer_pool_size

extra/mariabackup/innobackupex.cc

@@ -44,8 +44,8 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA
 #include <string.h>
 #include <mysql.h>
 #include <my_dir.h>
-#include <ut0mem.h>
 #include <os0file.h>
+#include "buf0buf.h"
 #include <srv0start.h>
 #include <algorithm>
 #include <mysqld.h>
@@ -594,8 +594,9 @@ static struct my_option ibx_long_options[] =
 	 "--apply-log.",
 	 (uchar*) &ibx_xtrabackup_use_memory,
 	 (uchar*) &ibx_xtrabackup_use_memory,
-	 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0,
-	 1024*1024L, 0},
+	 0, GET_LL, REQUIRED_ARG, 96 << 20,
+	 innodb_buffer_pool_extent_size, SIZE_T_MAX, 0,
+	 innodb_buffer_pool_extent_size, 0},
 
 	{"innodb-force-recovery", OPT_INNODB_FORCE_RECOVERY,
 	 "This option starts up the embedded InnoDB instance in crash "

extra/mariabackup/xtrabackup.cc

@@ -1418,8 +1418,8 @@ struct my_option xb_client_options[]= {
      "The value is used in place of innodb_buffer_pool_size. "
      "This option is only relevant when the --prepare option is specified.",
      (G_PTR *) &xtrabackup_use_memory, (G_PTR *) &xtrabackup_use_memory, 0,
-     GET_LL, REQUIRED_ARG, 100 * 1024 * 1024L, 1024 * 1024L, LONGLONG_MAX, 0,
-     1024 * 1024L, 0},
+     GET_LL, REQUIRED_ARG, 96 << 20, innodb_buffer_pool_extent_size,
+     SIZE_T_MAX, 0, innodb_buffer_pool_extent_size, 0},
     {"throttle", OPT_XTRA_THROTTLE,
      "limit count of IO operations (pairs of read&write) per second to IOS "
      "values (for '--backup')",
@@ -2479,7 +2479,7 @@ static bool innodb_init_param()
 	}
 
 	srv_sys_space.normalize_size();
-	srv_lock_table_size = 5 * (srv_buf_pool_size >> srv_page_size_shift);
+	srv_lock_table_size = 5 * buf_pool.curr_size();
 
 	/* -------------- Log files ---------------------------*/
 
@@ -2501,11 +2501,8 @@ static bool innodb_init_param()
 
 	srv_adaptive_flushing = FALSE;
 
-        /* We set srv_pool_size here in units of 1 kB. InnoDB internally
-        changes the value so that it becomes the number of database pages. */
-
-	srv_buf_pool_size = (ulint) xtrabackup_use_memory;
-	srv_buf_pool_chunk_unit = srv_buf_pool_size;
+	buf_pool.size_in_bytes_max = size_t(xtrabackup_use_memory);
+	buf_pool.size_in_bytes_requested = buf_pool.size_in_bytes_max;
 
 	srv_n_read_io_threads = (uint) innobase_read_io_threads;
 	srv_n_write_io_threads = (uint) innobase_write_io_threads;

include/my_sys.h

@@ -178,6 +178,9 @@ extern char *my_strndup(PSI_memory_key key, const char *from, size_t length, myf
 
 int my_init_large_pages(my_bool super_large_pages);
 uchar *my_large_malloc(size_t *size, myf my_flags);
+#if defined _WIN32 || defined HAVE_MMAP
+uchar *my_large_virtual_alloc(size_t *size);
+#endif
 void my_large_free(void *ptr, size_t size);
 void my_large_page_truncate(size_t *size);
 

mysql-test/suite/encryption/t/innodb_encrypt_temporary_tables.opt

@@ -1,2 +1,2 @@
---innodb_buffer_pool_size=5M
+--innodb_buffer_pool_size=6M
 --innodb_encrypt_temporary_tables=1

mysql-test/suite/innodb/r/innodb_buffer_pool_fail.result

@@ -1,9 +1,9 @@
-call mtr.add_suppression("InnoDB: Cannot allocate memory for the buffer pool");
+call mtr.add_suppression("InnoDB: Cannot map innodb_buffer_pool_size_max=");
 call mtr.add_suppression("InnoDB: Plugin initialization aborted at srv0start.cc.*");
 call mtr.add_suppression("Plugin 'InnoDB' init function returned error.");
 call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed.");
 #
 # MDEV-25019 memory allocation failures during startup cause server failure in different, confusing ways
 #
 # restart: --debug_dbug=+d,ib_buf_chunk_init_fails
-FOUND 1 /\[ERROR\] InnoDB: Cannot allocate memory for the buffer pool/ in mysqld.1.err
+FOUND 1 /\[ERROR\] InnoDB: Cannot map innodb_buffer_pool_size_max=16m/ in mysqld.1.err

mysql-test/suite/innodb/r/innodb_buffer_pool_resize.result

@@ -1,3 +1,6 @@
+#
+# MDEV-29445: Reorganize buffer pool (and remove chunks)
+#
 set global innodb_adaptive_hash_index=ON;
 select @@innodb_buffer_pool_size;
 @@innodb_buffer_pool_size
@@ -6,27 +9,31 @@ set global innodb_buffer_pool_size = 10485760;
 select @@innodb_buffer_pool_size;
 @@innodb_buffer_pool_size
 10485760
-create table t1 (id int not null, val int not null default '0', primary key (id)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
-create or replace view view0 as select 1 union all select 1;
-set @`v_id` := 0;
-set @`v_val` := 0;
-replace into t1 select (@`v_id` := (@`v_id` + 4) mod 4294967296) as id, (@`v_val` := (@`v_val` + 4) mod 4294967296) as val from view0 v0, view0 v1, view0 v2, view0 v3, view0 v4, view0 v5, view0 v6, view0 v7, view0 v8, view0 v9, view0 v10, view0 v11, view0 v12, view0 v13, view0 v14, view0 v15, view0 v16, view0 v17;
-set global innodb_buffer_pool_size = 64 * 1024 * 1024 + 512 * 1024;
-Warnings:
-Warning	1292	Truncated incorrect innodb_buffer_pool_size value: '67633152'
-select @@innodb_buffer_pool_size;
-@@innodb_buffer_pool_size
-68157440
+create table t1 (id int primary key, val int not null)
+ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+SET STATEMENT foreign_key_checks=0, unique_checks=0 FOR
+INSERT INTO t1 SELECT seq*4,seq*4 FROM seq_1_to_262144;
+set global innodb_buffer_pool_size = 7340032;
 select count(val) from t1;
 count(val)
 262144
 set global innodb_adaptive_hash_index=OFF;
-set global innodb_buffer_pool_size = 25165824;
+set global innodb_buffer_pool_size = 26214400;
+Warnings:
+Warning	1292	Truncated incorrect innodb_buffer_pool_size value: '26214400'
 select @@innodb_buffer_pool_size;
 @@innodb_buffer_pool_size
 25165824
 select count(val) from t1;
 count(val)
 262144
 drop table t1;
-drop view view0;
+SET GLOBAL innodb_max_purge_lag_wait = 0;
+SET @save_pct= @@GLOBAL.innodb_max_dirty_pages_pct;
+SET @save_pct_lwm= @@GLOBAL.innodb_max_dirty_pages_pct_lwm;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm = 0.0;
+SET GLOBAL innodb_max_dirty_pages_pct = 0.0;
+SET GLOBAL innodb_buffer_pool_size = @old_innodb_buffer_pool_size;
+SET GLOBAL innodb_adaptive_hash_index = @old_innodb_adaptive_hash_index;
+SET GLOBAL innodb_max_dirty_pages_pct = @save_pct;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm = @save_pct_lwm;

mysql-test/suite/innodb/r/innodb_buffer_pool_resize_temporary.result

@@ -4,7 +4,14 @@ SET GLOBAL innodb_limit_optimistic_insert_debug=2;
 SET GLOBAL innodb_buffer_pool_size=16777216;
 CREATE TEMPORARY TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
 INSERT INTO t1 SELECT seq FROM seq_1_to_200;
+SET GLOBAL innodb_max_purge_lag_wait=0;
+SET @save_pct= @@GLOBAL.innodb_max_dirty_pages_pct;
+SET @save_pct_lwm= @@GLOBAL.innodb_max_dirty_pages_pct_lwm;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm = 0.0;
+SET GLOBAL innodb_max_dirty_pages_pct = 0.0;
 SET GLOBAL innodb_buffer_pool_size=8388608;
+SET GLOBAL innodb_max_dirty_pages_pct = @save_pct;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm = @save_pct_lwm;
 SELECT COUNT(*),MIN(a),MAX(a) FROM t1;
 COUNT(*)	MIN(a)	MAX(a)
 200	1	200

mysql-test/suite/innodb/r/innodb_status_variables.result

@@ -20,7 +20,6 @@ INNODB_ASYNC_WRITES_WAIT_SLOT_SEC
 INNODB_BACKGROUND_LOG_SYNC
 INNODB_BUFFER_POOL_DUMP_STATUS
 INNODB_BUFFER_POOL_LOAD_STATUS
-INNODB_BUFFER_POOL_RESIZE_STATUS
 INNODB_BUFFER_POOL_LOAD_INCOMPLETE
 INNODB_BUFFER_POOL_PAGES_DATA
 INNODB_BUFFER_POOL_BYTES_DATA

mysql-test/suite/innodb/r/lock_memory_debug.result

@@ -5,9 +5,13 @@ call mtr.add_suppression("\\[Warning\\] InnoDB: Over 67 percent of the buffer po
 CREATE TABLE t1 (col1 INT) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1),(2),(3),(4),(5);
 SET STATEMENT debug_dbug='+d,innodb_skip_lock_bitmap' FOR
-INSERT INTO t1 SELECT a.* FROM t1 a, t1 b, t1 c, t1 d, t1 e, t1 f, t1 g LIMIT 45000;
+INSERT INTO t1 SELECT a.* FROM t1 a, t1 b, t1 c, t1 d, t1 e, t1 f, t1 g;
 ERROR HY000: The total number of locks exceeds the lock table size
 SELECT COUNT(*) FROM t1;
 COUNT(*)
 5
+INSERT INTO t1 SELECT a.* FROM t1 a, t1 b, t1 c, t1 d, t1 e, t1 f, t1 g;
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+78130
 DROP TABLE t1;

mysql-test/suite/innodb/r/log_upgrade_101_flags.result

@@ -1,7 +1,7 @@
 call mtr.add_suppression("InnoDB: The change buffer is corrupted");
 call mtr.add_suppression("InnoDB: Tablespace size stored in header is 768 pages, but the sum of data file sizes is 384 pages");
 call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of file");
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-undo-directory=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-force-recovery=5 --innodb-log-file-size=4m --innodb_page_size=32k --innodb_buffer_pool_size=10M
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-undo-directory=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-force-recovery=5 --innodb-log-file-size=4m --innodb_page_size=32k --innodb_buffer_pool_size=11M
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');

mysql-test/suite/innodb/r/recovery_memory.result

@@ -13,7 +13,7 @@ END LOOP
 connect con1,localhost,root,,,;
 CALL dorepeat();
 connection default;
-# restart: --innodb_buffer_pool_size=5242880
+# restart: --innodb_buffer_pool_size=6m
 DROP TABLE t1;
 DROP PROCEDURE dorepeat;
 #

mysql-test/suite/innodb/r/restart.result

@@ -30,19 +30,6 @@ SELECT * FROM td;
 a
 DROP TABLE tr,tc,td;
 #
-# MDEV-27467 innodb to enfore the minimum innodb_buffer_pool_size in SET (resize) the same as startup
-#
-SELECT @@innodb_buffer_pool_size INTO @innodb_buffer_pool_size_orig;
-SELECT CEILING((256 + 64) * @@innodb_page_size / 1048576) * 1048576 INTO @min_pool_size;
-EXECUTE IMMEDIATE 'SET GLOBAL innodb_buffer_pool_size = ?' USING (@min_pool_size -1);
-ERROR 42000: Variable 'innodb_buffer_pool_size' can't be set to the value of 'WRONG_VALUE'
-SHOW WARNINGS;
-Level	Code	Message
-Warning	1210	innodb_buffer_pool_size must be at least MIN_VAL for innodb_page_size=PAGE_SIZE
-Error	1231	Variable 'innodb_buffer_pool_size' can't be set to the value of 'WRONG_VALUE'
-EXECUTE IMMEDIATE 'SET GLOBAL innodb_buffer_pool_size = ?' USING (@min_pool_size);
-SET GLOBAL innodb_buffer_pool_size = @innodb_buffer_pool_size_orig;
-#
 # MDEV-27882 Innodb - recognise MySQL-8.0 innodb flags and give a specific error message
 #
 FOUND 1 /InnoDB: MySQL-8\.0 tablespace in \./ibdata1/ in attempted_start.err

mysql-test/suite/innodb/r/temp_truncate_freed.result

@@ -1,7 +1,7 @@
 set @old_innodb_buffer_pool_size = @@innodb_buffer_pool_size;
 set @old_immediate_scrub_data_val= @@innodb_immediate_scrub_data_uncompressed;
 SET GLOBAL innodb_immediate_scrub_data_uncompressed=1;
-SET GLOBAL innodb_buffer_pool_size= 16777216;
+SET GLOBAL innodb_buffer_pool_size=16777216;
 CREATE TEMPORARY TABLE t1(c1 MEDIUMTEXT) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (repeat(1,16777215));
 DROP TEMPORARY TABLE t1;

mysql-test/suite/innodb/t/innodb-index-online.opt

@@ -1,5 +1,5 @@
 --loose-innodb-sort-buffer-size=64k
 --loose-innodb-online-alter-log-max-size=128k
---loose-innodb-buffer-pool-size=5M
+--loose-innodb-buffer-pool-size=6M
 --loose-innodb-sys-indexes
 --loose-innodb-sys-fields

mysql-test/suite/innodb/t/innodb-table-online-master.opt

@@ -1 +1 @@
---innodb-sort-buffer-size=64k --innodb-online-alter-log-max-size=512k --innodb-buffer-pool-size=5M
+--innodb-sort-buffer-size=64k --innodb-online-alter-log-max-size=512k --innodb-buffer-pool-size=6M