diff --git a/Docs/README-core-dump-exclusion b/Docs/README-core-dump-exclusion new file mode 100644 index 0000000000000..28d38ddf8a854 --- /dev/null +++ b/Docs/README-core-dump-exclusion @@ -0,0 +1,108 @@ +Exclude things from core dumps (on Linux) +========================================= + +Movitation +---------- + +There are two reasons for wanting to reduce core dump size by excluding +certain memory areas from it: + +1. Reduce IO bandwidth and storage space requirements when dumping core + + This reduces the time it takes to take a core dump and also makes + transfering core files easier. + +2. Exclude sensitive data from core dumps + + At the same time backtrace information should remain complete and memory + regions that may be useful when doing post mortem debugging should be kept. + +Original motivation for implementation was that very large InnoDB instances +on machines with >300GB RAM, and accordingly large innodb buffer pool, were +crashing randomly every few days, and enabling core dumps on all of them was +not an option. Also, even after we finally managed to get hold of a core +dump the file size made it very hard to get it to an environment where it +could be analyzed without causng resource issues. + +Implementation +-------------- + +Linux supports options to madvise(2) that control whether a certain region +should be included in core dumps (MADV_DODUMP) or should be excluded +(MADV_DONTDUMP) starting with kernel version. Such memory regions need to +be page aligned so out of the box the feature is only useful when dealing +with large enough malloc() allocations or when directly dealing with +mmap()ed or shared memory regions directly. + +Certain memory allocations now call the new function + + exclude_from_coredump(void *ptr, size_t size, ulonglong flagmask); + +after successfully aquiring memory. A global variable "core_nodump" +may contain a set of memory regions to exclude. If the given flagmask +matches any of these, or if "core_nodump" has the 'magic' value "MAX", +the given memory region is marked to be excluded from core dumps using +the MADV_DONTDUMP advice. + +So far there's only support for excluding static buffers that stay for +the entire life time of the server process. Support for memory regions +allocated on demand, like those for temporary tables or sort buffers, +or that may be resized dynamically, like the query cache, is possible +but tricky as on freeing memory regions the MADV_DONTDUMP advice has to +be removed by an explicit MADV_DODUMP madvise() for the same region. +Otherwise other later memory allocations that happen to reuse fall in +the previously excluded memory range would accidentially be excluded +from being dumped. So at the time memory is freed it is also necessary +to remove the DONTDUMP advise, and for that not only the base address +is needed but also the original allocation size. As this is not always +explicitly preserved dynamic buffers have been excluded for this first +implementation. + + +Configuration +------------- + +A new configuration variable "core_nodump" has been added that controls +which buffer types should be excluded from core dump. The variable +can't be changed dynamically at run time as exclusions happen right +after allocating memory at startup. + +Possible values for "core_nodump" at this point are: + + Value | Description +--------------------|------------------------------------------------------ + NONE | Default: keep current behavior that dumps everything + INNODB_POOL_BUFFER | Exclude InnoDB pool buffer instances from core dump + MYISAM_KEY_BUFFER | Exclude MyISAM key buffer(s) from core dump + MAX | Apply all possible exclusion + + +Testing +------- + +A mysqld with the following minimalistic configuration + + [mysqld] + innodb_buffer_pool_size=8G + core-file + core-nodump=NONE + +will create a core dump of about 9GB when killed with `kill -11 $mysqld_pid`. + +The same mysqld with INNODB_BUFFER_POOL excluded from dumps only takes +about 900M, as was to be expected with the 8GB pool being excluded: + + [mysqld] + innodb_buffer_pool_size=8G + core-file + core-nodump=INNODB_BUFFER_POOL + +Backtraces can be retrieved sucessfully from both core dumps with gdb +and `thread apply all bt full`. + +As InnoDB buffer pool and Myisam key cache implementations can be considered +stable lack of those buffer contents should not affect core dump post mortem +analysis in any way. This may not be the case with new features like tablespace +encryption yet though, so core dump exclusions should be used carefully in such +contexts. + diff --git a/config.h.cmake b/config.h.cmake index 5e48085a61ae7..f7aa36e7c76de 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -206,6 +206,7 @@ #cmakedefine HAVE_NL_LANGINFO 1 #cmakedefine HAVE_MADVISE 1 #cmakedefine HAVE_DECL_MADVISE 1 +#cmakedefine HAVE_MADV_DONTDUMP 1 #cmakedefine HAVE_DECL_TGOTO 1 #cmakedefine HAVE_DECL_MHA_MAPSIZE_VA 1 #cmakedefine HAVE_MALLINFO 1 diff --git a/configure.cmake b/configure.cmake index 0057c7fb1003e..a19fde9a522b2 100644 --- a/configure.cmake +++ b/configure.cmake @@ -516,6 +516,7 @@ ENDIF() CHECK_SYMBOL_EXISTS(log2 math.h HAVE_LOG2) CHECK_SYMBOL_EXISTS(isnan math.h HAVE_ISNAN) CHECK_SYMBOL_EXISTS(rint math.h HAVE_RINT) +CHECK_SYMBOL_EXISTS(MADV_DONTDUMP "sys/mman.h" HAVE_MADV_DONTDUMP) # isinf() prototype not found on Solaris CHECK_CXX_SOURCE_COMPILES( diff --git a/include/my_core_no_dump.h b/include/my_core_no_dump.h new file mode 100644 index 0000000000000..61c59746b3a14 --- /dev/null +++ b/include/my_core_no_dump.h @@ -0,0 +1,66 @@ +/* + Copyright (c) 2016, 2017, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */ + +/* This is the include file that should be included 'first' in every C file. */ + +#ifndef MY_CORE_NO_DUMP_INCLUDED +#define MY_CORE_NO_DUMP_INCLUDED + +/* + Core dump control options to exclude certain buffers from core dump files + + There are two motivations for excluding things from core dumps: + + * resource utilization: stuff like the InnoDB pool buffer is rarely needed + for post mortem debugging, but on machines with large amounts of memory + just the time and file system space it takes to write a core dump can + become substantial. Large core dump sizes can also become an obstacle + when providing a core dump to a 3rd party for analysis + + * security: certain buffers, especially the InnoDB pool buffer or the + Aria page cache, are likely to contain sensitive user data. Excluding + these from a core dump can improve data security and especially can be + a requirement for being able to pass production server core dumps to + 3rd parties for analysis +*/ + +#include + +#ifdef HAVE_MADV_DONTDUMP + +#include + +/* Core dump exclusion constants */ +#define CORE_NODUMP_NONE (0) +#define CORE_NODUMP_MAX (1 << 1) +#define CORE_NODUMP_INNODB_POOL_BUFFER (1 << 2) + +extern ulong opt_core_nodump; + +inline void exclude_from_coredump(void *ptr, size_t size, ulong flags) +{ + if (opt_core_nodump & (flags | CORE_NODUMP_MAX)) { + madvise(ptr, size, MADV_DONTDUMP); + } +} + +#else /* HAVE_MADV_DONTDUMP */ + +#define exclude_from_coredump(ptr, size, flag) + +#endif /* HAVE_MADV_DONTDUMP */ + +#endif diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 4f83e19f90515..e8e7e93400b53 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -528,6 +528,10 @@ ulong feature_files_opened_with_delayed_keys; ulonglong denied_connections; my_decimal decimal_zero; +#ifdef HAVE_MADV_DONTDUMP +ulong opt_core_nodump; +#endif + /* Maximum length of parameter value which can be set through mysql_send_long_data() call. diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 1a6bb9a4e54f8..39a0fc318db45 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -63,6 +63,8 @@ #include "opt_range.h" #include "rpl_parallel.h" +#include "my_core_no_dump.h" + /* The rule for this file: everything should be 'static'. When a sys_var variable or a function from this file is - in very rare cases - needed @@ -5356,3 +5358,17 @@ static Sys_var_ulonglong Sys_max_thread_mem( "session variable MEM_USED", SESSION_VAR(max_mem_used), CMD_LINE(REQUIRED_ARG), VALID_RANGE(8192, ULONGLONG_MAX), DEFAULT(LONGLONG_MAX), BLOCK_SIZE(1)); + +#ifdef HAVE_MADV_DONTDUMP + +/* keep list in sync with constant definitiosn in my_core_no_dump.h */ +static const char *core_nodump_names[] = { "MAX", + "INNODB_POOL_BUFFER", + NULL}; + +static Sys_var_set Sys_core_nodump( + "core_nodump", "Things to exclude from core dumps", + GLOBAL_VAR(opt_core_nodump), CMD_LINE(REQUIRED_ARG), + core_nodump_names, DEFAULT(0)); + +#endif /* HAVE_MADV_DONTDUMP */ diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 9e07a4d70f093..07c7c856b5989 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -68,6 +68,9 @@ Created 11/5/1995 Heikki Tuuri #include "lzo/lzo1x.h" #endif +/* for core dump specific stuff */ +#include "my_core_no_dump.h" + /* IMPLEMENTATION OF THE BUFFER POOL ================================= @@ -1214,6 +1217,8 @@ buf_chunk_init( } } #endif // HAVE_LIBNUMA + // exclude from core dumps if configured to do so + exclude_from_coredump(chunk->mem, chunk->mem_size, CORE_NODUMP_INNODB_POOL_BUFFER); /* Allocate the block descriptors from the start of the memory block. */ diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 84e3412387d5d..c9fb3a340739d 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -65,6 +65,9 @@ Created 11/5/1995 Heikki Tuuri #include "fil0pagecompress.h" #include "ha_prototypes.h" +/* for core dump specific stuff */ +#include "my_core_no_dump.h" + /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); @@ -1282,6 +1285,8 @@ buf_chunk_init( } } #endif // HAVE_LIBNUMA + // exclude from core dumps if configured to do so + exclude_from_coredump(chunk->mem, chunk->mem_size, CORE_NODUMP_INNODB_POOL_BUFFER); /* Allocate the block descriptors from the start of the memory block. */