MDEV-10814: hartmut coredump exclusions (10.1)

Docs/README-core-dump-exclusion

@@ -0,0 +1,108 @@
+Exclude things from core dumps (on Linux)
+=========================================
+
+Movitation
+----------
+
+There are two reasons for wanting to reduce core dump size by excluding
+certain memory areas from it:
+
+1. Reduce IO bandwidth and storage space requirements when dumping core
+
+   This reduces the time it takes to take a core dump and also makes
+   transfering core files easier.
+
+2. Exclude sensitive data from core dumps
+
+   At the same time backtrace information should remain complete and memory
+   regions that may be useful when doing post mortem debugging should be kept.
+
+Original motivation for implementation was that very large InnoDB instances
+on machines with >300GB RAM, and accordingly large innodb buffer pool, were
+crashing randomly every few days, and enabling core dumps on all of them was
+not an option. Also, even after we finally managed to get hold of a core
+dump the file size made it very hard to get it to an environment where it
+could be analyzed without causng resource issues.
+
+Implementation
+--------------
+
+Linux supports options to madvise(2) that control whether a certain region
+should be included in core dumps (MADV_DODUMP) or should be excluded
+(MADV_DONTDUMP) starting with kernel version. Such memory regions need to
+be page aligned so out of the box the feature is only useful when dealing
+with large enough malloc() allocations or when directly dealing with
+mmap()ed or shared memory regions directly.
+
+Certain memory allocations now call the new function
+
+  exclude_from_coredump(void *ptr, size_t size, ulonglong flagmask);
+
+after successfully aquiring memory. A global variable "core_nodump"
+may contain a set of memory regions to exclude. If the given flagmask
+matches any of these, or if "core_nodump" has the 'magic' value "MAX",
+the given memory region is marked to be excluded from core dumps using
+the MADV_DONTDUMP advice.
+
+So far there's only support for excluding static buffers that stay for
+the entire life time of the server process. Support for memory regions
+allocated on demand, like those for temporary tables or sort buffers,
+or that may be resized dynamically, like the query cache, is possible
+but tricky as on freeing memory regions the MADV_DONTDUMP advice has to
+be removed by an explicit MADV_DODUMP madvise() for the same region.
+Otherwise other later memory allocations that happen to reuse fall in
+the previously excluded memory range would accidentially be excluded
+from being dumped. So at the time memory is freed it is also necessary
+to remove the DONTDUMP advise, and for that not only the base address
+is needed but also the original allocation size. As this is not always
+explicitly preserved dynamic buffers have been excluded for this first
+implementation.
+
+
+Configuration
+-------------
+
+A new configuration variable "core_nodump" has been added that controls
+which buffer types should be excluded from core dump. The variable
+can't be changed dynamically at run time as exclusions happen right
+after allocating memory at startup.
+
+Possible values for "core_nodump" at this point are:
+
+ Value              | Description                                          
+--------------------|------------------------------------------------------
+ NONE               | Default: keep current behavior that dumps everything 
+ INNODB_POOL_BUFFER | Exclude InnoDB pool buffer instances from core dump  
+ MYISAM_KEY_BUFFER  | Exclude MyISAM key buffer(s) from core dump          
+ MAX                | Apply all possible exclusion                         
+
+
+Testing
+-------
+
+A mysqld with the following minimalistic configuration 
+
+    [mysqld]
+    innodb_buffer_pool_size=8G
+    core-file
+    core-nodump=NONE
+
+will create a core dump of about 9GB when killed with `kill -11 $mysqld_pid`.
+
+The same mysqld with INNODB_BUFFER_POOL excluded from dumps only takes
+about 900M, as was to be expected with the 8GB pool being excluded:
+
+    [mysqld]
+    innodb_buffer_pool_size=8G
+    core-file
+    core-nodump=INNODB_BUFFER_POOL
+
+Backtraces can be retrieved sucessfully from both core dumps with gdb
+and `thread apply all bt full`.
+
+As InnoDB buffer pool and Myisam key cache implementations can be considered
+stable lack of those buffer contents should not affect core dump post mortem
+analysis in any way. This may not be the case with new features like tablespace
+encryption yet though, so core dump exclusions should be used carefully in such
+contexts.
+

config.h.cmake

@@ -206,6 +206,7 @@
 #cmakedefine HAVE_NL_LANGINFO 1
 #cmakedefine HAVE_MADVISE 1
 #cmakedefine HAVE_DECL_MADVISE 1
+#cmakedefine HAVE_MADV_DONTDUMP 1
 #cmakedefine HAVE_DECL_TGOTO 1
 #cmakedefine HAVE_DECL_MHA_MAPSIZE_VA 1
 #cmakedefine HAVE_MALLINFO 1

configure.cmake

@@ -516,6 +516,7 @@ ENDIF()
 CHECK_SYMBOL_EXISTS(log2  math.h HAVE_LOG2)
 CHECK_SYMBOL_EXISTS(isnan math.h HAVE_ISNAN)
 CHECK_SYMBOL_EXISTS(rint  math.h HAVE_RINT)
+CHECK_SYMBOL_EXISTS(MADV_DONTDUMP "sys/mman.h" HAVE_MADV_DONTDUMP)
 
 # isinf() prototype not found on Solaris
 CHECK_CXX_SOURCE_COMPILES(

include/my_core_no_dump.h

@@ -0,0 +1,66 @@
+/*
+   Copyright (c) 2016, 2017, MariaDB Corporation.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA */
+
+/* This is the include file that should be included 'first' in every C file. */
+
+#ifndef MY_CORE_NO_DUMP_INCLUDED
+#define MY_CORE_NO_DUMP_INCLUDED
+
+/*
+ Core dump control options to exclude certain buffers from core dump files
+
+ There are two motivations for excluding things from core dumps:
+
+ * resource utilization: stuff like the InnoDB pool buffer is rarely needed
+   for post mortem debugging, but on machines with large amounts of memory
+   just the time and file system space it takes to write a core dump can
+   become substantial. Large core dump sizes can also become an obstacle
+   when providing a core dump to a 3rd party for analysis
+
+ * security: certain buffers, especially the InnoDB pool buffer or the
+   Aria page cache, are likely to contain sensitive user data. Excluding
+   these from a core dump can improve data security and especially can be
+   a requirement for being able to pass production server core dumps to
+   3rd parties for analysis
+*/
+
+#include <my_global.h>
+
+#ifdef HAVE_MADV_DONTDUMP
+
+#include <sys/mman.h>
+
+/* Core dump exclusion constants */
+#define CORE_NODUMP_NONE                (0)
+#define CORE_NODUMP_MAX                 (1 << 1)
+#define CORE_NODUMP_INNODB_POOL_BUFFER  (1 << 2)
+
+extern ulong opt_core_nodump;
+
+inline void exclude_from_coredump(void *ptr, size_t size, ulong flags)
+{
+  if (opt_core_nodump & (flags | CORE_NODUMP_MAX)) {
+    madvise(ptr, size, MADV_DONTDUMP);
+  }
+}
+
+#else /* HAVE_MADV_DONTDUMP */
+
+#define exclude_from_coredump(ptr, size, flag)
+
+#endif /* HAVE_MADV_DONTDUMP */
+
+#endif

sql/mysqld.cc

@@ -528,6 +528,10 @@ ulong feature_files_opened_with_delayed_keys;
 ulonglong denied_connections;
 my_decimal decimal_zero;
 
+#ifdef HAVE_MADV_DONTDUMP
+ulong opt_core_nodump;
+#endif
+
 /*
   Maximum length of parameter value which can be set through
   mysql_send_long_data() call.

sql/sys_vars.cc

@@ -63,6 +63,8 @@
 #include "opt_range.h"
 #include "rpl_parallel.h"
 
+#include "my_core_no_dump.h"
+
 /*
   The rule for this file: everything should be 'static'. When a sys_var
   variable or a function from this file is - in very rare cases - needed
@@ -5356,3 +5358,17 @@ static Sys_var_ulonglong Sys_max_thread_mem(
        "session variable MEM_USED", SESSION_VAR(max_mem_used),
        CMD_LINE(REQUIRED_ARG), VALID_RANGE(8192,  ULONGLONG_MAX),
        DEFAULT(LONGLONG_MAX), BLOCK_SIZE(1));
+
+#ifdef HAVE_MADV_DONTDUMP
+
+/* keep list in sync with constant definitiosn in my_core_no_dump.h */
+static const char *core_nodump_names[] = { "MAX",
+					   "INNODB_POOL_BUFFER",
+					   NULL};
+
+static Sys_var_set Sys_core_nodump(
+       "core_nodump", "Things to exclude from core dumps",
+       GLOBAL_VAR(opt_core_nodump), CMD_LINE(REQUIRED_ARG),
+       core_nodump_names, DEFAULT(0));
+
+#endif /* HAVE_MADV_DONTDUMP */

storage/innobase/buf/buf0buf.cc

@@ -68,6 +68,9 @@ Created 11/5/1995 Heikki Tuuri
 #include "lzo/lzo1x.h"
 #endif
 
+/* for core dump specific stuff */
+#include "my_core_no_dump.h"
+
 /*
 		IMPLEMENTATION OF THE BUFFER POOL
 		=================================
@@ -1214,6 +1217,8 @@ buf_chunk_init(
 		}
 	}
 #endif // HAVE_LIBNUMA
+	// exclude from core dumps if configured to do so
+	exclude_from_coredump(chunk->mem, chunk->mem_size, CORE_NODUMP_INNODB_POOL_BUFFER);
 
 	/* Allocate the block descriptors from
 	the start of the memory block. */

storage/xtradb/buf/buf0buf.cc

@@ -65,6 +65,9 @@ Created 11/5/1995 Heikki Tuuri
 #include "fil0pagecompress.h"
 #include "ha_prototypes.h"
 
+/* for core dump specific stuff */
+#include "my_core_no_dump.h"
+
 /* prototypes for new functions added to ha_innodb.cc */
 trx_t* innobase_get_trx();
 
@@ -1282,6 +1285,8 @@ buf_chunk_init(
 		}
 	}
 #endif // HAVE_LIBNUMA
+	// exclude from core dumps if configured to do so
+	exclude_from_coredump(chunk->mem, chunk->mem_size, CORE_NODUMP_INNODB_POOL_BUFFER);
 
 	/* Allocate the block descriptors from
 	the start of the memory block. */