diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 053f9fbaee12b..9e4efd879f0a2 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1173,6 +1173,57 @@ buf_page_is_corrupted( } #ifndef UNIV_INNOCHECKSUM + +#if defined(DBUG_OFF) && defined(HAVE_MADVISE) && defined(MADV_DODUMP) +/** Enable buffers to be dumped to core files + +A convience function, not called anyhwere directly however +it is left available for gdb or any debugger to call +in the event that you want all of the memory to be dumped +to a core file. + +Returns number of errors found in madvise calls. */ +int +buf_madvise_do_dump() +{ + int ret= 0; + buf_pool_t* buf_pool; + ulint n; + buf_chunk_t* chunk; + + /* mirrors allocation in log_sys_init() */ + if (log_sys->buf) + { + ret+= madvise(log_sys->first_in_use ? log_sys->buf + : log_sys->buf - log_sys->buf_size, + log_sys->buf_size, + MADV_DODUMP); + } + /* mirrors recv_sys_init() */ + if (recv_sys->buf) + { + ret+= madvise(recv_sys->buf, recv_sys->len, MADV_DODUMP); + } + + buf_pool_mutex_enter_all(); + + for (int i= 0; i < srv_buf_pool_instances; i++) + { + buf_pool = buf_pool_from_array(i); + chunk = buf_pool->chunks; + + for (int n = buf_pool->n_chunks; n--; chunk++) + { + ret+= madvise(chunk->mem, chunk->mem_size(), MADV_DODUMP); + } + } + + buf_pool_mutex_exit_all(); + + return ret; +} +#endif + /** Dump a page to stderr. @param[in] read_buf database page @param[in] page_size page size */ @@ -1502,7 +1553,7 @@ buf_chunk_init( DBUG_EXECUTE_IF("ib_buf_chunk_init_fails", return(NULL);); chunk->mem = buf_pool->allocator.allocate_large(mem_size, - &chunk->mem_pfx); + &chunk->mem_pfx, true); if (UNIV_UNLIKELY(chunk->mem == NULL)) { @@ -1796,7 +1847,8 @@ buf_pool_init_instance( } buf_pool->allocator.deallocate_large( - chunk->mem, &chunk->mem_pfx); + chunk->mem, &chunk->mem_pfx, chunk->mem_size(), + true); } ut_free(buf_pool->chunks); buf_pool_mutex_exit(buf_pool); @@ -1943,7 +1995,7 @@ buf_pool_free_instance( } buf_pool->allocator.deallocate_large( - chunk->mem, &chunk->mem_pfx); + chunk->mem, &chunk->mem_pfx, true); } for (ulint i = BUF_FLUSH_LRU; i < BUF_FLUSH_N_TYPES; ++i) { @@ -2819,7 +2871,7 @@ buf_pool_resize() } buf_pool->allocator.deallocate_large( - chunk->mem, &chunk->mem_pfx); + chunk->mem, &chunk->mem_pfx, true); sum_freed += chunk->size; diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 779d4d238a0d6..716ca34b9282f 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -614,15 +614,15 @@ struct log_t{ mtr_commit and still ensure that insertions in the flush_list happen in the LSN order. */ - byte* buf_ptr; /*!< unaligned log buffer, which should - be of double of buf_size */ - byte* buf; /*!< log buffer currently in use; - this could point to either the first - half of the aligned(buf_ptr) or the + byte* buf; /*!< Memory of double the buf_size is + allocated here. This pointer will change + however to either the first half or the second half in turns, so that log write/flush to disk don't block concurrent mtrs which will write - log to this buffer */ + log to this buffer. Care to switch back + to the first half before freeing/resizing + must be undertaken. */ bool first_in_use; /*!< true if buf points to the first half of the aligned(buf_ptr), false if the second half */ diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 8409ec75c7033..8bab2408605f5 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -217,6 +217,7 @@ struct recv_sys_t{ /*!< this is TRUE when a log rec application batch is running */ byte* buf; /*!< buffer for parsing log records */ + size_t buf_size; /*!< size of buf */ ulint len; /*!< amount of data in buf */ lsn_t parse_start_lsn; /*!< this is the lsn from which we were able to diff --git a/storage/innobase/include/ut0new.h b/storage/innobase/include/ut0new.h index d51fb40a73ecf..254c764a842cd 100644 --- a/storage/innobase/include/ut0new.h +++ b/storage/innobase/include/ut0new.h @@ -129,6 +129,10 @@ with (), thus: #include /* strlen(), strrchr(), strncmp() */ #include "my_global.h" /* needed for headers from mysql/psi/ */ +#if !defined(DBUG_OFF) && defined(HAVE_MADVISE) +#include +#endif + /* JAN: TODO: missing 5.7 header */ #ifdef HAVE_MYSQL_MEMORY_H #include "mysql/psi/mysql_memory.h" /* PSI_MEMORY_CALL() */ @@ -234,6 +238,45 @@ struct ut_new_pfx_t { #endif }; +static void ut_allocate_trace_dontdump(void * ptr, + size_t bytes, + bool dontdump, + ut_new_pfx_t* pfx, + const char* file) +{ + ut_a(ptr != NULL); + +#if defined(DBUG_OFF) && defined(HAVE_MADVISE) && defined(MADV_DONTDUMP) + if (dontdump && madvise(ptr, bytes, MADV_DONTDUMP)) { + ib::warn() << "Failed to set memory to DONTDUMP: " + << strerror(errno) + << " ptr " << ptr + << " size " << bytes; + } +#endif + if (pfx != NULL) { +#ifdef UNIV_PFS_MEMORY + allocate_trace(bytes, file, pfx); +#endif /* UNIV_PFS_MEMORY */ + pfx->m_size = bytes; + } +} + +static void ut_dodump(void* ptr, size_t m_size) +{ + if (ptr == NULL) { + return; + } +#if defined(DBUG_OFF) && defined(HAVE_MADVISE) && defined(MADV_DODUMP) + if (madvise(ptr, m_size, MADV_DODUMP)) { + ib::warn() << "Failed to set memory to DODUMP: " + << strerror(errno) + << " ptr " << ptr + << " size " << m_size; + } +#endif +} + /** Allocator class for allocating memory from inside std::* containers. @tparam T type of allocated object @tparam oom_fatal whether to commit suicide when running out of memory */ @@ -294,6 +337,7 @@ class ut_allocator { @param[in] file file name of the caller @param[in] set_to_zero if true, then the returned memory is initialized with 0x0 bytes. + @param[in] throw_on_error if true, raize exception if too big @return pointer to the allocated memory */ pointer allocate( @@ -566,6 +610,8 @@ class ut_allocator { /** Allocate a large chunk of memory that can hold 'n_elements' objects of type 'T' and trace the allocation. @param[in] n_elements number of elements + @param[in] dontdump if true, advise the OS is not to core + dump this memory. @param[out] pfx storage for the description of the allocated memory. The caller must provide space for this one and keep it until the memory is no longer needed and then pass it to @@ -574,7 +620,8 @@ class ut_allocator { pointer allocate_large( size_type n_elements, - ut_new_pfx_t* pfx) + ut_new_pfx_t* pfx, + bool dontdump = false) { if (n_elements == 0 || n_elements > max_size()) { return(NULL); @@ -585,13 +632,11 @@ class ut_allocator { pointer ptr = reinterpret_cast( os_mem_alloc_large(&n_bytes)); -#ifdef UNIV_PFS_MEMORY - if (ptr != NULL) { - allocate_trace(n_bytes, NULL, pfx); + if (ptr == NULL) { + return NULL; } -#else - pfx->m_size = n_bytes; -#endif /* UNIV_PFS_MEMORY */ + + ut_allocate_trace_dontdump(ptr, n_bytes, dontdump, pfx, NULL); return(ptr); } @@ -600,17 +645,26 @@ class ut_allocator { deallocation. @param[in,out] ptr pointer to memory to free @param[in] pfx descriptor of the memory, as returned by - allocate_large(). */ + allocate_large(). + @param[in] dodump if true, advise the OS to include this + memory again if a core dump occurs. */ void deallocate_large( pointer ptr, - const ut_new_pfx_t* pfx) + const ut_new_pfx_t* pfx, + size_t size, + bool dodump = false) { + if (dodump) { + ut_dodump(ptr, size); + } #ifdef UNIV_PFS_MEMORY - deallocate_trace(pfx); + if (pfx) { + deallocate_trace(pfx); + } #endif /* UNIV_PFS_MEMORY */ - os_mem_free_large(ptr, pfx->m_size); + os_mem_free_large(ptr, size); } #ifdef UNIV_PFS_MEMORY @@ -842,6 +896,10 @@ ut_delete_array( ut_allocator(key).allocate( \ n_bytes, NULL, __FILE__, false, false)) +#define ut_malloc_dontdump(n_bytes) static_cast( \ + ut_allocator(PSI_NOT_INSTRUMENTED).allocate_large( \ + n_bytes, true)) + #define ut_zalloc(n_bytes, key) static_cast( \ ut_allocator(key).allocate( \ n_bytes, NULL, __FILE__, true, false)) @@ -865,6 +923,10 @@ ut_delete_array( #define ut_free(ptr) ut_allocator(PSI_NOT_INSTRUMENTED).deallocate( \ reinterpret_cast(ptr)) +#define ut_free_dodump(ptr, size) static_cast( \ + ut_allocator(PSI_NOT_INSTRUMENTED).deallocate_large( \ + ptr, NULL, size, true)) + #else /* UNIV_PFS_MEMORY */ /* Fallbacks when memory tracing is disabled at compile time. */ @@ -887,6 +949,14 @@ ut_delete_array( #define ut_malloc_nokey(n_bytes) ::malloc(n_bytes) +static inline void *ut_malloc_dontdump(size_t n_bytes) +{ + void *ptr = os_mem_alloc_large(&n_bytes); + + ut_allocate_trace_dontdump(ptr, n_bytes, true, NULL, NULL); + return ptr; +} + #define ut_zalloc_nokey(n_bytes) ::calloc(1, n_bytes) #define ut_zalloc_nokey_nofatal(n_bytes) ::calloc(1, n_bytes) @@ -895,6 +965,12 @@ ut_delete_array( #define ut_free(ptr) ::free(ptr) +static inline void ut_free_dodump(void *ptr, size_t size) +{ + ut_dodump(ptr, size); + os_mem_free_large(ptr, size); +} + #endif /* UNIV_PFS_MEMORY */ #endif /* ut0new_h */ diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 899a48c80b9f7..66b253055664b 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -226,16 +226,18 @@ log_buffer_extend( log_sys->buf_free -= move_start; log_sys->buf_next_to_write -= move_start; + /* free previous after getting the right address */ + if (!log_sys->first_in_use) { + log_sys->buf -= log_sys->buf_size; + } + ut_free_dodump(log_sys->buf, log_sys->buf_size * 2); + /* reallocate log buffer */ srv_log_buffer_size = len / UNIV_PAGE_SIZE + 1; - ut_free(log_sys->buf_ptr); - log_sys->buf_size = LOG_BUFFER_SIZE; - log_sys->buf_ptr = static_cast( - ut_zalloc_nokey(log_sys->buf_size * 2 + OS_FILE_LOG_BLOCK_SIZE)); log_sys->buf = static_cast( - ut_align(log_sys->buf_ptr, OS_FILE_LOG_BLOCK_SIZE)); + ut_malloc_dontdump(log_sys->buf_size * 2)); log_sys->first_in_use = true; @@ -723,10 +725,8 @@ log_sys_init() log_sys->buf_size = LOG_BUFFER_SIZE; - log_sys->buf_ptr = static_cast( - ut_zalloc_nokey(log_sys->buf_size * 2 + OS_FILE_LOG_BLOCK_SIZE)); log_sys->buf = static_cast( - ut_align(log_sys->buf_ptr, OS_FILE_LOG_BLOCK_SIZE)); + ut_malloc_dontdump(log_sys->buf_size * 2)); log_sys->first_in_use = true; @@ -1085,12 +1085,12 @@ log_buffer_switch() OS_FILE_LOG_BLOCK_SIZE); if (log_sys->first_in_use) { - ut_ad(log_sys->buf == ut_align(log_sys->buf_ptr, + ut_ad(log_sys->buf == ut_align(log_sys->buf, OS_FILE_LOG_BLOCK_SIZE)); log_sys->buf += log_sys->buf_size; } else { log_sys->buf -= log_sys->buf_size; - ut_ad(log_sys->buf == ut_align(log_sys->buf_ptr, + ut_ad(log_sys->buf == ut_align(log_sys->buf, OS_FILE_LOG_BLOCK_SIZE)); } @@ -2254,8 +2254,10 @@ log_shutdown() { log_group_close_all(); - ut_free(log_sys->buf_ptr); - log_sys->buf_ptr = NULL; + if (!log_sys->first_in_use) { + log_sys->buf -= log_sys->buf_size; + } + ut_free_dodump(log_sys->buf, log_sys->buf_size * 2); log_sys->buf = NULL; ut_free(log_sys->checkpoint_buf_ptr); log_sys->checkpoint_buf_ptr = NULL; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 59f277095863c..5be52ec06960b 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -438,7 +438,9 @@ recv_sys_close() os_event_destroy(recv_sys->flush_end); } - ut_free(recv_sys->buf); + if (recv_sys->buf != NULL) { + ut_free_dodump(recv_sys->buf, recv_sys->buf_size); + } ut_ad(!recv_writer_thread_active); mutex_free(&recv_sys->writer_mutex); @@ -553,7 +555,8 @@ recv_sys_init() } recv_sys->buf = static_cast( - ut_malloc_nokey(RECV_PARSING_BUF_SIZE)); + ut_malloc_dontdump(RECV_PARSING_BUF_SIZE)); + recv_sys->buf_size = RECV_PARSING_BUF_SIZE; recv_sys->addr_hash = hash_create(size / 512); recv_sys->progress_time = ut_time(); @@ -588,8 +591,9 @@ recv_sys_debug_free(void) hash_table_free(recv_sys->addr_hash); mem_heap_free(recv_sys->heap); - ut_free(recv_sys->buf); + ut_free_dodump(recv_sys->buf, recv_sys->buf_size); + recv_sys->buf_size = 0; recv_sys->buf = NULL; recv_sys->heap = NULL; recv_sys->addr_hash = NULL; diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 64ae384ec3787..c0264d1953ddd 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -71,6 +71,7 @@ enum row_op { /** Log block for modifications during online ALTER TABLE */ struct row_log_buf_t { byte* block; /*!< file block buffer */ + size_t size; /*!< length of block in bytes */ ut_new_pfx_t block_pfx; /*!< opaque descriptor of "block". Set by ut_allocator::allocate_large() and fed to ut_allocator::deallocate_large(). */ @@ -265,6 +266,7 @@ row_log_block_allocate( if (log_buf.block == NULL) { DBUG_RETURN(false); } + log_buf.size = srv_sort_buf_size; } DBUG_RETURN(true); } @@ -279,7 +281,7 @@ row_log_block_free( DBUG_ENTER("row_log_block_free"); if (log_buf.block != NULL) { ut_allocator(mem_key_row_log_buf).deallocate_large( - log_buf.block, &log_buf.block_pfx); + log_buf.block, &log_buf.block_pfx, log_buf.size); log_buf.block = NULL; } DBUG_VOID_RETURN; diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index a0b73c4670374..2928ca51ddb2d 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -4619,6 +4619,7 @@ row_merge_build_indexes( merge_file_t* merge_files; row_merge_block_t* block; ut_new_pfx_t block_pfx; + size_t block_size; ut_new_pfx_t crypt_pfx; row_merge_block_t* crypt_block = NULL; ulint i; @@ -4654,7 +4655,8 @@ row_merge_build_indexes( /* This will allocate "3 * srv_sort_buf_size" elements of type row_merge_block_t. The latter is defined as byte. */ - block = alloc.allocate_large(3 * srv_sort_buf_size, &block_pfx); + block_size = 3 * srv_sort_buf_size; + block = alloc.allocate_large(block_size, &block_pfx); if (block == NULL) { DBUG_RETURN(DB_OUT_OF_MEMORY); @@ -4664,7 +4666,7 @@ row_merge_build_indexes( if (log_tmp_is_encrypted()) { crypt_block = static_cast( - alloc.allocate_large(3 * srv_sort_buf_size, + alloc.allocate_large(block_size, &crypt_pfx)); if (crypt_block == NULL) { @@ -5035,10 +5037,10 @@ row_merge_build_indexes( ut_free(merge_files); - alloc.deallocate_large(block, &block_pfx); + alloc.deallocate_large(block, &block_pfx, block_size); if (crypt_block) { - alloc.deallocate_large(crypt_block, &crypt_pfx); + alloc.deallocate_large(crypt_block, &crypt_pfx, block_size); } DICT_TF2_FLAG_UNSET(new_table, DICT_TF2_FTS_ADD_DOC_ID);