diff --git a/BUILD.bazel b/BUILD.bazel index 795e392e70..905dca16b7 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -63,6 +63,8 @@ LINKOPTS = [ "-Wl,-U,_ProfilerStop", "-Wl,-U,__Z13GetStackTracePPvii", "-Wl,-U,_RegisterThriftProtocol", + "-Wl,-U,_mallctl", + "-Wl,-U,_malloc_stats_print", ], "//conditions:default": [ "-lrt", diff --git a/CMakeLists.txt b/CMakeLists.txt index d2a140a7a2..9b354d42b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -322,7 +322,10 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() # for *.so diff --git a/config_brpc.sh b/config_brpc.sh index ddaa6daa65..cf4544ed30 100755 --- a/config_brpc.sh +++ b/config_brpc.sh @@ -198,6 +198,8 @@ if [ "$SYSTEM" = "Darwin" ]; then DYNAMIC_LINKINGS="$DYNAMIC_LINKINGS -Wl,-U,_ProfilerStop" DYNAMIC_LINKINGS="$DYNAMIC_LINKINGS -Wl,-U,__Z13GetStackTracePPvii" DYNAMIC_LINKINGS="$DYNAMIC_LINKINGS -Wl,-U,_RegisterThriftProtocol" + DYNAMIC_LINKINGS="$DYNAMIC_LINKINGS -Wl,-U,_mallctl" + DYNAMIC_LINKINGS="$DYNAMIC_LINKINGS -Wl,-U,_malloc_stats_print" fi append_linking() { if [ -f $1/lib${2}.a ]; then diff --git a/docs/cn/heap_profiler.md b/docs/cn/heap_profiler.md index d005aae926..941f0f7a6c 100644 --- a/docs/cn/heap_profiler.md +++ b/docs/cn/heap_profiler.md @@ -107,3 +107,61 @@ brpc还提供一个类似的growth profiler分析内存的分配去向(不考 1. 安装[standalone pprof](https://github.com/google/pprof),并把下载的pprof二进制文件路径写入环境变量GOOGLE_PPROF_BINARY_PATH中 2. 安装llvm-symbolizer(将函数符号转化为函数名),直接用brew安装即可:`brew install llvm` + +# Jemalloc Heap Profiler + +## 开启方法 + +1. 编译[jemalloc](https://github.com/jemalloc/jemalloc)时需--enable-prof以支持profiler, 安装完成后bin目录下会有jeprof文件。 +2. 启动进程前最好配置env `JEPROF_FILE=/xxx/jeprof`,否则进程默认用$PATH里的jeprof解析。 +3. 进程开启profiler: + - 启动进程并开启profiler功能:`MALLOC_CONF="prof:true" LD_PRELOAD=/xxx/lib/libjemalloc.so ./bin/test_server`,MALLOC_CONF是env项,prof:true只做一些初始化动作,并不会采样,但[prof_active](https://jemalloc.net/jemalloc.3.html#opt.prof_active)默认是true,所以进程启动就会采样。 + - 若静态链接jemalloc:`MALLOC_CONF="prof:true" ./bin/test_server`。 + - 或通过下面的gflags控制,gflags不会反应MALLOC_CONF值。 +4. 相关gflags说明: + - FLAGS_je_prof_active:true:开启采样,false:关闭采样。 + - FLAGS_je_prof_dump:修改值会生成heap文件,用于手动操作jeprof分析。 + - FLAGS_je_prof_reset:清理已采样数据和重置profiler选项,并且动态设置采样率,[默认](https://jemalloc.net/jemalloc.3.html#opt.lg_prof_sample)2^19B(512K),对性能影响可忽略。 +5. 若要做memory leak: + - `MALLOC_CONF="prof:true,prof_leak:true,prof_final:true" LD_PRELOAD=/xxx/lib/libjemalloc.so ./bin/test_server` ,进程退出时生成heap文件。 + - 注:可`kill pid`优雅退出,不可`kill -9 pid`;可用`FLAGS_graceful_quit_on_sigterm=true FLAGS_graceful_quit_on_sighup=true`来支持优雅退出。 + +注: + - 每次dump的都是从采样至今的所有数据,若触发了reset,接来下dump的是从reset至今的所有数据,方便做diff。 + - 更多jemalloc profiler选项请参考[官网](https://jemalloc.net/jemalloc.3.html),如`prof_leak_error:true`则检测到内存泄漏,进程立即退出。 + +## 样例 + +- jeprof命令`jeprof ip:port/pprof/heap`。 + +![img](../images/cmd_jeprof_text.png) + +- curl生成text格式`curl ip:port/pprof/heap?display=text`。 + +![img](../images/curl_jeprof_text.png) + +- curl生成svg图片格式`curl ip:port/pprof/heap?display=svg`。 + +![img](../images/curl_jeprof_svg.png) + +- curl生成火焰图`curl ip:port/pprof/heap?display=flamegraph`。需配置env FLAMEGRAPH_PL_PATH=/xxx/flamegraph.pl,[flamegraph](https://github.com/brendangregg/FlameGraph) + +![img](../images/curl_jeprof_flamegraph.png) + +- curl获取内存统计信息`curl ip:port/pprof/heap?display=stats&opts=Ja`或`curl ip:port/memory?opts=Ja`,更多opts请参考[opts](https://github.com/jemalloc/jemalloc/blob/dev/include/jemalloc/internal/stats.h#L9)。 + +![img](../images/je_stats_print.png) + + - 内存使用量可关注: + 1. jemalloc.stats下的 + - [resident](https://jemalloc.net/jemalloc.3.html#stats.resident) + - [metadata](https://jemalloc.net/jemalloc.3.html#stats.metadata) + - [allocated](https://jemalloc.net/jemalloc.3.html#stats.allocated):jeprof分析的就是这部分内存。 + - [active](https://jemalloc.net/jemalloc.3.html#stats.active):active - allocated ≈ unuse。 + 2. stats.arenas下的: + - [resident](https://jemalloc.net/jemalloc.3.html#stats.arenas.i.resident) + - [pactive](https://jemalloc.net/jemalloc.3.html#stats.arenas.i.pactive) + - [base](https://jemalloc.net/jemalloc.3.html#stats.arenas.i.base):含义近似metadata + - [small.allocated](https://jemalloc.net/jemalloc.3.html#stats.arenas.i.small.allocated) + - [large.allocated](https://jemalloc.net/jemalloc.3.html#stats.arenas.i.large.allocated):arena allocated ≈ small.allocated + large.allocated + diff --git a/docs/images/cmd_jeprof_text.png b/docs/images/cmd_jeprof_text.png new file mode 100644 index 0000000000..7669bc8087 Binary files /dev/null and b/docs/images/cmd_jeprof_text.png differ diff --git a/docs/images/curl_jeprof_flamegraph.png b/docs/images/curl_jeprof_flamegraph.png new file mode 100644 index 0000000000..2c6b6b3c7e Binary files /dev/null and b/docs/images/curl_jeprof_flamegraph.png differ diff --git a/docs/images/curl_jeprof_svg.png b/docs/images/curl_jeprof_svg.png new file mode 100644 index 0000000000..ed71499101 Binary files /dev/null and b/docs/images/curl_jeprof_svg.png differ diff --git a/docs/images/curl_jeprof_text.png b/docs/images/curl_jeprof_text.png new file mode 100644 index 0000000000..b7cf53d3d8 Binary files /dev/null and b/docs/images/curl_jeprof_text.png differ diff --git a/docs/images/je_stats_print.png b/docs/images/je_stats_print.png new file mode 100644 index 0000000000..35f87697b0 Binary files /dev/null and b/docs/images/je_stats_print.png differ diff --git a/example/asynchronous_echo_c++/CMakeLists.txt b/example/asynchronous_echo_c++/CMakeLists.txt index 60854454e2..2946e29963 100644 --- a/example/asynchronous_echo_c++/CMakeLists.txt +++ b/example/asynchronous_echo_c++/CMakeLists.txt @@ -130,7 +130,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(asynchronous_echo_client client.cpp ${PROTO_SRC}) diff --git a/example/auto_concurrency_limiter/CMakeLists.txt b/example/auto_concurrency_limiter/CMakeLists.txt index c19a8f4001..4d4802c8d4 100644 --- a/example/auto_concurrency_limiter/CMakeLists.txt +++ b/example/auto_concurrency_limiter/CMakeLists.txt @@ -124,7 +124,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" "-Wl,-U,__Z13GetStackTracePPvii" - "-Wl,-U,_RegisterThriftProtocol") + "-Wl,-U,_RegisterThriftProtocol" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(asynchronous_echo_client client.cpp ${PROTO_SRC}) diff --git a/example/backup_request_c++/CMakeLists.txt b/example/backup_request_c++/CMakeLists.txt index 7aab5e5768..5b91114b50 100644 --- a/example/backup_request_c++/CMakeLists.txt +++ b/example/backup_request_c++/CMakeLists.txt @@ -130,7 +130,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(backup_request_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/baidu_proxy_and_generic_call/CMakeLists.txt b/example/baidu_proxy_and_generic_call/CMakeLists.txt index 8cc9c0f135..88c691dff5 100644 --- a/example/baidu_proxy_and_generic_call/CMakeLists.txt +++ b/example/baidu_proxy_and_generic_call/CMakeLists.txt @@ -122,7 +122,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/bthread_tag_echo_c++/CMakeLists.txt b/example/bthread_tag_echo_c++/CMakeLists.txt index 56aec0df5d..3dd7a24522 100644 --- a/example/bthread_tag_echo_c++/CMakeLists.txt +++ b/example/bthread_tag_echo_c++/CMakeLists.txt @@ -136,7 +136,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/cancel_c++/CMakeLists.txt b/example/cancel_c++/CMakeLists.txt index 07d80b4c4a..d6246c67fe 100644 --- a/example/cancel_c++/CMakeLists.txt +++ b/example/cancel_c++/CMakeLists.txt @@ -130,7 +130,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(cancel_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/cascade_echo_c++/CMakeLists.txt b/example/cascade_echo_c++/CMakeLists.txt index b6d8c0b75f..bfb11025fd 100644 --- a/example/cascade_echo_c++/CMakeLists.txt +++ b/example/cascade_echo_c++/CMakeLists.txt @@ -129,7 +129,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(cascade_echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/dynamic_partition_echo_c++/CMakeLists.txt b/example/dynamic_partition_echo_c++/CMakeLists.txt index 2394b7fc34..a5724d2ce8 100644 --- a/example/dynamic_partition_echo_c++/CMakeLists.txt +++ b/example/dynamic_partition_echo_c++/CMakeLists.txt @@ -136,7 +136,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(dynamic_partition_echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/echo_c++/CMakeLists.txt b/example/echo_c++/CMakeLists.txt index 7746957b17..54fdd5707a 100644 --- a/example/echo_c++/CMakeLists.txt +++ b/example/echo_c++/CMakeLists.txt @@ -130,7 +130,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/grpc_c++/CMakeLists.txt b/example/grpc_c++/CMakeLists.txt index 516735d6ec..6250d2aba9 100644 --- a/example/grpc_c++/CMakeLists.txt +++ b/example/grpc_c++/CMakeLists.txt @@ -130,7 +130,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" "-Wl,-U,__Z13GetStackTracePPvii" - "-Wl,-U,_RegisterThriftProtocol") + "-Wl,-U,_RegisterThriftProtocol" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(server server.cpp ${PROTO_SRC} ${PROTO_HEADER} ) diff --git a/example/http_c++/CMakeLists.txt b/example/http_c++/CMakeLists.txt index 3fd9a088a3..c5e7d40a5a 100644 --- a/example/http_c++/CMakeLists.txt +++ b/example/http_c++/CMakeLists.txt @@ -137,7 +137,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(http_client http_client.cpp) diff --git a/example/memcache_c++/CMakeLists.txt b/example/memcache_c++/CMakeLists.txt index 5941c344cd..7236f01c43 100644 --- a/example/memcache_c++/CMakeLists.txt +++ b/example/memcache_c++/CMakeLists.txt @@ -130,7 +130,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(memcache_client client.cpp) diff --git a/example/multi_threaded_echo_c++/CMakeLists.txt b/example/multi_threaded_echo_c++/CMakeLists.txt index c13b34bdde..896483afc5 100644 --- a/example/multi_threaded_echo_c++/CMakeLists.txt +++ b/example/multi_threaded_echo_c++/CMakeLists.txt @@ -136,7 +136,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/multi_threaded_echo_fns_c++/CMakeLists.txt b/example/multi_threaded_echo_fns_c++/CMakeLists.txt index 887541b8fd..c934f7aa88 100644 --- a/example/multi_threaded_echo_fns_c++/CMakeLists.txt +++ b/example/multi_threaded_echo_fns_c++/CMakeLists.txt @@ -136,7 +136,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(multi_threaded_echo_fns_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/nshead_extension_c++/CMakeLists.txt b/example/nshead_extension_c++/CMakeLists.txt index 6ddf382671..5aa496ac63 100644 --- a/example/nshead_extension_c++/CMakeLists.txt +++ b/example/nshead_extension_c++/CMakeLists.txt @@ -130,7 +130,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(nshead_extension_client client.cpp) diff --git a/example/nshead_pb_extension_c++/CMakeLists.txt b/example/nshead_pb_extension_c++/CMakeLists.txt index 8d0edf6030..0516a9ebe6 100644 --- a/example/nshead_pb_extension_c++/CMakeLists.txt +++ b/example/nshead_pb_extension_c++/CMakeLists.txt @@ -130,7 +130,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(nshead_pb_extension_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/parallel_echo_c++/CMakeLists.txt b/example/parallel_echo_c++/CMakeLists.txt index 5bd081836e..b70d7571be 100644 --- a/example/parallel_echo_c++/CMakeLists.txt +++ b/example/parallel_echo_c++/CMakeLists.txt @@ -136,7 +136,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(parallel_echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/partition_echo_c++/CMakeLists.txt b/example/partition_echo_c++/CMakeLists.txt index 636e19b431..2e09c388ca 100644 --- a/example/partition_echo_c++/CMakeLists.txt +++ b/example/partition_echo_c++/CMakeLists.txt @@ -136,7 +136,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/rdma_performance/CMakeLists.txt b/example/rdma_performance/CMakeLists.txt index 0fe0fe20b6..de730720be 100644 --- a/example/rdma_performance/CMakeLists.txt +++ b/example/rdma_performance/CMakeLists.txt @@ -136,7 +136,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/redis_c++/CMakeLists.txt b/example/redis_c++/CMakeLists.txt index 1447e60698..fc4745c872 100644 --- a/example/redis_c++/CMakeLists.txt +++ b/example/redis_c++/CMakeLists.txt @@ -140,7 +140,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(redis_cli redis_cli.cpp) diff --git a/example/rpcz_echo_c++/CMakeLists.txt b/example/rpcz_echo_c++/CMakeLists.txt index 53c1669028..e377acc1d1 100644 --- a/example/rpcz_echo_c++/CMakeLists.txt +++ b/example/rpcz_echo_c++/CMakeLists.txt @@ -130,7 +130,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(rpcz_echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/selective_echo_c++/CMakeLists.txt b/example/selective_echo_c++/CMakeLists.txt index 4b304fb6c0..4931121821 100644 --- a/example/selective_echo_c++/CMakeLists.txt +++ b/example/selective_echo_c++/CMakeLists.txt @@ -136,7 +136,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(selective_echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/session_data_and_thread_local/CMakeLists.txt b/example/session_data_and_thread_local/CMakeLists.txt index 3678ab761d..ea1c838174 100644 --- a/example/session_data_and_thread_local/CMakeLists.txt +++ b/example/session_data_and_thread_local/CMakeLists.txt @@ -137,7 +137,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(session_data_and_thread_local_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/example/streaming_echo_c++/CMakeLists.txt b/example/streaming_echo_c++/CMakeLists.txt index ca4579d308..ebefaf4f06 100644 --- a/example/streaming_echo_c++/CMakeLists.txt +++ b/example/streaming_echo_c++/CMakeLists.txt @@ -130,7 +130,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" - "-Wl,-U,__Z13GetStackTracePPvii") + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) endif() add_executable(streaming_echo_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) diff --git a/src/brpc/builtin/memory_service.cpp b/src/brpc/builtin/memory_service.cpp index 01932c98a5..13589d0bb9 100644 --- a/src/brpc/builtin/memory_service.cpp +++ b/src/brpc/builtin/memory_service.cpp @@ -21,6 +21,7 @@ #include "brpc/closure_guard.h" // ClosureGuard #include "brpc/builtin/memory_service.h" #include "brpc/details/tcmalloc_extension.h" +#include "brpc/details/jemalloc_profiler.h" namespace brpc { @@ -58,6 +59,15 @@ static void get_tcmalloc_memory_info(butil::IOBuf& out) { os.move_to(out); } +static void get_jemalloc_memory_info(Controller* cntl) { + const brpc::URI& uri = cntl->http_request().uri(); + cntl->http_response().set_content_type("text/plain"); + + const std::string* uri_opts = uri.GetQuery("opts"); + std::string opts = !uri_opts || uri_opts->empty() ? "Ja" : *uri_opts; + cntl->response_attachment().append(StatsPrint(opts)); +} + void MemoryService::default_method(::google::protobuf::RpcController* cntl_base, const ::brpc::MemoryRequest*, ::brpc::MemoryResponse*, @@ -70,8 +80,11 @@ void MemoryService::default_method(::google::protobuf::RpcController* cntl_base, if (IsTCMallocEnabled()) { butil::IOBufBuilder os; get_tcmalloc_memory_info(resp); + } else if (HasJemalloc()) { + // support ip:port/memory?opts=Ja + get_jemalloc_memory_info(cntl); } else { - resp.append("tcmalloc is not enabled"); + resp.append("tcmalloc or jemalloc is not enabled"); cntl->http_response().set_status_code(HTTP_STATUS_FORBIDDEN); return; } diff --git a/src/brpc/builtin/pprof_service.cpp b/src/brpc/builtin/pprof_service.cpp index e959169874..e22144f2ad 100644 --- a/src/brpc/builtin/pprof_service.cpp +++ b/src/brpc/builtin/pprof_service.cpp @@ -34,6 +34,7 @@ #include "brpc/builtin/pprof_service.h" #include "brpc/builtin/common.h" #include "brpc/details/tcmalloc_extension.h" +#include "brpc/details/jemalloc_profiler.h" #include "bthread/bthread.h" // bthread_usleep #include "butil/fd_guard.h" @@ -213,6 +214,12 @@ void PProfService::heap( ::google::protobuf::Closure* done) { ClosureGuard done_guard(done); Controller* cntl = static_cast(controller_base); + + if (HasJemalloc()) { + JeControlProfile(cntl); + return; + } + MallocExtension* malloc_ext = MallocExtension::instance(); if (malloc_ext == NULL || !has_TCMALLOC_SAMPLE_PARAMETER()) { const char* extra_desc = ""; diff --git a/src/brpc/details/jemalloc_profiler.cpp b/src/brpc/details/jemalloc_profiler.cpp new file mode 100644 index 0000000000..ea9dcac49b --- /dev/null +++ b/src/brpc/details/jemalloc_profiler.cpp @@ -0,0 +1,331 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include "brpc/controller.h" +#include "brpc/http_header.h" +#include "brpc/reloadable_flags.h" +#include "brpc/uri.h" +#include "butil/files/file_path.h" +#include "butil/iobuf.h" +#include "butil/popen.h" +#include "gflags/gflags.h" +#include "gflags/gflags_declare.h" +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +// weak symbol: resolved at runtime by the linker if we are using jemalloc, nullptr otherwise +int BAIDU_WEAK mallctl(const char*, void*, size_t*, void*, size_t); +void BAIDU_WEAK malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, const char *opts); +} + +namespace brpc { + +// https://jemalloc.net/jemalloc.3.html +DEFINE_bool(je_prof_active, false, "control jemalloc prof.active, jemalloc profiling enabled but inactive," + "it toggle profiling at any time during process running"); +DEFINE_int32(je_prof_dump, 0, "control jemalloc prof.dump, change this only dump profile"); +DEFINE_int32(je_prof_reset, 19, "control jemalloc prof.reset, reset all memory profile statistics, " + "and optionally update the sample rate, default 2^19 B"); + +DECLARE_int32(max_flame_graph_width); + +// define in src/brpc/builtin/pprof_service.cpp +extern int MakeProfName(ProfilingType type, char* buf, size_t buf_len); + +static bool HasInit(const std::string& fn) { + static std::set fns; + if (fns.count(fn) > 0) { + return true; + } + fns.insert(fn); + return false; +} + +bool HasJemalloc() { + return mallctl != nullptr; +} + +// env need MALLOC_CONF="prof:true" before process start +static bool HasEnableJemallocProfile() { + bool prof = false; + size_t len = sizeof(prof); + int ret = mallctl("opt.prof", &prof, &len, nullptr, 0); + if (ret != 0) { + LOG(WARNING) << "mallctl get opt.prof err, ret:" << ret; + return false; + } + return prof; +} + +static void WriteCb(void *opaque, const char *str) { + // maybe call n times WriteCb by single malloc_stats_print + static_cast(opaque)->append(str); +} + +std::string StatsPrint(const std::string& opts) { + if (malloc_stats_print == nullptr) { + return "your jemalloc no support malloc_stats_print"; + } + + std::string stat_str; + malloc_stats_print(WriteCb, &stat_str, opts.c_str()); + return stat_str; +} + +static int JeProfileActive(bool active) { + if (!HasJemalloc()) { + LOG(WARNING) << "no jemalloc"; + return -1; + } + + if (!HasEnableJemallocProfile()) { + LOG(WARNING) << "jemalloc have not set opt.prof before start"; + return -1; + } + + int ret = mallctl("prof.active", nullptr, nullptr, &active, sizeof(active)); + if (ret != 0) { + LOG(WARNING) << "mallctl set prof.active:" << active << " err, ret:" << ret; + return ret; + } + LOG(INFO) << "mallctl set prof.active:" << active << " succ"; + return 0; +} + +static std::string JeProfileDump() { + if (!HasJemalloc()) { + LOG(WARNING) << "no jemalloc"; + return ""; + } + + if (!HasEnableJemallocProfile()) { + LOG(WARNING) << "jemalloc have not set opt.prof before start"; + return ""; + } + + char prof_name[256]; + if (MakeProfName(PROFILING_HEAP, prof_name, sizeof(prof_name)) != 0) { + LOG(WARNING) << "Fail to create .prof file, " << berror(); + return ""; + } + butil::File::Error error; + const butil::FilePath dir = butil::FilePath(prof_name).DirName(); + if (!butil::CreateDirectoryAndGetError(dir, &error)) { + LOG(WARNING) << "Fail to create directory= " << dir.value(); + return ""; + } + + const char* p_prof_name = prof_name; + int ret = mallctl("prof.dump", NULL, NULL, (void*)&p_prof_name, sizeof(p_prof_name)); + if (ret != 0) { + LOG(WARNING) << "mallctl set prof.dump:" << p_prof_name << " err, ret:" << ret; + return ""; + } + LOG(INFO) << "heap profile dump:" << prof_name << " succ"; + return prof_name; +} + +static int JeProfileReset(size_t lg_sample) { + if (!HasJemalloc()) { + LOG(WARNING) << "no jemalloc"; + return -1; + } + + if (!HasEnableJemallocProfile()) { + LOG(WARNING) << "jemalloc have not set opt.prof before start"; + return -1; + } + + int ret = mallctl("prof.reset", nullptr, nullptr, &lg_sample, sizeof(lg_sample)); + if (ret != 0) { + LOG(WARNING) << "mallctl set prof.reset:" << lg_sample << " err, ret:" << ret; + return ret; + } + LOG(INFO) << "mallctl set prof.reset:" << lg_sample << " succ"; + + FLAGS_je_prof_active = false; + if (FLAGS_je_prof_active) { + LOG(WARNING) << "reset FLAGS_je_prof_active fail"; + return -1; + } + + return 0; +} + +void JeControlProfile(Controller* cntl) { + const brpc::URI& uri = cntl->http_request().uri(); + // http:ip:port/pprof/heap?display=(text|svg|stats|flamegraph) + const std::string* uri_display = uri.GetQuery("display"); + + butil::IOBuf& buf = cntl->response_attachment(); + cntl->http_response().set_content_type("text/plain"); + + // support ip:port/pprof/heap?display=stats + if (uri_display != nullptr && *uri_display == "stats") { + const std::string* uri_opts = uri.GetQuery("opts"); + std::string opts = !uri_opts || uri_opts->empty() ? "Ja" : *uri_opts; + buf.append(StatsPrint(opts)); + return; + } + + if (!HasEnableJemallocProfile()) { + cntl->SetFailed(ENOMETHOD, "Heap profiler is not enabled, (no MALLOC_CONF=prof:true in env)"); + return; + } + + // only dump profile + const std::string prof_name = JeProfileDump(); + if (prof_name.empty()) { + cntl->SetFailed(-1, "Fail to dump profile"); + buf.append("\nFail to dump profile"); + return; + } + + // support jeprof ip:port/pprof/heap + if (uri_display == nullptr || uri_display->empty()) { + std::string content; + if (!butil::ReadFileToString(butil::FilePath(prof_name), &content)) { + LOG(WARNING) << "read " << prof_name << " fail"; + return; + } + buf.append(content); + return; + } + + // support curl/browser + buf.append(prof_name); + + std::string jeprof; + const char* f_jeprof = std::getenv("JEPROF_FILE"); + if (f_jeprof != nullptr && butil::PathExists(butil::FilePath(f_jeprof))) { + jeprof = f_jeprof; + } else { + LOG(WARNING) << "env JEPROF_FILE invalid"; + buf.append("\nenv JEPROF_FILE invalid"); + jeprof = "jeprof "; // use PATH + } + + char process_path[500] = {}; + ssize_t len = butil::GetProcessAbsolutePath(process_path, sizeof(process_path)); + if (len == -1) { + LOG(WARNING) << "GetProcessAbsolutePath of process err"; + buf.append("\nGetProcessAbsolutePath of process err"); + return; + } + const std::string process_file(process_path, len); + + std::string cmd_str = jeprof + " " + process_file + " " + prof_name; + bool display_img = false; + if (*uri_display == "svg") { + cmd_str += " --svg "; + display_img = true; + } else if (*uri_display == "flamegraph") { + const char* flamegraph_tool = getenv("FLAMEGRAPH_PL_PATH"); + if (!flamegraph_tool) { + LOG(WARNING) << " display: " << *uri_display << " invalid, env FLAMEGRAPH_PL_PATH invalid"; + buf.append("\ndisplay:" + *uri_display + " invalid, env FLAMEGRAPH_PL_PATH invalid"); + return; + } + const int width_size = FLAGS_max_flame_graph_width > 0 ? FLAGS_max_flame_graph_width : 1200; + cmd_str += " --collapsed | " + std::string(flamegraph_tool) + " --colors mem --width " + std::to_string(width_size); + display_img = true; + } else if (*uri_display == "text") { + cmd_str += " --text "; + } else { + LOG(WARNING) << " display: " << *uri_display << " invalid"; + buf.append("\ndisplay:" + *uri_display + " invalid"); + return; + } + butil::IOBufBuilder builder; + if (butil::read_command_output(builder, cmd_str.c_str()) != 0) { + buf.append("\nread_command_output <" + cmd_str + "> fail"); + LOG(WARNING) << "read_command_output <" + cmd_str + "> fail"; + return; + } + + if (display_img) { + buf.swap(builder.buf()); + cntl->http_response().set_content_type("image/svg+xml"); + } else { + buf.append("\ncmd: " + cmd_str + "\n"); + buf.append(builder.buf()); + } +} + +static bool validate_je_prof_active(const char*, bool enable) { + if (!HasJemalloc()) { + return true; + } + + if (!HasInit(__func__)) { + return true; + } + + if (JeProfileActive(enable) != 0) { + LOG(WARNING) << "JeControlSample err"; + return false; + } + + return true; +} + +static bool validate_je_prof_dump(const char*, int32_t val) { + if (!HasJemalloc()) { + return true; + } + if (!HasInit(__func__)) { + return true; + } + + const std::string prof_name = JeProfileDump(); + if (prof_name.empty()) { + LOG(WARNING) << "Fail to dump profile"; + return false; + } + return true; +} + +static bool validate_je_prof_reset(const char*, int32_t val) { + if (!HasJemalloc()) { + return true; + } + if (!HasInit(__func__)) { + return true; + } + + if (JeProfileReset(val) != 0) { + LOG(WARNING) << "JeProfileReset err"; + return false; + } + + return true; +} + +// e.g: curl ip:port/flags/je_prof_active?setvalue=true or update flags in web +BRPC_VALIDATE_GFLAG(je_prof_active, validate_je_prof_active); +BRPC_VALIDATE_GFLAG(je_prof_dump, validate_je_prof_dump); +BRPC_VALIDATE_GFLAG(je_prof_reset, validate_je_prof_reset); + +} + diff --git a/src/brpc/details/jemalloc_profiler.h b/src/brpc/details/jemalloc_profiler.h new file mode 100644 index 0000000000..8468065f97 --- /dev/null +++ b/src/brpc/details/jemalloc_profiler.h @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +namespace brpc { + +bool HasJemalloc(); + +// opts: Ja +// more see ref: https://github.com/jemalloc/jemalloc/blob/dev/include/jemalloc/internal/stats.h#L9 +std::string StatsPrint(const std::string& opts); + +void JeControlProfile(Controller* cntl); + +} +