From e2b92f0839b5177375caf0b84f8d3c64afab844e Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Fri, 28 Mar 2025 05:49:25 +0000 Subject: [PATCH 1/3] Add collect_heap_pprofs.sh script for easier collection of heap profiles Signed-off-by: Dom Del Nano --- scripts/collect_heap_pprofs.sh | 63 ++++++++++++++++++++++ scripts/download_heap_prof_mapped_files.sh | 14 ++--- src/pxl_scripts/px/collect_heap_dumps.pxl | 8 +++ 3 files changed, 79 insertions(+), 6 deletions(-) create mode 100755 scripts/collect_heap_pprofs.sh create mode 100644 src/pxl_scripts/px/collect_heap_dumps.pxl diff --git a/scripts/collect_heap_pprofs.sh b/scripts/collect_heap_pprofs.sh new file mode 100755 index 00000000000..86001790de6 --- /dev/null +++ b/scripts/collect_heap_pprofs.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +usage() { + echo "This script downloads all of the files listed in the mappings section of a heap profile." + echo "" + echo "Usage: $0 [...]" + echo " : the directory where the heap profile and memory mapped files will be stored. It will be created if it does not exist." + echo " : the ID of the Vizier cluster to connect to." + echo "Common gcloud ssh options include --project." + exit 1 +} + +set -e + +heap_profile_dir="$1" +cluster_id="$2" +script_dir=$(dirname "$(realpath "$0")") +repo_root=$(git rev-parse --show-toplevel) + +if [ -z "$heap_profile_dir" ] || [ -z "$cluster_id" ]; then + usage +fi + +mkdir -p "$heap_profile_dir" + +pxl_heap_output_file="${heap_profile_dir}/raw_output_from_hot_table_test.json" + +px run -o json -c "$cluster_id" -f "${repo_root}/src/pxl_scripts/px/collect_heap_dumps.pxl" > "$pxl_heap_output_file" + +while IFS= read -r line; do + hostname=$(echo "$line" | jq -r '.hostname') + heap_content=$(echo "$line" | jq -r '.heap') + echo "$heap_content" > "${heap_profile_dir}/${hostname}.txt" + echo "Wrote ${heap_profile_dir}/${hostname}.txt" +done < "$pxl_heap_output_file" + +nodes=() +for file in "${heap_profile_dir}"/*.txt; do + hostname=$(basename "${file%.*}") + nodes+=("$hostname") + hostname_dir="${heap_profile_dir}/${hostname}" + mkdir -p "$hostname_dir" +done + +for node in "${nodes[@]}"; do + "${script_dir}/download_heap_prof_mapped_files.sh" "${heap_profile_dir}/${node}.txt" "$node" "${@:3}" +done diff --git a/scripts/download_heap_prof_mapped_files.sh b/scripts/download_heap_prof_mapped_files.sh index 82986cc64f1..d6c5b4c698d 100755 --- a/scripts/download_heap_prof_mapped_files.sh +++ b/scripts/download_heap_prof_mapped_files.sh @@ -20,13 +20,14 @@ usage() { echo "This script downloads all of the files listed in the mappings section of a heap profile." echo "" echo "Usage: $0 [...]" + echo "Common gcloud ssh options include --project." exit 1 } set -e heap_profile="$1" node_name="$2" -output_dir=/tmp/prof_bins +output_dir="${heap_profile%.txt}" if [ -z "$heap_profile" ] || [ -z "$node_name" ]; then usage @@ -44,7 +45,8 @@ mkdir -p "$output_dir" mappings=$(awk 'BEGIN{m=0} /MAPPED_LIBRARIES/{m=1} { if(m) { print $6 }}' "$heap_profile" | grep "^/" | sort | uniq) err_file="$output_dir/gcloud_error.log" -procs=$(gcloud compute ssh --command='ps ax' "$node_name" "${@:3}" 2> "$err_file") || cat "$err_file" && rm "$err_file" +zone=$(gcloud compute instances list "${@:3}" --filter="$node_name" --format="table(name, zone)"| tail -n 1 | awk '{print $2}') +procs=$(gcloud compute ssh --zone "$zone" --command='ps ax' "$node_name" "${@:3}" 2> "$err_file") || cat "$err_file" && rm "$err_file" # Find the mapping that corresponds to a process on the node. # We assume that the process was started by running one of the files in the mappings @@ -79,15 +81,15 @@ output_on_err() { } # Create tar archive on node. -output_on_err gcloud compute ssh --command="$create_tar_cmd" "$node_name" "${@:3}" +output_on_err gcloud compute ssh --zone "$zone" --command="$create_tar_cmd" "$node_name" "${@:3}" # Copy archive to local machine. -output_on_err gcloud compute scp "${@:3}" "$USER@$node_name:~/$tar_file" "/tmp/$tar_file" +output_on_err gcloud compute scp --zone "$zone" "${@:3}" "$USER@$node_name:~/$tar_file" "${output_dir}/$tar_file" # Cleanup tar archive on node. -output_on_err gcloud compute ssh --command="rm ~/$tar_file" "$node_name" "${@:3}" +output_on_err gcloud compute ssh --zone "$zone" --command="rm ~/$tar_file" "$node_name" "${@:3}" -tar --strip-components=1 -C "$output_dir" -xzf "/tmp/$tar_file" +tar --strip-components=1 -C "$output_dir" -xzf "${output_dir}/$tar_file" echo "Dumped mapped binaries to $output_dir" echo "Run 'PPROF_BINARY_PATH=$output_dir pprof -http=localhost:8888 $heap_profile' to visualize the profile." diff --git a/src/pxl_scripts/px/collect_heap_dumps.pxl b/src/pxl_scripts/px/collect_heap_dumps.pxl new file mode 100644 index 00000000000..632aee49abf --- /dev/null +++ b/src/pxl_scripts/px/collect_heap_dumps.pxl @@ -0,0 +1,8 @@ +import px + +df = px.GetAgentStatus(False) +df = df[['asid', 'hostname']] +heap_stats = px._HeapGrowthStacks() +df = df.merge(heap_stats, how='inner', left_on='asid', right_on='asid') +df = df[['hostname', 'heap']] +px.display(df) From 4a6c382433508f462202eb46226df7a7d933d72f Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Fri, 28 Mar 2025 06:05:29 +0000 Subject: [PATCH 2/3] Fix linting Signed-off-by: Dom Del Nano --- src/pxl_scripts/px/collect_heap_dumps.pxl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/pxl_scripts/px/collect_heap_dumps.pxl b/src/pxl_scripts/px/collect_heap_dumps.pxl index 632aee49abf..0f13080ac2a 100644 --- a/src/pxl_scripts/px/collect_heap_dumps.pxl +++ b/src/pxl_scripts/px/collect_heap_dumps.pxl @@ -1,3 +1,19 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + import px df = px.GetAgentStatus(False) From ce2a501059ba20edb5b916b2b7e14536da74dc6b Mon Sep 17 00:00:00 2001 From: Dom Del Nano Date: Fri, 28 Mar 2025 23:05:40 +0000 Subject: [PATCH 3/3] Move set -x to shebang line Signed-off-by: Dom Del Nano --- scripts/collect_heap_pprofs.sh | 4 +--- scripts/download_heap_prof_mapped_files.sh | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/collect_heap_pprofs.sh b/scripts/collect_heap_pprofs.sh index 86001790de6..a4dd040b772 100755 --- a/scripts/collect_heap_pprofs.sh +++ b/scripts/collect_heap_pprofs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e # Copyright 2018- The Pixie Authors. # @@ -26,8 +26,6 @@ usage() { exit 1 } -set -e - heap_profile_dir="$1" cluster_id="$2" script_dir=$(dirname "$(realpath "$0")") diff --git a/scripts/download_heap_prof_mapped_files.sh b/scripts/download_heap_prof_mapped_files.sh index d6c5b4c698d..8d86bfd11e0 100755 --- a/scripts/download_heap_prof_mapped_files.sh +++ b/scripts/download_heap_prof_mapped_files.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e # Copyright 2018- The Pixie Authors. # @@ -23,7 +23,6 @@ usage() { echo "Common gcloud ssh options include --project." exit 1 } -set -e heap_profile="$1" node_name="$2"