Skip to content

Commit 9c8cca4

Browse files
djaglowskidjaglowski
committed
Benchmark script parameterized and opens profiles
1 parent 44772ab commit 9c8cca4

File tree

5 files changed

+60
-25
lines changed

5 files changed

+60
-25
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
dev/*
2-
.DS_Store
2+
.DS_Store
3+
tmp/*

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ require (
66
cloud.google.com/go/logging v1.0.0
77
github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 // indirect
88
github.com/antonmedv/expr v1.4.5
9+
github.com/awslabs/amazon-log-agent-benchmark-tool v0.0.0-20200505071244-c5f96e667e52 // indirect
910
github.com/dustin/go-humanize v1.0.0 // indirect
1011
github.com/elastic/go-elasticsearch/v8 v8.0.0-20200429055951-e9eb76d55d12
1112
github.com/elastic/go-sysinfo v1.1.0 // indirect

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ github.com/antonmedv/expr v1.8.1/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmH
3838
github.com/antonmedv/expr v1.8.2 h1:BfkVHGudYqq7jp3Ji33kTn+qZ9D19t/Mndg0ag/Ycq4=
3939
github.com/antonmedv/expr v1.8.2/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8=
4040
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
41+
github.com/awslabs/amazon-log-agent-benchmark-tool v0.0.0-20200505071244-c5f96e667e52 h1:zB4A0kii6QBWHph1pF4/aKOdoEJuqWjfxzBNM4UF9UU=
42+
github.com/awslabs/amazon-log-agent-benchmark-tool v0.0.0-20200505071244-c5f96e667e52/go.mod h1:idbfcjPo1Aieh2gwxiAMGen6jjPMkLaYzW6X580vuk4=
4143
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
4244
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
4345
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=

‎scripts/benchmark.sh‎

100644100755
Lines changed: 43 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,58 @@
11
#!/bin/bash
22

3+
BPLOG_ROOT="$GOPATH/src/github.com/bluemedora/bplogagent"
4+
5+
PROJECT="bindplane-agent-dev-0"
6+
ZONE="us-central1-a"
7+
INSTANCE="rhel6"
8+
9+
if ! [ -x "$(command -v gcloud)" ]; then
10+
echo 'Error: gcloud is not installed.' >&2
11+
exit 1
12+
fi
13+
14+
if ! [ -x "$(command -v dot)" ]; then
15+
echo 'Error: dot is not installed.' >&2
16+
exit 1
17+
fi
18+
319
echo "Building log agent"
4-
GOOS=linux go build -o /tmp/bplogagent ../cmd
20+
GOOS=linux go build -o /tmp/bplogagent $BPLOG_ROOT/cmd
521

6-
echo "Copying log agent"
7-
gcloud beta compute --project "***REMOVED***" ssh --zone "us-east1-b" 'instance-1' -- 'rm -rf /tmp/benchmark && mkdir /tmp/benchmark'
8-
gcloud beta compute --project "***REMOVED***" scp --zone "us-east1-b" /tmp/bplogagent 'instance-1:/tmp/benchmark'
22+
echo "Building benchmark tool"
23+
GOOS=linux go build -o /tmp/logbench github.com/awslabs/amazon-log-agent-benchmark-tool/cmd/logbench/
924

10-
echo "Downloading dependencies"
11-
gcloud beta compute --project "***REMOVED***" ssh --zone "us-east1-b" 'instance-1' -- \
12-
'gsutil cp gs://bplogagent-logbench/LogBench /tmp/benchmark/LogBench && chmod +x /tmp/benchmark/LogBench'
13-
gcloud beta compute --project "***REMOVED***" ssh --zone "us-east1-b" 'instance-1' -- \
14-
'gsutil cp gs://bplogagent-logbench/config.yaml /tmp/benchmark/config.yaml'
25+
echo "Setting up benchmark run on $PROJECT $ZONE $INSTANCE"
26+
gcloud beta compute --project "$PROJECT" ssh --zone "$ZONE" "$INSTANCE" -- 'rm -rf ~/benchmark && mkdir ~/benchmark'
27+
gcloud beta compute --project "$PROJECT" scp --zone "$ZONE" /tmp/bplogagent "$INSTANCE:~/benchmark"
28+
gcloud beta compute --project "$PROJECT" scp --zone "$ZONE" /tmp/logbench "$INSTANCE:~/benchmark/LogBench"
29+
gcloud beta compute --project "$PROJECT" scp --zone "$ZONE" $BPLOG_ROOT/scripts/benchmark/config.yaml "$INSTANCE:~/benchmark/config.yaml"
30+
gcloud beta compute --project "$PROJECT" ssh --zone "$ZONE" "$INSTANCE" -- 'chmod -R 777 ~/benchmark'
1531

1632
echo "Running single-file benchmark"
17-
gcloud beta compute --project "***REMOVED***" ssh --zone "us-east1-b" 'instance-1' -- \
18-
'set -m ;
19-
cd /tmp/benchmark ;
20-
/tmp/benchmark/LogBench -log stream.log -rate 100 -t 60s -r 30s -f 2s /tmp/benchmark/bplogagent --config /tmp/benchmark/config.yaml > output1 &
33+
gcloud beta compute --project "$PROJECT" ssh --zone "$ZONE" "$INSTANCE" -- \
34+
'set -m
35+
~/benchmark/LogBench -log stream.log -rate 100 -t 60s -r 30s -f 2s ~/benchmark/bplogagent --config ~/benchmark/config.yaml > ~/benchmark/output1 2>&1 &
2136
sleep 10;
22-
curl http://localhost:6060/debug/pprof/profile?seconds=30 > /tmp/benchmark/profile1 ;
37+
curl http://localhost:6060/debug/pprof/profile?seconds=30 > ~/benchmark/profile1 ;
2338
fg ; '
2439

2540
echo "Running 20-file benchmark"
26-
gcloud beta compute --project "***REMOVED***" ssh --zone "us-east1-b" 'instance-1' -- \
27-
'set -m ;
28-
cd /tmp/benchmark ;
29-
/tmp/benchmark/LogBench -log $(echo stream{1..20}.log | tr " " ,) -rate 100 -t 60s -r 30s -f 2s /tmp/benchmark/bplogagent --config /tmp/benchmark/config.yaml > output20 &
41+
gcloud beta compute --project "$PROJECT" ssh --zone "$ZONE" "$INSTANCE" -- \
42+
'set -m
43+
~/benchmark/LogBench -log $(echo stream{1..20}.log | tr " " ,) -rate 100 -t 60s -r 30s -f 2s ~/benchmark/bplogagent --config ~/benchmark/config.yaml > ~/benchmark/output20 2>&1 &
3044
sleep 10;
31-
curl http://localhost:6060/debug/pprof/profile?seconds=30 > /tmp/benchmark/profile20 ;
45+
curl http://localhost:6060/debug/pprof/profile?seconds=30 > ~/benchmark/profile20 ;
3246
fg ; '
3347

34-
echo "Retrieving results"
35-
output_dir="/tmp/benchmark/$(date +%Y-%m-%d_%H-%M-%S)"
48+
output_dir="$BPLOG_ROOT/tmp/$(date +%Y-%m-%d_%H-%M-%S)"
3649
mkdir -p $output_dir
37-
gcloud beta compute --project "***REMOVED***" scp --zone "us-east1-b" 'instance-1:/tmp/benchmark/output*' "$output_dir"
38-
gcloud beta compute --project "***REMOVED***" scp --zone "us-east1-b" 'instance-1:/tmp/benchmark/profile*' "$output_dir"
39-
echo "Results are located in $output_dir"
50+
echo "Results will be located in $output_dir"
51+
52+
echo "Retrieving results"
53+
gcloud beta compute --project "$PROJECT" scp --zone "$ZONE" "$INSTANCE:~/benchmark/output*" "$output_dir"
54+
gcloud beta compute --project "$PROJECT" scp --zone "$ZONE" "$INSTANCE:~/benchmark/profile*" "$output_dir"
55+
56+
echo "Opening profiles"
57+
go tool pprof -http localhost:6001 $output_dir/profile1 &
58+
go tool pprof -http localhost:6020 $output_dir/profile20 &

‎scripts/benchmark/config.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
plugins:
2+
- id: fileinput
3+
type: file_input
4+
include: ["stream*.log"]
5+
output: google_cloud
6+
7+
- id: google_cloud
8+
type: google_cloud_output
9+
project_id: ***REMOVED***
10+
credentials: >-
11+
***REMOVED***
12+
database_file: /home/djaglowski/benchmark/logagent.db

0 commit comments

Comments
 (0)