Add make targets for benchmark tooling

Author: kishen-v

Makefile

@@ -17,10 +17,36 @@ else
 	@echo "benchmark tool already installed..."
 endif
 
-.PHONY: bench-put
-bench-put: build install-benchmark
-	@echo "Running benchmark: put $(ARGS)"
-	./scripts/benchmark_test.sh put $(ARGS)
+.PHONY: bench-lease-keepalive bench-put bench-txn-mixed bench-watch-latency bench-watch bench-range-key
+## ----------------------------------------------------------------------------
+## etcd Benchmark Operations
+## Run the corresponding operation with the relevant arguments set through ARGS, after setting up an etcd server to bench against.
+## Eg. make bench-put ARGS="--clients=100 --conns=10"
+## Targets:
+##   bench-lease-keepalive:     Run the benchmark lease-keepalive operation with optional ARGS.
+bench-lease-keepalive: TEST := lease-keepalive
+##   bench-put:                 Run the benchmark put operation with optional ARGS.
+bench-put: TEST := put
+##   bench-txn-mixed:           Run the benchmark txn-mixed operation with optional ARGS.
+bench-txn-mixed: TEST := txn-mixed
+##   bench-watch-latency:       Run the benchmark watch-latency operation with optional ARGS.
+bench-watch-latency: TEST := watch-latency
+##   bench-watch:               Run the benchmark watch operation with optional ARGS.
+bench-watch: TEST := watch
+##   bench-range-key:           Run the benchmark range-key operation with optional ARGS.
+bench-range-key: TEST := range
+bench-range-key: ARGS := key
+
+bench-lease-keepalive bench-put bench-txn-mixed bench-watch-latency bench-watch bench-range-key: build install-benchmark
+	@echo "Running benchmark: $(TEST) $(ARGS)"
+	./scripts/benchmark_test.sh "$(TEST):$(ARGS)"
+
+##   bench:                     Run a custom benchmark with TEST_ARGS which can be a combination of tests with arguments.
+##                              Eg. make bench TEST_ARGS='put:"--clients=1000" range:"key --conns=10"'
+## ----------------------------------------------------------------------------
+.PHONY: bench
+bench: build install-benchmark
+	./scripts/benchmark_test.sh $(TEST_ARGS)
 
 PLATFORMS=linux-amd64 linux-386 linux-arm linux-arm64 linux-ppc64le linux-s390x darwin-amd64 darwin-arm64 windows-amd64 windows-arm64
 
@@ -253,3 +279,7 @@ markdown-diff-lint:
 .PHONY: update-go-workspace
 update-go-workspace:
 	./scripts/update_go_workspace.sh
+
+.PHONY: help
+help:
+	@sed -ne '/@sed/!s/## //p' $(MAKEFILE_LIST)

pkg/report/perfdash.go

@@ -74,7 +74,7 @@ func (r *report) writePerfDashReport(benchmarkOp string) {
 		artifactsDir = "./_artifacts"
 	}
 
-	fileName := fmt.Sprintf("EtcdAPI_benchmark_%s.json", time.Now().UTC().Format(time.RFC3339))
+	fileName := fmt.Sprintf("EtcdAPI_benchmark_%s_%s.json", benchmarkOp, time.Now().UTC().Format(time.RFC3339))
 	err := os.MkdirAll(artifactsDir, 0o755)
 	if err != nil {
 		fmt.Println("Error creating artifacts directory:", err)

scripts/benchmark_test.sh

@@ -13,53 +13,100 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This script runs a benchmark on a locally started etcd server
+# This utility script helps to set up the test environment for benchmark tests and runs the provided tests sequentially.
 
 set -euo pipefail
 
 source ./scripts/test_lib.sh
 
-COMMON_BENCHMARK_FLAGS="--report-perfdash"
+declare -A BENCHMARK_FLAGS
+BENCHMARKS_TO_RUN=()
+COMMON_BENCHMARK_FLAGS=(--report-perfdash)
 
-if [[ $# -lt 1 ]]; then
-  echo "Usage: $0 <benchmark-name> [tester args...]"
-  exit 1
-fi
+parse_args() {
+  for arg in "$@"; do
+    if [[ "$arg" == *:* ]]; then
+      key="${arg%%:*}"
+      value="${arg#*:}"
+      BENCHMARKS_TO_RUN+=("$key")
+      BENCHMARK_FLAGS["$key"]="$value"
+    else
+      BENCHMARKS_TO_RUN+=("$arg")
+      BENCHMARK_FLAGS["$arg"]=""
+    fi
+  done
 
-BENCHMARK_NAME="$1"
-ARGS="${*:2}"
+  if [ ${#BENCHMARKS_TO_RUN[@]} -eq 0 ]; then
+    log_error "Usage: ./benchmark_test.sh 'benchmark-operation:\"test_arguments\"'"
+    log_error "Example: ./benchmark_test.sh 'put:\"--clients=1000\" range:\"--conns=100\"'"
+    exit 1
+  fi
+}
 
-echo "Starting the etcd server..."
+start_etcd() {
+  log_callout "Setup: Starting the etcd server..."
+  # Create a directory for etcd data under /tmp/etcd
+  mkdir -p /tmp/etcd
+  DATA_DIR=$(mktemp -d /tmp/etcd/data-XXXXXX)
+  ARTIFACTS_DIR="${ARTIFACTS:-./_artifacts}"
+  mkdir -p "$ARTIFACTS_DIR"
+  ETCD_LOG_FILE="${ARTIFACTS_DIR}/etcd-${bench}-$(date +%Y%m%d-%H%M%S).log"
+  log_callout -e "Setup: etcd log file path set to $ETCD_LOG_FILE. DATA_DIR set to $DATA_DIR"
+  ./bin/etcd --data-dir="$DATA_DIR" > "${ETCD_LOG_FILE}" 2>&1 &
+  ETCD_PID=$!
+  RETRY=0
+  MAX_RETRY_ATTEMPTS=10
 
-# Create a directory for etcd data under /tmp/etcd
-mkdir -p /tmp/etcd
-DATA_DIR=$(mktemp -d /tmp/etcd/data-XXXXXX)
-./bin/etcd --data-dir="$DATA_DIR" > /tmp/etcd.log 2>&1 &
-etcd_pid=$!
+  # Set up the trap to handle errors/interrupts which leads to cleaning up of the DATADIR and stopping the etcd server.
+  trap 'stop_and_cleanup_etcd' EXIT
 
-trap 'log_warning -e "Stopping etcd server - PID $etcd_pid";
-      kill $etcd_pid 2>/dev/null;
-      rm -rf $DATA_DIR;
-      log_success "Deleted the contents from $DATA_DIR related to benchmark test"' EXIT
+  # Poll until etcd is healthy
+  until curl -fs http://127.0.0.1:2379/health | grep -q '"health":"true"'; do
+    RETRY=$((RETRY + 1))
+    if [[ $RETRY -gt $MAX_RETRY_ATTEMPTS ]]; then
+      log_error -e "Setup: Failed to confirm etcd health after $MAX_RETRY_ATTEMPTS attempts."
+      exit 1
+    fi
+    log_warning -e "Setup: Waiting for etcd to be healthy... (retry: $RETRY/$MAX_RETRY_ATTEMPTS)"
+    sleep 1
+  done
+  log_success -e "Setup: etcd is healthy and running on pid $ETCD_PID"
+}
 
-# Wait until etcd becomes healthy
-for retry in {1..10}; do
-  if ./bin/etcdctl endpoint health --cluster> /dev/null 2>&1; then
-    log_success -e "\\netcd is healthy"
-    break
-  fi
-  log_warning -e "\\nWaiting for etcd to be healthy..."
-  sleep 1
-  if [[ $retry -eq 10 ]]; then
-    log_error -e "\\nFailed to confirm etcd health after $retry attempts. Check /tmp/etcd.log for more information"
-    exit 1
+stop_and_cleanup_etcd() {
+  trap - EXIT
+  log_warning -e "Cleanup: Stopping etcd server - PID $ETCD_PID"
+  kill "$ETCD_PID" 2>/dev/null || true
+  rm -rf "$DATA_DIR"
+  log_success "Cleanup: Deleted the DATA_DIR contents from $DATA_DIR related to benchmark test"
+}
+
+run_benchmark() {
+  local bench=$1
+  local args="${BENCHMARK_FLAGS[$bench]:-}"
+
+  if [[ -z "$args" ]]; then
+    log_callout -e "\\nPerforming benchmark $bench with default arguments"
+    benchmark "$bench" "${COMMON_BENCHMARK_FLAGS[@]}"
+  else
+    log_callout -e "\\nPerforming benchmark $bench with arguments: $args"
+    read -r -a TESTER_OPTIONS <<< "$args"
+
+    printf "Running: benchmark %s %s %s\n" \
+      "$bench" "${TESTER_OPTIONS[*]}" "${COMMON_BENCHMARK_FLAGS[*]}"
+
+    benchmark "$bench" "${TESTER_OPTIONS[@]}" "${COMMON_BENCHMARK_FLAGS[@]}"
   fi
-done
+}
+
+main() {
+  parse_args "$@"
 
-log_success -e "etcd process is running with PID $etcd_pid"
+  for bench in "${BENCHMARKS_TO_RUN[@]}"; do
+    start_etcd
+    run_benchmark "$bench"
+    stop_and_cleanup_etcd
+  done
+}
 
-log_callout -e "\\nPerforming benchmark $BENCHMARK_NAME with arguments: $ARGS"
-read -r -a TESTER_OPTIONS <<< "$ARGS"
-log_callout "Running: benchmark $BENCHMARK_NAME ${TESTER_OPTIONS[*]} $COMMON_BENCHMARK_FLAGS"
-benchmark "$BENCHMARK_NAME" "${TESTER_OPTIONS[@]}" $COMMON_BENCHMARK_FLAGS
-log_callout "Completed: benchmark $BENCHMARK_NAME ${TESTER_OPTIONS[*]} $COMMON_BENCHMARK_FLAGS"
+main "$@"