From 58ca67a08d5e3fe2612a523f655bc64a5d0a2f27 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 18 Dec 2024 14:43:37 -0500
Subject: [PATCH] aws jmh benchmark cleanups (#14072)

* remove hardcoded lucene version, gets benchie working again

* add graviton4 instance type (c8g)

* use https clone to not require agent forwarding

* document prerequisites needed for this to work

* convert README to markdown

* apparently .md files need a license but .txt files do not
---
 dev-tools/aws-jmh/README.md          | 39 ++++++++++++++++++++++++++++
 dev-tools/aws-jmh/README.txt         | 13 ----------
 dev-tools/aws-jmh/group_vars/all.yml |  7 ++++-
 dev-tools/aws-jmh/playbook.yml       | 25 ++++++++++++------
 4 files changed, 62 insertions(+), 22 deletions(-)
 create mode 100644 dev-tools/aws-jmh/README.md
 delete mode 100644 dev-tools/aws-jmh/README.txt

diff --git a/dev-tools/aws-jmh/README.md b/dev-tools/aws-jmh/README.md
new file mode 100644
index 00000000000..398a3fbdaa5
--- /dev/null
+++ b/dev-tools/aws-jmh/README.md
@@ -0,0 +1,39 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+ -->
+
+Runs lucene microbenchmarks across a variety of CPUs in EC2.
+
+Example:
+
+```console
+export AWS_ACCESS_KEY_ID=xxxxx
+export AWS_SECRET_ACCESS_KEY=yyyy
+make PATCH_BRANCH=rmuir:some-speedup
+```
+
+Results file will be in build/report.txt
+
+You can also pass additional JMH args if you want:
+
+```console
+make PATCH_BRANCH=rmuir:some-speedup JMH_ARGS='float -p size=756'
+```
+
+Prerequisites:
+
+1. It is expected that you have an ed25519 ssh key, use `ssh-keygen -t ed25519` to make one.
+2. AWS key's IAM user needs `AmazonEC2FullAccess` and `AWSCloudFormationFullAccess` permissions at a minimum.
diff --git a/dev-tools/aws-jmh/README.txt b/dev-tools/aws-jmh/README.txt
deleted file mode 100644
index ee2d4bb322f..00000000000
--- a/dev-tools/aws-jmh/README.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-Runs lucene microbenchmarks across a variety of CPUs in EC2.
-
-Example:
-
-export AWS_ACCESS_KEY_ID=xxxxx
-export AWS_SECRET_ACCESS_KEY=yyyy
-make PATCH_BRANCH=rmuir:some-speedup
-
-Results file will be in build/report.txt
-
-You can also pass additional JMH args if you want:
-
-make PATCH_BRANCH=rmuir:some-speedup JMH_ARGS='float -p size=756'
diff --git a/dev-tools/aws-jmh/group_vars/all.yml b/dev-tools/aws-jmh/group_vars/all.yml
index 23c14c70b1e..20d4aaf3e3e 100644
--- a/dev-tools/aws-jmh/group_vars/all.yml
+++ b/dev-tools/aws-jmh/group_vars/all.yml
@@ -43,6 +43,9 @@ instances:
   - name: graviton3
     type: c7g.large
     ami: "{{ arm_ami }}"
+  - name: graviton4
+    type: c8g.large
+    ami: "{{ arm_ami }}"
 
 jdk_version: 21.0.1
 x86_jdk_url: "https://download.java.net/java/GA/jdk{{ jdk_version }}/415e3f918a1f4062a0074a2794853d0d/12/GPL/openjdk-{{ jdk_version }}_linux-x64_bin.tar.gz"
@@ -53,7 +56,9 @@ main_branch: "apache:main"
 patch_branch: "{{ lookup('env', 'PATCH_BRANCH') }}"
 
 assemble_command: ./gradlew -p lucene/benchmark-jmh assemble
-jmh_command: java -jar lucene/benchmark-jmh/build/benchmarks/lucene-benchmark-jmh-10.0.0-SNAPSHOT.jar -rf text
+# must match benchmark jar to run
+jmh_jar_path: "lucene/benchmark-jmh/build/benchmarks/"
+jmh_jar_glob: "lucene-benchmark-jmh-*.jar"
 jmh_args: "{{ lookup('env', 'JMH_ARGS', default='-p size=1024') }}"
 
 # services that cause noise and don't help us
diff --git a/dev-tools/aws-jmh/playbook.yml b/dev-tools/aws-jmh/playbook.yml
index 197a415c23e..82393d5b876 100644
--- a/dev-tools/aws-jmh/playbook.yml
+++ b/dev-tools/aws-jmh/playbook.yml
@@ -90,18 +90,16 @@
 
     - name: Checkout main
       git:
-        repo: "git@github.com:{{ (main_branch | split(':'))[0] }}/lucene.git"
+        repo: "https://github.com/{{ (main_branch | split(':'))[0] }}/lucene.git"
         version: "{{ (main_branch | split(':'))[1] }}"
         dest: main
-        accept_newhostkey: true
         depth: 1
 
     - name: Checkout patch
       git:
-        repo: "git@github.com:{{ (patch_branch | split(':'))[0] }}/lucene.git"
+        repo: "https://github.com/{{ (patch_branch | split(':'))[0] }}/lucene.git"
         version: "{{ (patch_branch | split(':'))[1] }}"
         dest: patch
-        accept_newhostkey: true
         depth: 1
 
     - name: Assemble Sources
@@ -112,14 +110,25 @@
         - main
         - patch
 
-    - name: Run benchmark
-      command:
-        cmd: "{{ jmh_command }} -rff ~/{{ item }}.txt {{ jmh_args }}"
-        chdir: "{{ item }}"
+    - name: Locate benchmark jar
+      find:
+        paths: "{{ item }}/{{ jmh_jar_path }}"
+        pattern: "{{ jmh_jar_glob }}"
+      register: found_jmh_jars
       loop:
         - main
         - patch
 
+    - name: Run benchmark
+      command:
+        cmd: "java -jar {{ jmh_jar }} -rf text -rff ~/{{ item }}.txt {{ jmh_args }}"
+      loop:
+        - main
+        - patch
+      vars:
+        jmh_jars: "{{ found_jmh_jars.results | selectattr('item', 'eq', item) | map(attribute='files') }}"
+        jmh_jar: "{{ jmh_jars | first | map(attribute='path') | first }}"
+
     - name: Read main results
       command:
         cmd: "cat main.txt"