script to run microbenchmarks across different ec2 instance types (#12787)

dev-tools/aws-jmh/Makefile

@@ -0,0 +1,45 @@
+.PHONY: test clean env
+.DELETE_ON_ERROR:
+
+# python interpreter to use
+export PYTHON=python3
+# improve ansible output
+export ANSIBLE_CALLBACK_RESULT_FORMAT=yaml
+# reasonable default verbosity, crank to debug issues
+export ANSIBLE_VERBOSITY=0
+# pipeline operations over ssh
+export ANSIBLE_PIPELINING=true
+# show breakdown of task times
+export ANSIBLE_CALLBACKS_ENABLED=profile_tasks
+# not useful since they are throwaways
+export ANSIBLE_HOST_KEY_CHECKING=false
+# run all machines in parallel
+export ANSIBLE_FORKS=10
+
+ifeq ($(AWS_ACCESS_KEY_ID),)
+  $(error AWS_ACCESS_KEY_ID is not set)
+endif
+
+ifeq ($(AWS_SECRET_ACCESS_KEY),)
+  $(error AWS_SECRET_ACCESS_KEY is not set)
+endif
+
+ifeq ($(PATCH_BRANCH),)
+  $(error PATCH_BRANCH is not set)
+endif
+
+# runs benchmark across AWS instances
+# always destroys anything created out there
+test: env
+	bash -c "set -ex; trap 'build/.env/bin/ansible-playbook destroy.yml' EXIT; build/.env/bin/ansible-playbook playbook.yml"
+
+clean:
+	rm -rf build
+
+# keep venv in sync with requirements.txt
+env: build/.env/bin/activate
+build/.env/bin/activate: requirements.txt
+	rm -rf build/.env
+	$(PYTHON) -m venv build/.env
+	build/.env/bin/pip install --no-input -U -r requirements.txt
+	touch build/.env/bin/activate

dev-tools/aws-jmh/README.txt

@@ -0,0 +1,13 @@
+Runs lucene microbenchmarks across a variety of CPUs in EC2.
+
+Example:
+
+export AWS_ACCESS_KEY_ID=xxxxx
+export AWS_SECRET_ACCESS_KEY=yyyy
+make PATCH_BRANCH=rmuir:some-speedup
+
+Results file will be in build/report.txt
+
+You can also pass additional JMH args if you want:
+
+make PATCH_BRANCH=rmuir:some-speedup JMH_ARGS='float -p size=756'

dev-tools/aws-jmh/cloud-init.yml.j2

@@ -0,0 +1,8 @@
+#cloud-config
+package_upgrade: false
+package_update: false
+system_info:
+  default_user:
+    name: {{ user }}
+    ssh_authorized_keys:
+      - {{ lookup('file', ssh_key + '.pub') }}

dev-tools/aws-jmh/cloudformation.yml.j2

@@ -0,0 +1,49 @@
+AWSTemplateFormatVersion: 2010-09-09
+Resources:
+
+  SecurityGroupIngress:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      VpcId: {{ vpc_info.vpcs[0].id }}
+      GroupDescription: Allow inbound SSH traffic
+      SecurityGroupIngress:
+        - IpProtocol: "tcp"
+          FromPort: 22
+          ToPort: 22
+          CidrIp: "0.0.0.0/0"
+      Tags:
+        - Key: Name
+          Value: {{ stack_name }}-inbound-ssh
+
+  SecurityGroupEgress:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      VpcId: {{ vpc_info.vpcs[0].id }}
+      GroupDescription: Allow all outbound traffic
+      SecurityGroupEgress:
+        - IpProtocol: "-1"
+          CidrIp: "0.0.0.0/0"
+      Tags:
+        - Key: Name
+          Value: {{ stack_name }}-outbound-all
+
+{% for instance in instances %}
+  {{ instance.name }}Instance:
+    Type: AWS::EC2::Instance
+    DependsOn:
+      - SecurityGroupIngress
+      - SecurityGroupEgress
+    Properties:
+      InstanceType: {{ instance.type }}
+      InstanceInitiatedShutdownBehavior: terminate
+      ImageId: {{ instance.ami }}
+      SecurityGroupIds:
+        - !Ref SecurityGroupIngress
+        - !Ref SecurityGroupEgress
+      UserData: {{ lookup('template', 'cloud-init.yml.j2') | b64encode }}
+      Tags:
+        - Key: Name
+          Value: {{ stack_name }}-{{ instance.name }}
+        - Key: ShortName
+          Value: {{ instance.name }}
+{% endfor %}

dev-tools/aws-jmh/destroy.yml

@@ -0,0 +1,9 @@
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Destroy cloudformation stack
+      cloudformation:
+        stack_name: "{{ stack_name }}"
+        region: "{{ region }}"
+        state: absent
+      delegate_to: localhost

dev-tools/aws-jmh/group_vars/all.yml

@@ -0,0 +1,65 @@
+# cloudformation stack name
+stack_name: lucene-jmh
+
+# region to launch instances, make sure they are supported!
+region: us-east-1
+
+# ssh keypair to use for accessing instances
+ssh_key: ~/.ssh/id_ed25519
+
+# user to create remotely
+user: lucene
+
+# ami to use
+x86_ami: ami-04cb4ca688797756f
+arm_ami: ami-06f9c0b2ce386dda7
+
+# instance types to spin up
+instances:
+  - name: haswell
+    type: c4.large
+    ami: "{{ x86_ami }}"
+  - name: cascadelake
+    type: c5.large
+    ami: "{{ x86_ami }}"
+  - name: icelake
+    type: c6i.large
+    ami: "{{ x86_ami }}"
+  - name: sapphirerapids
+    type: c7i.large
+    ami: "{{ x86_ami }}"
+  - name: zen2
+    type: c5a.large
+    ami: "{{ x86_ami }}"
+  - name: zen3
+    type: c6a.large
+    ami: "{{ x86_ami }}"
+  - name: zen4
+    type: c7a.large
+    ami: "{{ x86_ami }}"
+  - name: graviton2
+    type: c6g.large
+    ami: "{{ arm_ami }}"
+  - name: graviton3
+    type: c7g.large
+    ami: "{{ arm_ami }}"
+
+jdk_version: 21.0.1
+x86_jdk_url: "https://download.java.net/java/GA/jdk{{ jdk_version }}/415e3f918a1f4062a0074a2794853d0d/12/GPL/openjdk-{{ jdk_version }}_linux-x64_bin.tar.gz"
+arm_jdk_url: "https://download.java.net/java/GA/jdk{{ jdk_version }}/415e3f918a1f4062a0074a2794853d0d/12/GPL/openjdk-{{ jdk_version }}_linux-aarch64_bin.tar.gz"
+jdk_url: "{{ arm_jdk_url if ansible_architecture == 'aarch64' else x86_jdk_url }}"
+
+main_branch: "apache:main"
+patch_branch: "{{ lookup('env', 'PATCH_BRANCH') }}"
+
+assemble_command: ./gradlew -p lucene/benchmark-jmh assemble
+jmh_command: java -jar lucene/benchmark-jmh/build/benchmarks/lucene-benchmark-jmh-10.0.0-SNAPSHOT.jar -rf text
+jmh_args: "{{ lookup('env', 'JMH_ARGS', default='-p size=1024') }}"
+
+# services that cause noise and don't help us
+disable_services:
+  - atd.service
+  - logrotate.timer
+  - sysstat-collect.timer
+  - sysstat-summary.timer
+  - systemd-tmpfiles-clean.timer

dev-tools/aws-jmh/playbook.yml

@@ -0,0 +1,155 @@
+
+# spin up aws hardware
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Lookup default VPC
+      ec2_vpc_net_info:
+        region: "{{ region }}"
+        filters:
+          isDefault: true
+      register: vpc_info
+
+    - name: Create cloudformation stack
+      cloudformation:
+        stack_name: "{{ stack_name }}"
+        region: "{{ region }}"
+        template_body: "{{ lookup('template', 'cloudformation.yml.j2') }}"
+
+    - name: Gather instance details
+      ec2_instance_info:
+        region: "{{ region }}"
+        filters:
+          instance-state-name: ["pending", "running"]
+          "tag:aws:cloudformation:stack-name": "{{ stack_name }}"
+      register: ec2_instances
+
+    - name: Add instances to inventory
+      add_host:
+        name: "{{ item['tags']['ShortName'] }}"
+        ansible_host: "{{ item['public_ip_address'] }}"
+        ansible_user: "{{ user }}"
+        ansible_ssh_private_key_file: "{{ ssh_key }}"
+      args: "{{ item }}"
+      with_items: "{{ ec2_instances.instances }}"
+
+- hosts: all
+  gather_facts: false
+  tasks:
+    - name: Wait for connection
+      wait_for_connection:
+
+    - name: Gather facts
+      setup:
+
+    - name: Configure kernel
+      command:
+        argv:
+          - grubby
+          - --remove-args=selinux_1 security=selinux quiet
+          - --args=mitigations=0 random.trust_cpu=1 loglevel=7 selinux=0
+          - --update-kernel=ALL
+      become: true
+
+    - name: Install packages
+      package:
+        name:
+          - git
+      become: true
+
+    - name: Disable noisy services
+      command:
+        cmd: "systemctl disable --now {{ item }}"
+      loop: "{{ disable_services }}"
+      failed_when: false
+      become: true
+
+    - name: Reboot machine
+      reboot:
+      become: true
+
+    - name: Download JDK
+      unarchive:
+        src: "{{ jdk_url }}"
+        dest: .
+        remote_src: true
+
+    - name: Configure JDK
+      blockinfile:
+        path: .bashrc
+        block: |
+          export JAVA_HOME=/home/{{ user }}/jdk-{{ jdk_version }}
+          export PATH=$JAVA_HOME/bin:$PATH
+
+    - name: Configure Gradle
+      blockinfile:
+        path: .gradle/gradle.properties
+        create: true
+        block: |
+          org.gradle.daemon=false
+
+    - name: Checkout main
+      git:
+        repo: "git@github.com:{{ (main_branch | split(':'))[0] }}/lucene.git"
+        version: "{{ (main_branch | split(':'))[1] }}"
+        dest: main
+        accept_newhostkey: true
+        depth: 1
+
+    - name: Checkout patch
+      git:
+        repo: "git@github.com:{{ (patch_branch | split(':'))[0] }}/lucene.git"
+        version: "{{ (patch_branch | split(':'))[1] }}"
+        dest: patch
+        accept_newhostkey: true
+        depth: 1
+
+    - name: Assemble Sources
+      command:
+        cmd: "{{ assemble_command }}"
+        chdir: "{{ item }}"
+      loop:
+        - main
+        - patch
+
+    - name: Run benchmark
+      command:
+        cmd: "{{ jmh_command }} -rff ~/{{ item }}.txt {{ jmh_args }}"
+        chdir: "{{ item }}"
+      loop:
+        - main
+        - patch
+
+    - name: Read main results
+      command:
+        cmd: "cat main.txt"
+      register: main_out
+
+    - name: Read patch results
+      command:
+        cmd: "cat patch.txt"
+      register: patch_out
+
+    - name: Write Report
+      copy:
+        dest: "build/{{ inventory_hostname }}.log"
+        content: |
+
+          {{ inventory_hostname }}: `{{ ansible_processor }}`
+
+          main
+          ```
+          {{ main_out.stdout }}
+          ```
+          patch
+          ```
+          {{ patch_out.stdout }}
+          ```
+      delegate_to: localhost
+
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Create combined report
+      shell:
+        cmd: "cat build/*.log > build/report.txt"

dev-tools/aws-jmh/requirements.txt

@@ -0,0 +1,4 @@
+# automate botnet with ansible
+ansible<9
+# aws api
+boto3<2

gradle/validation/rat-sources.gradle

@@ -82,6 +82,9 @@ allprojects {
                     // projects. Include their sources in the scan.
                     include "buildSrc/src/**"
                     include "dev-tools/missing-doclet/src/**"
+
+                    // do not let RAT attempt to scan a python venv, it gets lost and confused...
+                    exclude "dev-tools/aws-jmh/build/**"
                     break
 
                 case ":lucene:analysis:morfologik":