Refactor Master Pipeline (#1278)

* Refactor master pipeline

* Try cacheing target too

* Setup signing as template

* Change to steps

* Actually use cached maven for deploy

* Optionally set up signing files during initial build

* Split deployment to parallel jobs

* Split deployment to parallel jobs

* Add more signing options

* Shift settings.xml to temp directory

* Shift settings.xml to temp directory 2

* Temporary debug mode

* Skip install... we already did it

* deploy:deploy

* jar:jar

* Add deploy improvements to github deploy

* Remove debug in sonatype deploy

cache-target-tasks-template.yml

@@ -4,4 +4,4 @@ steps:
     displayName: Cache ${{ value }} target
     inputs:
       key: maven | $(Build.BuildId) | targets | "${{ value }}"
-      path: $(System.DefaultWorkingDirectory)/${{ value }}/target/
+      path: $(System.DefaultWorkingDirectory)/${{ value }}/

master-branch-pipeline.yml

@@ -5,104 +5,124 @@ pr: none
 trigger:
 - master
 
-pool:
+parameters:
-  vmImage: "ubuntu-22.04"
+  - name: modulesToCache
+    displayName: Module Target Directories to Cache
+    type: object
+    default:
+      modules:
+        - org.hl7.fhir.utilities
+        - org.hl7.fhir.dstu2
+        - org.hl7.fhir.dstu2016may
+        - org.hl7.fhir.dstu3
+        - org.hl7.fhir.r4
+        - org.hl7.fhir.r4b
+        - org.hl7.fhir.r5
+        - org.hl7.fhir.convertors
+        - org.hl7.fhir.validation
+        - org.hl7.fhir.validation.cli
+        - org.hl7.fhir.report
+  - name: modulesToTest
+    displayName: Modules to Test
+    type: object
+    default:
+      modules:
+        - utilities
+        - dstu2
+        - dstu2016may
+        - dstu3
+        - r4
+        - r4b
+        - r5
+        - convertors
+        - validation
 
 variables:
-- group: PGP_VAR_GROUP
+  - group: PGP_VAR_GROUP
-- group: SONATYPE_VAR_GROUP
+  - group: SONATYPE_VAR_GROUP
-- group: GIT_VAR_GROUP
+  - group: GIT_VAR_GROUP
+  - name: MAVEN_CACHE_FOLDER
+    value: $(Pipeline.Workspace)/.m2/repository
 
-steps:
+jobs:
-  # We need a valid signing key. 
+  - template: setup-and-cache-job-template.yml
-  # The next two steps download the public and private keys from the
+    parameters:
-  # secure library files. 
+      modulesToCache:
-  - task: DownloadSecureFile@1
+        ${{ parameters.modulesToCache.modules }}
-    displayName: 'Download public key.'
+      signArtifacts: true
-    inputs:
+  - template: test-unit-jobs-template.yml
-      secureFile: public.key
+    parameters:
+      testUnits:
+        - name: ubuntu_java_11
+          vmImage: "ubuntu-latest"
+          javaToolOptions:
+          jdkVersion: 1.11
+          skipJaCoCo: true
+          modules:
+            ${{ parameters.modulesToTest.modules }}
 
-  - task: DownloadSecureFile@1
+  - job: deploy_to_sonatype
-    displayName: 'Download private key.'
+    dependsOn:
-    inputs:
+      - ${{ each module in parameters.modulesToTest.modules }}:
-      secureFile: private.key
+          - ubuntu_java_11_${{ module }}
+    pool:
+      vmImage: "ubuntu-latest"
+    steps:
+      - task: Cache@2
+        displayName: Cache maven artifacts
+        inputs:
+          key: maven | $(Build.BuildId) | artifacts
+          path: $(MAVEN_CACHE_FOLDER)
+      - template: cache-target-tasks-template.yml
+        parameters:
+          modules:
+            ${{ parameters.modulesToCache.modules }}
 
-  # Import both the private and public keys into gpg for signing.
+      - template: setup-signing-tasks-template.yml
-  - bash: |
-      gpg --import --no-tty --batch --yes $(Agent.TempDirectory)/public.key
-      gpg --import --no-tty --batch --yes $(Agent.TempDirectory)/private.key
-      gpg --list-keys --keyid-format LONG
-      gpg --list-secret-keys --keyid-format LONG
-    displayName: 'Import signing keys.'
 
-  # For creating the snapshot release with maven, we need to build a fake settings.xml 
+      # Deploy the SNAPSHOT artifact to sonatype nexus.
-  # for it to read from. This is done for the master branch merges only.
+      # This is done for the master branch merges only.
-  - bash: |
+      - task: Maven@3
-      cat >$(System.DefaultWorkingDirectory)/settings.xml <<EOL
+        displayName: 'Deploy to Sonatype staging'
-      <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+        inputs:
-                xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          mavenPomFile: '$(System.DefaultWorkingDirectory)/pom.xml'
-                xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+          goals: jar:jar deploy:deploy
-                                    https://maven.apache.org/xsd/settings-1.0.0.xsd">
+          javaHomeOption: 'JDKVersion'
-        <servers>
+          jdkVersionOption: '1.11'
-          <server>
+          jdkArchitectureOption: 'x64'
-            <id>github</id>
+          options: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) --settings $(Agent.TempDirectory)/settings.xml -pl "!org.hl7.fhir.report, !org.hl7.fhir.validation.cli" -Dmaven.test.skip -DdeployToSonatype'
-            <username>$(GIT_USER_NAME)</username>
+          mavenOptions: '-Xmx768m -Dmaven.resolver.transport=wagon'
-            <password>$(GIT_PAT)</password>
+          publishJUnitResults: false
-          </server>
-          <server>
-            <id>ossrh</id>
-            <username>$(SONATYPE_USER)</username>
-            <password>$(SONATYPE_PASS)</password>
-          </server>
-          <server>
-            <id>$(PGP_KEYNAME)</id>
-            <passphrase>$(PGP_PASSPHRASE)</passphrase>
-          </server>
-          <server>
-            <id>github-releases</id>
-            <username>markiantorno</username>
-            <password>$(GIT_PACKAGE_PAT)</password>
-          </server>
-        </servers>
-        <profiles>
-          <profile>
-            <id>release</id>
-            <activation>
-              <activeByDefault>true</activeByDefault>
-            </activation>
-            <properties>
-              <gpg.keyname>$(PGP_KEYNAME)</gpg.keyname>
-            </properties>
-          </profile>
-        </profiles>
-      </settings>
-      EOL
-    displayName: 'Create .mvn/settings.xml'
 
-  # Deploy the SNAPSHOT artifact to sonatype nexus.
+  - job: deploy_to_github
-  # This is done for the master branch merges only.
+    dependsOn:
-  - task: Maven@3
+      - ${{ each module in parameters.modulesToTest.modules }}:
-    displayName: 'Deploy to Sonatype staging'
+          - ubuntu_java_11_${{ module }}
-    inputs:
+    pool:
-      mavenPomFile: '$(System.DefaultWorkingDirectory)/pom.xml'
+      vmImage: "ubuntu-latest"
-      goals: deploy
+    steps:
-      javaHomeOption: 'JDKVersion'
+      - task: Cache@2
-      jdkVersionOption: '1.11'
+        displayName: Cache maven artifacts
-      jdkArchitectureOption: 'x64'
+        inputs:
-      options: '--settings $(System.DefaultWorkingDirectory)/settings.xml -pl "!org.hl7.fhir.report, !org.hl7.fhir.validation.cli" -DdeployToSonatype'
+          key: maven | $(Build.BuildId) | artifacts
-      mavenOptions: '-Xmx768m -Dmaven.resolver.transport=wagon'
+          path: $(MAVEN_CACHE_FOLDER)
-      publishJUnitResults: false
+      - template: cache-target-tasks-template.yml
+        parameters:
+          modules:
+            ${{ parameters.modulesToCache.modules }}
 
-  # Deploy the SNAPSHOT artifact to GitHub packages.
+      - template: setup-signing-tasks-template.yml
-  # This is done for the master branch merges only.
+
-  - task: Maven@3
+      # Deploy the SNAPSHOT artifact to GitHub packages.
-    displayName: 'Deploy to GitHub packages'
+      # This is done for the master branch merges only.
-    inputs:
+      - task: Maven@3
-      mavenPomFile: '$(System.DefaultWorkingDirectory)/pom.xml'
+        displayName: 'Deploy to GitHub packages'
-      goals: deploy
+        inputs:
-      javaHomeOption: 'JDKVersion'
+          mavenPomFile: '$(System.DefaultWorkingDirectory)/pom.xml'
-      jdkVersionOption: '1.11'
+          goals: jar:jar deploy:deploy
-      jdkArchitectureOption: 'x64'
+          javaHomeOption: 'JDKVersion'
-      options: '--settings $(System.DefaultWorkingDirectory)/settings.xml -pl "!org.hl7.fhir.report, !org.hl7.fhir.validation.cli" -Dmaven.test.skip -DdeployToGitHub'
+          jdkVersionOption: '1.11'
-      mavenOptions: '-Xmx768m  -Dmaven.resolver.transport=wagon'
+          jdkArchitectureOption: 'x64'
-      publishJUnitResults: false
+          options: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) --settings $(Agent.TempDirectory)/settings.xml -pl "!org.hl7.fhir.report, !org.hl7.fhir.validation.cli" -Dmaven.test.skip -DdeployToGitHub'
+          mavenOptions: '-Xmx768m  -Dmaven.resolver.transport=wagon'
+          publishJUnitResults: false

setup-and-cache-job-template.yml

@@ -19,10 +19,15 @@ jobs:
         inputs:
           targetType: 'inline'
           script: mkdir -p $(MAVEN_CACHE_FOLDER); pwd; ls -al $(MAVEN_CACHE_FOLDER)
+      - ${{ if eq(parameters.signArtifacts, true) }}:
+        - template: setup-signing-tasks-template.yml
       - task: Maven@3
         inputs:
           mavenPomFile: 'pom.xml'
-          options: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -DskipTests'
+          ${{ if eq(parameters.signArtifacts, true) }}:
+            options: '--settings $(Agent.TempDirectory)/settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -DskipTests -DdeployToSonatype'
+          ${{ else }}:
+            options: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -DskipTests'
           mavenOptions: '-Xmx3072m'
           javaHomeOption: 'JDKVersion'
           jdkVersionOption: '1.11'

setup-signing-tasks-template.yml

@@ -0,0 +1,65 @@
+steps:
+  # We need a valid signing key.
+  # The next two steps download the public and private keys from the
+  # secure library files.
+  - task: DownloadSecureFile@1
+    displayName: 'Download public key.'
+    inputs:
+      secureFile: public.key
+
+  - task: DownloadSecureFile@1
+    displayName: 'Download private key.'
+    inputs:
+      secureFile: private.key
+
+  # Import both the private and public keys into gpg for signing.
+  - bash: |
+      gpg --import --no-tty --batch --yes $(Agent.TempDirectory)/public.key
+      gpg --import --no-tty --batch --yes $(Agent.TempDirectory)/private.key
+      gpg --list-keys --keyid-format LONG
+      gpg --list-secret-keys --keyid-format LONG
+    displayName: 'Import signing keys.'
+
+  # For creating the snapshot release with maven, we need to build a fake settings.xml
+  # for it to read from. This is done for the master branch merges only.
+  - bash: |
+      cat >$(Agent.TempDirectory)/settings.xml <<EOL
+      <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+                xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+                                    https://maven.apache.org/xsd/settings-1.0.0.xsd">
+        <servers>
+          <server>
+            <id>github</id>
+            <username>$(GIT_USER_NAME)</username>
+            <password>$(GIT_PAT)</password>
+          </server>
+          <server>
+            <id>ossrh</id>
+            <username>$(SONATYPE_USER)</username>
+            <password>$(SONATYPE_PASS)</password>
+          </server>
+          <server>
+            <id>$(PGP_KEYNAME)</id>
+            <passphrase>$(PGP_PASSPHRASE)</passphrase>
+          </server>
+          <server>
+            <id>github-releases</id>
+            <username>markiantorno</username>
+            <password>$(GIT_PACKAGE_PAT)</password>
+          </server>
+        </servers>
+        <profiles>
+          <profile>
+            <id>release</id>
+            <activation>
+              <activeByDefault>true</activeByDefault>
+            </activation>
+            <properties>
+              <gpg.keyname>$(PGP_KEYNAME)</gpg.keyname>
+            </properties>
+          </profile>
+        </profiles>
+      </settings>
+      EOL
+    displayName: 'Create settings.xml'