From 233d0ecf86f7a93cc2be2d8b8c7f4056ba0c9ce3 Mon Sep 17 00:00:00 2001 From: dotasek Date: Fri, 26 May 2023 09:32:13 -0400 Subject: [PATCH] Refactor Master Pipeline (#1278) * Refactor master pipeline * Try cacheing target too * Setup signing as template * Change to steps * Actually use cached maven for deploy * Optionally set up signing files during initial build * Split deployment to parallel jobs * Split deployment to parallel jobs * Add more signing options * Shift settings.xml to temp directory * Shift settings.xml to temp directory 2 * Temporary debug mode * Skip install... we already did it * deploy:deploy * jar:jar * Add deploy improvements to github deploy * Remove debug in sonatype deploy --- cache-target-tasks-template.yml | 2 +- master-branch-pipeline.yml | 206 +++++++++++++++++-------------- setup-and-cache-job-template.yml | 7 +- setup-signing-tasks-template.yml | 65 ++++++++++ 4 files changed, 185 insertions(+), 95 deletions(-) create mode 100644 setup-signing-tasks-template.yml diff --git a/cache-target-tasks-template.yml b/cache-target-tasks-template.yml index 9117866f1..f29d289ac 100644 --- a/cache-target-tasks-template.yml +++ b/cache-target-tasks-template.yml @@ -4,4 +4,4 @@ steps: displayName: Cache ${{ value }} target inputs: key: maven | $(Build.BuildId) | targets | "${{ value }}" - path: $(System.DefaultWorkingDirectory)/${{ value }}/target/ + path: $(System.DefaultWorkingDirectory)/${{ value }}/ diff --git a/master-branch-pipeline.yml b/master-branch-pipeline.yml index 5974ce776..c2d85d889 100644 --- a/master-branch-pipeline.yml +++ b/master-branch-pipeline.yml @@ -5,104 +5,124 @@ pr: none trigger: - master -pool: - vmImage: "ubuntu-22.04" +parameters: + - name: modulesToCache + displayName: Module Target Directories to Cache + type: object + default: + modules: + - org.hl7.fhir.utilities + - org.hl7.fhir.dstu2 + - org.hl7.fhir.dstu2016may + - org.hl7.fhir.dstu3 + - org.hl7.fhir.r4 + - org.hl7.fhir.r4b + - org.hl7.fhir.r5 + - org.hl7.fhir.convertors + - org.hl7.fhir.validation + - org.hl7.fhir.validation.cli + - org.hl7.fhir.report + - name: modulesToTest + displayName: Modules to Test + type: object + default: + modules: + - utilities + - dstu2 + - dstu2016may + - dstu3 + - r4 + - r4b + - r5 + - convertors + - validation variables: -- group: PGP_VAR_GROUP -- group: SONATYPE_VAR_GROUP -- group: GIT_VAR_GROUP + - group: PGP_VAR_GROUP + - group: SONATYPE_VAR_GROUP + - group: GIT_VAR_GROUP + - name: MAVEN_CACHE_FOLDER + value: $(Pipeline.Workspace)/.m2/repository -steps: - # We need a valid signing key. - # The next two steps download the public and private keys from the - # secure library files. - - task: DownloadSecureFile@1 - displayName: 'Download public key.' - inputs: - secureFile: public.key +jobs: + - template: setup-and-cache-job-template.yml + parameters: + modulesToCache: + ${{ parameters.modulesToCache.modules }} + signArtifacts: true + - template: test-unit-jobs-template.yml + parameters: + testUnits: + - name: ubuntu_java_11 + vmImage: "ubuntu-latest" + javaToolOptions: + jdkVersion: 1.11 + skipJaCoCo: true + modules: + ${{ parameters.modulesToTest.modules }} - - task: DownloadSecureFile@1 - displayName: 'Download private key.' - inputs: - secureFile: private.key + - job: deploy_to_sonatype + dependsOn: + - ${{ each module in parameters.modulesToTest.modules }}: + - ubuntu_java_11_${{ module }} + pool: + vmImage: "ubuntu-latest" + steps: + - task: Cache@2 + displayName: Cache maven artifacts + inputs: + key: maven | $(Build.BuildId) | artifacts + path: $(MAVEN_CACHE_FOLDER) + - template: cache-target-tasks-template.yml + parameters: + modules: + ${{ parameters.modulesToCache.modules }} - # Import both the private and public keys into gpg for signing. - - bash: | - gpg --import --no-tty --batch --yes $(Agent.TempDirectory)/public.key - gpg --import --no-tty --batch --yes $(Agent.TempDirectory)/private.key - gpg --list-keys --keyid-format LONG - gpg --list-secret-keys --keyid-format LONG - displayName: 'Import signing keys.' + - template: setup-signing-tasks-template.yml - # For creating the snapshot release with maven, we need to build a fake settings.xml - # for it to read from. This is done for the master branch merges only. - - bash: | - cat >$(System.DefaultWorkingDirectory)/settings.xml < - - - github - $(GIT_USER_NAME) - $(GIT_PAT) - - - ossrh - $(SONATYPE_USER) - $(SONATYPE_PASS) - - - $(PGP_KEYNAME) - $(PGP_PASSPHRASE) - - - github-releases - markiantorno - $(GIT_PACKAGE_PAT) - - - - - release - - true - - - $(PGP_KEYNAME) - - - - - EOL - displayName: 'Create .mvn/settings.xml' + # Deploy the SNAPSHOT artifact to sonatype nexus. + # This is done for the master branch merges only. + - task: Maven@3 + displayName: 'Deploy to Sonatype staging' + inputs: + mavenPomFile: '$(System.DefaultWorkingDirectory)/pom.xml' + goals: jar:jar deploy:deploy + javaHomeOption: 'JDKVersion' + jdkVersionOption: '1.11' + jdkArchitectureOption: 'x64' + options: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) --settings $(Agent.TempDirectory)/settings.xml -pl "!org.hl7.fhir.report, !org.hl7.fhir.validation.cli" -Dmaven.test.skip -DdeployToSonatype' + mavenOptions: '-Xmx768m -Dmaven.resolver.transport=wagon' + publishJUnitResults: false - # Deploy the SNAPSHOT artifact to sonatype nexus. - # This is done for the master branch merges only. - - task: Maven@3 - displayName: 'Deploy to Sonatype staging' - inputs: - mavenPomFile: '$(System.DefaultWorkingDirectory)/pom.xml' - goals: deploy - javaHomeOption: 'JDKVersion' - jdkVersionOption: '1.11' - jdkArchitectureOption: 'x64' - options: '--settings $(System.DefaultWorkingDirectory)/settings.xml -pl "!org.hl7.fhir.report, !org.hl7.fhir.validation.cli" -DdeployToSonatype' - mavenOptions: '-Xmx768m -Dmaven.resolver.transport=wagon' - publishJUnitResults: false + - job: deploy_to_github + dependsOn: + - ${{ each module in parameters.modulesToTest.modules }}: + - ubuntu_java_11_${{ module }} + pool: + vmImage: "ubuntu-latest" + steps: + - task: Cache@2 + displayName: Cache maven artifacts + inputs: + key: maven | $(Build.BuildId) | artifacts + path: $(MAVEN_CACHE_FOLDER) + - template: cache-target-tasks-template.yml + parameters: + modules: + ${{ parameters.modulesToCache.modules }} - # Deploy the SNAPSHOT artifact to GitHub packages. - # This is done for the master branch merges only. - - task: Maven@3 - displayName: 'Deploy to GitHub packages' - inputs: - mavenPomFile: '$(System.DefaultWorkingDirectory)/pom.xml' - goals: deploy - javaHomeOption: 'JDKVersion' - jdkVersionOption: '1.11' - jdkArchitectureOption: 'x64' - options: '--settings $(System.DefaultWorkingDirectory)/settings.xml -pl "!org.hl7.fhir.report, !org.hl7.fhir.validation.cli" -Dmaven.test.skip -DdeployToGitHub' - mavenOptions: '-Xmx768m -Dmaven.resolver.transport=wagon' - publishJUnitResults: false + - template: setup-signing-tasks-template.yml + + # Deploy the SNAPSHOT artifact to GitHub packages. + # This is done for the master branch merges only. + - task: Maven@3 + displayName: 'Deploy to GitHub packages' + inputs: + mavenPomFile: '$(System.DefaultWorkingDirectory)/pom.xml' + goals: jar:jar deploy:deploy + javaHomeOption: 'JDKVersion' + jdkVersionOption: '1.11' + jdkArchitectureOption: 'x64' + options: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) --settings $(Agent.TempDirectory)/settings.xml -pl "!org.hl7.fhir.report, !org.hl7.fhir.validation.cli" -Dmaven.test.skip -DdeployToGitHub' + mavenOptions: '-Xmx768m -Dmaven.resolver.transport=wagon' + publishJUnitResults: false \ No newline at end of file diff --git a/setup-and-cache-job-template.yml b/setup-and-cache-job-template.yml index c346f5ccf..311b0ce8d 100644 --- a/setup-and-cache-job-template.yml +++ b/setup-and-cache-job-template.yml @@ -19,10 +19,15 @@ jobs: inputs: targetType: 'inline' script: mkdir -p $(MAVEN_CACHE_FOLDER); pwd; ls -al $(MAVEN_CACHE_FOLDER) + - ${{ if eq(parameters.signArtifacts, true) }}: + - template: setup-signing-tasks-template.yml - task: Maven@3 inputs: mavenPomFile: 'pom.xml' - options: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -DskipTests' + ${{ if eq(parameters.signArtifacts, true) }}: + options: '--settings $(Agent.TempDirectory)/settings.xml -Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -DskipTests -DdeployToSonatype' + ${{ else }}: + options: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -DskipTests' mavenOptions: '-Xmx3072m' javaHomeOption: 'JDKVersion' jdkVersionOption: '1.11' diff --git a/setup-signing-tasks-template.yml b/setup-signing-tasks-template.yml new file mode 100644 index 000000000..d1cb803f1 --- /dev/null +++ b/setup-signing-tasks-template.yml @@ -0,0 +1,65 @@ +steps: + # We need a valid signing key. + # The next two steps download the public and private keys from the + # secure library files. + - task: DownloadSecureFile@1 + displayName: 'Download public key.' + inputs: + secureFile: public.key + + - task: DownloadSecureFile@1 + displayName: 'Download private key.' + inputs: + secureFile: private.key + + # Import both the private and public keys into gpg for signing. + - bash: | + gpg --import --no-tty --batch --yes $(Agent.TempDirectory)/public.key + gpg --import --no-tty --batch --yes $(Agent.TempDirectory)/private.key + gpg --list-keys --keyid-format LONG + gpg --list-secret-keys --keyid-format LONG + displayName: 'Import signing keys.' + + # For creating the snapshot release with maven, we need to build a fake settings.xml + # for it to read from. This is done for the master branch merges only. + - bash: | + cat >$(Agent.TempDirectory)/settings.xml < + + + github + $(GIT_USER_NAME) + $(GIT_PAT) + + + ossrh + $(SONATYPE_USER) + $(SONATYPE_PASS) + + + $(PGP_KEYNAME) + $(PGP_PASSPHRASE) + + + github-releases + markiantorno + $(GIT_PACKAGE_PAT) + + + + + release + + true + + + $(PGP_KEYNAME) + + + + + EOL + displayName: 'Create settings.xml' \ No newline at end of file