From 8fbd20115143e0f2de66f650dfedb527908ee01f Mon Sep 17 00:00:00 2001 From: Steve Ebersole Date: Wed, 14 Aug 2024 07:54:42 -0500 Subject: [PATCH] HHH-18485 - Documentation symlinks on release --- .../orm/ReleaseFamilyIdentifier.java | 18 +++ .../hibernate/orm/docs/DescriptorAccess.java | 6 +- .../orm/docs/DocumentationPublishing.java | 51 ++++--- .../docs/DocumentationPublishingPlugin.java | 69 +++++++--- .../orm/docs/GenerateDescriptorTask.java | 78 ++++++++--- .../org/hibernate/orm/docs/PublishTask.java | 3 +- .../orm/docs/UpdateSymLinksTask.java | 127 ++++++++++++++++++ 7 files changed, 296 insertions(+), 56 deletions(-) create mode 100644 local-build-plugins/src/main/java/org/hibernate/orm/docs/UpdateSymLinksTask.java diff --git a/local-build-plugins/src/main/java/org/hibernate/orm/ReleaseFamilyIdentifier.java b/local-build-plugins/src/main/java/org/hibernate/orm/ReleaseFamilyIdentifier.java index 9c41b12456..ac7648fc9a 100644 --- a/local-build-plugins/src/main/java/org/hibernate/orm/ReleaseFamilyIdentifier.java +++ b/local-build-plugins/src/main/java/org/hibernate/orm/ReleaseFamilyIdentifier.java @@ -7,6 +7,7 @@ package org.hibernate.orm; import java.io.Serializable; +import java.util.Objects; /** * Major/family version component pair @@ -45,6 +46,23 @@ public class ReleaseFamilyIdentifier implements Comparable 0; } diff --git a/local-build-plugins/src/main/java/org/hibernate/orm/docs/DescriptorAccess.java b/local-build-plugins/src/main/java/org/hibernate/orm/docs/DescriptorAccess.java index 596800e765..e093041ab5 100644 --- a/local-build-plugins/src/main/java/org/hibernate/orm/docs/DescriptorAccess.java +++ b/local-build-plugins/src/main/java/org/hibernate/orm/docs/DescriptorAccess.java @@ -20,7 +20,8 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; /** - * Models the published doc descriptor file + * Helper for {@linkplain #loadProject() loading} the project documentation descriptor and + * {@linkplain #storeProject storing} it to file. * * @author Steve Ebersole */ @@ -46,6 +47,9 @@ public class DescriptorAccess { } } + /** + * Store the descriptor to file + */ public static void storeProject(ProjectDocumentationDescriptor project, File jsonFile) { prepareJsonFile( jsonFile ); diff --git a/local-build-plugins/src/main/java/org/hibernate/orm/docs/DocumentationPublishing.java b/local-build-plugins/src/main/java/org/hibernate/orm/docs/DocumentationPublishing.java index 41dfc6dba5..5edc3cd9c5 100644 --- a/local-build-plugins/src/main/java/org/hibernate/orm/docs/DocumentationPublishing.java +++ b/local-build-plugins/src/main/java/org/hibernate/orm/docs/DocumentationPublishing.java @@ -25,12 +25,21 @@ import org.hibernate.orm.ReleaseFamilyIdentifier; public class DocumentationPublishing { public static final String DSL_NAME = "documentationPublishing"; + public static final String RSYNC_SERVER = "filemgmt-prod-sync.jboss.org"; + public static final String SFTP_SERVER = "filemgmt-prod.jboss.org"; + + public static final String DOC_SERVER_BASE_DIR = "/docs_htdocs/hibernate"; + + public static final String DESCRIPTOR_FILE = "doc-pub/orm.json"; + private final Project project; private final DirectoryProperty stagingDirectory; - private final Property docServerUrl; - private final Property docDescriptorUploadUrl; + private final Property rsyncDocServer; + private final Property sftpDocServer; + private final Property serverBaseDir; + private final RegularFileProperty updatedJsonFile; private final ReleaseFamilyIdentifier releaseFamilyIdentifier; @@ -43,18 +52,22 @@ public class DocumentationPublishing { .directoryProperty() .convention( project.getLayout().getBuildDirectory().dir( "documentation" ) ); - docServerUrl = project.getObjects() - .property( String.class ) - .convention( "filemgmt-prod-sync.jboss.org:/docs_htdocs/hibernate/orm" ); - docDescriptorUploadUrl = project.getObjects() + rsyncDocServer = project.getObjects() .property( String.class ) - .convention( "filemgmt-prod-sync.jboss.org:/docs_htdocs/hibernate/_outdated-content/orm.json" ); + .convention( RSYNC_SERVER ); + sftpDocServer = project.getObjects() + .property( String.class ) + .convention( SFTP_SERVER ); + + serverBaseDir = project.getObjects() + .property( String.class ) + .convention( DOC_SERVER_BASE_DIR ); updatedJsonFile = project.getObjects() .fileProperty() - .convention( project.getLayout().getBuildDirectory().file( "doc-pub/orm.json" ) ); + .convention( project.getLayout().getBuildDirectory().file( DESCRIPTOR_FILE ) ); releaseFamilyIdentifier = ReleaseFamilyIdentifier.parse( project.getVersion().toString() ); } @@ -63,24 +76,22 @@ public class DocumentationPublishing { return releaseFamilyIdentifier; } - public Property getDocServerUrl() { - return docServerUrl; + public Property getRsyncDocServer() { + return rsyncDocServer; + } + + public Property getSftpDocServer() { + return sftpDocServer; + } + + public Property getServerBaseDir() { + return serverBaseDir; } public DirectoryProperty getStagingDirectory() { return stagingDirectory; } - /** - * Where to upload the {@link #getUpdatedJsonFile() documentation descriptor} - */ - public Property getDocDescriptorUploadUrl() { - return docDescriptorUploadUrl; - } - - /** - * THe ORM documentation descriptor - */ public Provider getUpdatedJsonFile() { return updatedJsonFile; } diff --git a/local-build-plugins/src/main/java/org/hibernate/orm/docs/DocumentationPublishingPlugin.java b/local-build-plugins/src/main/java/org/hibernate/orm/docs/DocumentationPublishingPlugin.java index 0318221e1b..3be3cccec2 100644 --- a/local-build-plugins/src/main/java/org/hibernate/orm/docs/DocumentationPublishingPlugin.java +++ b/local-build-plugins/src/main/java/org/hibernate/orm/docs/DocumentationPublishingPlugin.java @@ -6,8 +6,11 @@ */ package org.hibernate.orm.docs; +import java.util.Locale; + import org.gradle.api.Plugin; import org.gradle.api.Project; +import org.gradle.api.provider.Provider; import org.gradle.api.tasks.TaskProvider; import static org.hibernate.orm.docs.DocumentationPublishing.DSL_NAME; @@ -17,10 +20,12 @@ import static org.hibernate.orm.docs.PublishTask.UPLOAD_TASK_NAME; /** * Plugin for helping with publishing documentation to the doc server -
    - *
  • Publishes a config extension ({@link DocumentationPublishing}) under {@value DocumentationPublishing#DSL_NAME}
    • - *
  • Creates a task ({@value PublishTask#UPLOAD_TASK_NAME}) to upload the documentation to the doc server
    • - *
  • Creates a task ({@value GenerateDescriptorTask#GEN_DESC_TASK_NAME}) to create the "published doc descriptor" (JSON) file
    • - *
  • Creates a task ({@value PublishDescriptorTask#UPLOAD_DESC_TASK_NAME}) to upload the "published doc descriptor" (JSON) file to the doc server
    • + *
  • Publishes a {@link DocumentationPublishing DSL extension} under {@value DocumentationPublishing#DSL_NAME}
    • + *
  • Creates a task to upload the documentation to the doc server - {@value PublishTask#UPLOAD_TASK_NAME}
    • + *
  • Creates a task to create the doc descriptor (JSON) file - {@value GenerateDescriptorTask#GEN_DESC_TASK_NAME}
    • + *
  • Creates a task to upload the doc descriptor (JSON) file to the doc server - {@value PublishDescriptorTask#UPLOAD_DESC_TASK_NAME}
    • + *
  • Creates a task to update symlinks on the doc server - {@value UpdateSymLinksTask#SYMLINKS_TASK_NAME}
    • + *
  • Creates a task to upload the migration guide to the doc server - {@value PublishMigrationGuide#NAME}
    • *
* * @author Steve Ebersole @@ -31,6 +36,7 @@ public class DocumentationPublishingPlugin implements Plugin { final DocumentationPublishing docPubDsl = project.getExtensions().create( DSL_NAME, DocumentationPublishing.class ); final boolean isSnapshot = project.getVersion().toString().endsWith( "-SNAPSHOT" ); + final boolean isFinal = project.getVersion().toString().endsWith( ".Final" ); final TaskProvider generateDescriptorTask = project.getTasks().register( GEN_DESC_TASK_NAME, @@ -38,6 +44,8 @@ public class DocumentationPublishingPlugin implements Plugin { (task) -> { task.getCurrentlyBuildingFamily().convention( docPubDsl.getReleaseFamilyIdentifier() ); task.getJsonFile().convention( docPubDsl.getUpdatedJsonFile() ); + + task.onlyIf( (t) -> isFinal ); } ); @@ -45,22 +53,11 @@ public class DocumentationPublishingPlugin implements Plugin { UPLOAD_DESC_TASK_NAME, PublishDescriptorTask.class, (task) -> { - task.getDocDescriptorUploadUrl().convention( docPubDsl.getDocDescriptorUploadUrl() ); + task.getDocDescriptorUploadUrl().convention( defaultDescriptorUploadUrl( docPubDsl ) ); task.getJsonFile().convention( docPubDsl.getUpdatedJsonFile() ); task.dependsOn( generateDescriptorTask ); - task.onlyIf( (t) -> !isSnapshot && generateDescriptorTask.get().getDidWork() ); - } - ); - - //noinspection unused - final TaskProvider publishMigrationGuideTask = project.getTasks().register( - PublishMigrationGuide.NAME, - PublishMigrationGuide.class, - (task) -> { - task.getCurrentlyBuildingFamily().convention( docPubDsl.getReleaseFamilyIdentifier() ); - task.getDocServerUrl().convention( docPubDsl.getDocServerUrl() ); - task.getMigrationGuideDirectory().convention( project.getLayout().getBuildDirectory().dir( "documentation/migration-guide" ) ); + task.onlyIf( (t) -> generateDescriptorTask.get().getDidWork() && generateDescriptorTask.get().needsUpload() ); } ); @@ -70,12 +67,48 @@ public class DocumentationPublishingPlugin implements Plugin { PublishTask.class, (task) -> { task.getBuildingFamily().convention( docPubDsl.getReleaseFamilyIdentifier() ); - task.getDocServerUrl().convention( docPubDsl.getDocServerUrl() ); task.getStagingDirectory().convention( docPubDsl.getStagingDirectory() ); + task.getDocServerUrl().convention( defaultDocUploadUrl( docPubDsl ) ); task.dependsOn( uploadDescriptorTask ); task.onlyIf( (t) -> !isSnapshot ); } ); + + //noinspection unused + final TaskProvider symLinkTask = project.getTasks().register( + UpdateSymLinksTask.SYMLINKS_TASK_NAME, + UpdateSymLinksTask.class, + (task) -> { + task.getBuildingFamily().convention( docPubDsl.getReleaseFamilyIdentifier() ); + task.getSftpDocServer().convention( docPubDsl.getSftpDocServer() ); + task.getServerBaseDir().convention( docPubDsl.getServerBaseDir() ); + + task.dependsOn( generateDescriptorTask ); + task.dependsOn( uploadTask ); + task.onlyIf( (t) -> generateDescriptorTask.get().getDidWork() && generateDescriptorTask.get().needsSymLinkUpdate() ); + } + ); + + //noinspection unused + final TaskProvider publishMigrationGuideTask = project.getTasks().register( + PublishMigrationGuide.NAME, + PublishMigrationGuide.class, + (task) -> { + task.getCurrentlyBuildingFamily().convention( docPubDsl.getReleaseFamilyIdentifier() ); + task.getDocServerUrl().convention( defaultDocUploadUrl( docPubDsl ) ); + task.getMigrationGuideDirectory().convention( project.getLayout().getBuildDirectory().dir( "documentation/migration-guide" ) ); + } + ); + } + + private Provider defaultDescriptorUploadUrl(DocumentationPublishing dsl) { + return dsl.getRsyncDocServer() + .map( (server) -> String.format( Locale.ROOT, "%s:%s/_outdated-content/orm.json", server, dsl.getServerBaseDir().get() ) ); + } + + private Provider defaultDocUploadUrl(DocumentationPublishing dsl) { + return dsl.getRsyncDocServer() + .map( (server) -> String.format( Locale.ROOT, "%s:%s/orm", server, dsl.getServerBaseDir().get() ) ); } } diff --git a/local-build-plugins/src/main/java/org/hibernate/orm/docs/GenerateDescriptorTask.java b/local-build-plugins/src/main/java/org/hibernate/orm/docs/GenerateDescriptorTask.java index d2a52c7a6b..155e5eeafa 100644 --- a/local-build-plugins/src/main/java/org/hibernate/orm/docs/GenerateDescriptorTask.java +++ b/local-build-plugins/src/main/java/org/hibernate/orm/docs/GenerateDescriptorTask.java @@ -7,6 +7,9 @@ package org.hibernate.orm.docs; import java.io.File; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; import org.gradle.api.DefaultTask; import org.gradle.api.file.RegularFileProperty; @@ -18,19 +21,25 @@ import org.gradle.api.tasks.TaskAction; import org.hibernate.orm.ReleaseFamilyIdentifier; /** + * Task for creating the JSON "documentation descriptor" for ORM + * * @author Steve Ebersole */ public abstract class GenerateDescriptorTask extends DefaultTask { public static final String GEN_DESC_TASK_NAME = "generateDocumentationDescriptor"; - private final RegularFileProperty jsonFile; + private final Property currentlyBuildingFamily; + private final RegularFileProperty jsonFile; + + private boolean needsUpload; + private boolean needsSymLinkUpdate; public GenerateDescriptorTask() { setGroup( "documentation" ); setDescription( "Generates the documentation publication descriptor (JSON)" ); - jsonFile = getProject().getObjects().fileProperty(); currentlyBuildingFamily = getProject().getObjects().property( ReleaseFamilyIdentifier.class ); + jsonFile = getProject().getObjects().fileProperty(); } @Input @@ -43,14 +52,47 @@ public abstract class GenerateDescriptorTask extends DefaultTask { return jsonFile; } + /** + * Whether we determined, during {@linkplain #generateDescriptor}, that uploading the + * doc descriptor was needed. + * + * @see PublishDescriptorTask + */ + public boolean needsUpload() { + return getDidWork() && needsUpload; + } + + /** + * Whether we determined, during {@linkplain #generateDescriptor}, that updating the + * doc server symlinks was needed. + * + * @see UpdateSymLinksTask + */ + public boolean needsSymLinkUpdate() { + return getDidWork() && needsSymLinkUpdate; + } + @TaskAction public void generateDescriptor() { final ProjectDocumentationDescriptor descriptor = DescriptorAccess.loadProject(); + final boolean isFinal = getProject().getVersion().toString().endsWith( ".Final" ); + final Set processedReleases = new HashSet<>(); ReleaseFamilyIdentifier newest = null; boolean foundCurrentRelease = false; - for ( ReleaseFamilyDocumentation releaseFamily : descriptor.getReleaseFamilies() ) { + final Iterator itr = descriptor.getReleaseFamilies().iterator(); + while ( itr.hasNext() ) { + final ReleaseFamilyDocumentation releaseFamily = itr.next(); + + // NOTE: sometimes releases get duplicated in the descriptor... + // let's clean those up if we run across them + if ( !processedReleases.add( releaseFamily.getName() ) ) { + itr.remove(); + needsUpload = true; + continue; + } + if ( newest == null || releaseFamily.getName().newerThan( newest ) ) { newest = releaseFamily.getName(); @@ -61,27 +103,31 @@ public abstract class GenerateDescriptorTask extends DefaultTask { } } - if ( ! foundCurrentRelease ) { - final ReleaseFamilyDocumentation newEntry = new ReleaseFamilyDocumentation(); - newEntry.setName( currentlyBuildingFamily.get() ); - descriptor.addReleaseFamily( newEntry ); - setDidWork( true ); - } + if ( isFinal ) { + // we are releasing a Final - possibly do some other things - // we only want to update "stable" to `currentlyBuildingFamily` when- - // 1. we are currently building a Final - // 2. currentlyBuildingFamily is the newest + if ( !foundCurrentRelease ) { + // this release is not yet tracked in the descriptor - add it + final ReleaseFamilyDocumentation newEntry = new ReleaseFamilyDocumentation(); + newEntry.setName( currentlyBuildingFamily.get() ); + descriptor.addReleaseFamily( newEntry ); + setDidWork( true ); + needsUpload = true; + } - if ( currentlyBuildingFamily.get().newerThan( newest ) ) { - descriptor.setStableFamily( currentlyBuildingFamily.get() ); - setDidWork( true ); + if ( currentlyBuildingFamily.get().newerThan( newest ) ) { + // this release is newer than any currently tracked in the descriptor + descriptor.setStableFamily( currentlyBuildingFamily.get() ); + setDidWork( true ); + needsSymLinkUpdate = true; + } } DescriptorAccess.storeProject( descriptor, jsonFile.get().getAsFile() ); } public static void main(String... args) { - final File jsonFile = new File( "/home/sebersole/projects/hibernate-orm/6.0/hibernate-orm-build/target/doc-pub/orm.json" ); + final File jsonFile = new File( "/tmp/hibernate-orm-build/doc-pub/orm.json" ); final ProjectDocumentationDescriptor projectDoc = DescriptorAccess.loadProject(); DescriptorAccess.storeProject( projectDoc, jsonFile ); } diff --git a/local-build-plugins/src/main/java/org/hibernate/orm/docs/PublishTask.java b/local-build-plugins/src/main/java/org/hibernate/orm/docs/PublishTask.java index 77e48c1547..cad231547d 100644 --- a/local-build-plugins/src/main/java/org/hibernate/orm/docs/PublishTask.java +++ b/local-build-plugins/src/main/java/org/hibernate/orm/docs/PublishTask.java @@ -64,8 +64,9 @@ public abstract class PublishTask extends DefaultTask { getProject().getLogger().lifecycle( "Uploading documentation `{}` -> `{}`", stagingDirPath, url ); final ExecResult result = getProject().exec( (exec) -> { exec.executable( "rsync" ); - exec.args("--port=2222", "-avz", "--links", "--delete", stagingDirPathContent, url ); + exec.args("-avz", "--delete", stagingDirPathContent, url ); } ); getProject().getLogger().lifecycle( "Done uploading documentation - {}", result.getExitValue() == 0 ? "success" : "failure" ); + setDidWork( true ); } } diff --git a/local-build-plugins/src/main/java/org/hibernate/orm/docs/UpdateSymLinksTask.java b/local-build-plugins/src/main/java/org/hibernate/orm/docs/UpdateSymLinksTask.java new file mode 100644 index 0000000000..d8a8e34b1d --- /dev/null +++ b/local-build-plugins/src/main/java/org/hibernate/orm/docs/UpdateSymLinksTask.java @@ -0,0 +1,127 @@ +/* + * Hibernate, Relational Persistence for Idiomatic Java + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later. + * See the lgpl.txt file in the root directory or http://www.gnu.org/licenses/lgpl-2.1.html. + */ +package org.hibernate.orm.docs; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Locale; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import org.gradle.api.DefaultTask; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.TaskAction; + +import org.hibernate.orm.ReleaseFamilyIdentifier; + +import static org.hibernate.orm.docs.DocumentationPublishing.DOC_SERVER_BASE_DIR; +import static org.hibernate.orm.docs.DocumentationPublishing.SFTP_SERVER; + +/** + * Updates the "current" and "stable" symlinks on the doc server + * + * @author Steve Ebersole + */ +public class UpdateSymLinksTask extends DefaultTask { + public static final String SYMLINKS_TASK_NAME = "updateDocSymLinks"; + + private final Property sftpDocServer; + private final Property serverBaseDir; + private final Property buildingFamily; + + public UpdateSymLinksTask() { + setGroup( "documentation" ); + setDescription( "Updates the 'current' and 'stable' symlinks on the documentation server" ); + + buildingFamily = getProject().getObjects().property( ReleaseFamilyIdentifier.class ); + sftpDocServer = getProject().getObjects().property( String.class ); + serverBaseDir = getProject().getObjects().property( String.class ); + } + + public Property getBuildingFamily() { + return buildingFamily; + } + + public Property getSftpDocServer() { + return sftpDocServer; + } + + public Property getServerBaseDir() { + return serverBaseDir; + } + + @TaskAction + public void updateSymLinks() throws Exception { + updateSymLinks( buildingFamily.get().toExternalForm(), sftpDocServer.get(), serverBaseDir.get() ); + } + + private static void updateSymLinks(String releaseName, String sftpServer, String serverBaseDir) throws Exception { + final File commandFile = createCommandFile( releaseName, serverBaseDir ); + System.out.println( "SFTP command file : " + commandFile.getAbsolutePath() ); + + final Process sftpProcess = new ProcessBuilder() + .command( "sh", "sftp", "-b", commandFile.getAbsolutePath(), sftpServer ) + .redirectInput( ProcessBuilder.Redirect.INHERIT ) + .start(); + + ExecutorService service = Executors.newFixedThreadPool( 2 ); + try ( InputStream is = sftpProcess.getInputStream(); InputStream es = sftpProcess.getErrorStream(); + Closeable pool = service::shutdownNow ) { + service.submit( () -> drain( is, System.out::println ) ); + service.submit( () -> drain( es, System.err::println ) ); + service.shutdown(); + + final boolean isFinished = sftpProcess.waitFor( 15, TimeUnit.SECONDS ); + if ( !isFinished ) { + System.out.println( "Forcibly ending sftp" ); + sftpProcess.destroyForcibly(); + } + } + } + + private static void drain(InputStream stream, Consumer consumer) { + try (InputStreamReader in = new InputStreamReader( stream, StandardCharsets.UTF_8 ); + BufferedReader bufferedReader = new BufferedReader( in ); ) { + bufferedReader.lines().forEach( consumer ); + } + catch (IOException e) { + throw new RuntimeException( e ); + } + } + + + private static File createCommandFile(String releaseName, String serverBaseDir) throws IOException { + final File commandFile = File.createTempFile( "hibernate-orm-release-doc-symlink-" + releaseName, "-cmd.txt" ); + + try (FileWriter commandFileWriter = new FileWriter( commandFile )) { + commandFileWriter.write( "cd " + serverBaseDir + "/stable\n" ); + commandFileWriter.write( "rm orm\n" ); + commandFileWriter.write( String.format( Locale.ROOT, "ln -s ../orm/%s orm\n", releaseName ) ); + + commandFileWriter.write( "cd " + serverBaseDir + "/orm\n" ); + commandFileWriter.write( "rm current\n" ); + commandFileWriter.write( String.format( Locale.ROOT, "ln -s %s current\n", releaseName ) ); + + commandFileWriter.flush(); + } + + return commandFile; + } + + public static void main(String[] args) throws Exception { + System.out.println( "Starting UpdateSymLinksTask" ); + updateSymLinks( "6.6", SFTP_SERVER, DOC_SERVER_BASE_DIR ); + } +}