From e1c6ef2efa9d87fdfc7474ca63998a13a3929874 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Tue, 8 Nov 2016 16:33:55 -0800 Subject: [PATCH] HADOOP-13789. Hadoop Common includes generated test protos in both jar and test-jar. Contributed by Sean Busbey. --- hadoop-common-project/hadoop-common/pom.xml | 6 +- .../hadoop-hdfs-client/pom.xml | 2 - hadoop-hdfs-project/hadoop-hdfs/pom.xml | 2 - .../hadoop-mapreduce-client-common/pom.xml | 2 - .../hadoop-mapreduce-client-shuffle/pom.xml | 2 - .../maven/plugin/protoc/ProtocMojo.java | 228 +------------- .../maven/plugin/protoc/ProtocRunner.java | 283 ++++++++++++++++++ .../maven/plugin/protoc/ProtocTestMojo.java | 61 ++++ .../maven/plugin/protoc/package-info.java | 98 ++++++ .../hadoop-yarn/hadoop-yarn-api/pom.xml | 2 - .../hadoop-yarn/hadoop-yarn-client/pom.xml | 2 - .../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 - .../pom.xml | 2 - .../hadoop-yarn-server-common/pom.xml | 2 - .../hadoop-yarn-server-nodemanager/pom.xml | 2 - .../pom.xml | 8 +- .../hadoop-yarn-server-tests/pom.xml | 6 +- 17 files changed, 454 insertions(+), 256 deletions(-) create mode 100644 hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java create mode 100644 hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocTestMojo.java create mode 100644 hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/package-info.java diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 645d4951a05..37f0b70283a 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -369,7 +369,6 @@ compile-protoc - generate-sources protoc @@ -397,14 +396,12 @@ GenericRefreshProtocol.proto - ${project.build.directory}/generated-sources/java compile-test-protoc - generate-test-sources - protoc + test-protoc ${protobuf.version} @@ -419,7 +416,6 @@ test_rpc_service.proto - ${project.build.directory}/generated-test-sources/java diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index 193f423b783..8ce5f3f230d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -138,7 +138,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> compile-protoc - generate-sources protoc @@ -164,7 +163,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> ReconfigurationProtocol.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index b3ad50a1805..5427db8929f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -325,7 +325,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> compile-protoc - generate-sources protoc @@ -352,7 +351,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> fsimage.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml index fb3ceb606df..f78e83bc079 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml @@ -61,7 +61,6 @@ compile-protoc - generate-sources protoc @@ -82,7 +81,6 @@ MRClientProtocol.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml index 856d074ba60..41a5220f8f0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml @@ -59,7 +59,6 @@ compile-protoc - generate-sources protoc @@ -76,7 +75,6 @@ ShuffleHandlerRecovery.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java index df479fd4872..2d0fc623417 100644 --- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java @@ -13,8 +13,6 @@ */ package org.apache.hadoop.maven.plugin.protoc; -import org.apache.hadoop.maven.plugin.util.Exec; -import org.apache.hadoop.maven.plugin.util.FileSetUtils; import org.apache.maven.model.FileSet; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; @@ -22,22 +20,13 @@ import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; import org.apache.maven.project.MavenProject; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.zip.CRC32; +/** + * Mojo to generate java classes from .proto files using protoc. + * See package info for examples of use in a maven pom. + */ @Mojo(name="protoc", defaultPhase = LifecyclePhase.GENERATE_SOURCES) public class ProtocMojo extends AbstractMojo { @@ -63,213 +52,10 @@ public class ProtocMojo extends AbstractMojo { "${project.build.directory}/hadoop-maven-plugins-protoc-checksums.json") private String checksumPath; - /** - * Compares include and source file checksums against previously computed - * checksums stored in a json file in the build directory. - */ - public class ChecksumComparator { - - private final Map storedChecksums; - private final Map computedChecksums; - - private final File checksumFile; - - ChecksumComparator(String checksumPath) throws IOException { - checksumFile = new File(checksumPath); - // Read in the checksums - if (checksumFile.exists()) { - ObjectMapper mapper = new ObjectMapper(); - storedChecksums = mapper - .readValue(checksumFile, new TypeReference>() { - }); - } else { - storedChecksums = new HashMap<>(0); - } - computedChecksums = new HashMap<>(); - } - - public boolean hasChanged(File file) throws IOException { - if (!file.exists()) { - throw new FileNotFoundException( - "Specified protoc include or source does not exist: " + file); - } - if (file.isDirectory()) { - return hasDirectoryChanged(file); - } else if (file.isFile()) { - return hasFileChanged(file); - } else { - throw new IOException("Not a file or directory: " + file); - } - } - - private boolean hasDirectoryChanged(File directory) throws IOException { - File[] listing = directory.listFiles(); - boolean changed = false; - if (listing == null) { - // not changed. - return false; - } - // Do not exit early, since we need to compute and save checksums - // for each file within the directory. - for (File f : listing) { - if (f.isDirectory()) { - if (hasDirectoryChanged(f)) { - changed = true; - } - } else if (f.isFile()) { - if (hasFileChanged(f)) { - changed = true; - } - } else { - getLog().debug("Skipping entry that is not a file or directory: " - + f); - } - } - return changed; - } - - private boolean hasFileChanged(File file) throws IOException { - long computedCsum = computeChecksum(file); - - // Return if the generated csum matches the stored csum - Long storedCsum = storedChecksums.get(file.getCanonicalPath()); - if (storedCsum == null || storedCsum.longValue() != computedCsum) { - // It has changed. - return true; - } - return false; - } - - private long computeChecksum(File file) throws IOException { - // If we've already computed the csum, reuse the computed value - final String canonicalPath = file.getCanonicalPath(); - if (computedChecksums.containsKey(canonicalPath)) { - return computedChecksums.get(canonicalPath); - } - // Compute the csum for the file - CRC32 crc = new CRC32(); - byte[] buffer = new byte[1024*64]; - try (BufferedInputStream in = - new BufferedInputStream(new FileInputStream(file))) { - while (true) { - int read = in.read(buffer); - if (read <= 0) { - break; - } - crc.update(buffer, 0, read); - } - } - // Save it in the generated map and return - final long computedCsum = crc.getValue(); - computedChecksums.put(canonicalPath, computedCsum); - return crc.getValue(); - } - - public void writeChecksums() throws IOException { - ObjectMapper mapper = new ObjectMapper(); - try (BufferedOutputStream out = new BufferedOutputStream( - new FileOutputStream(checksumFile))) { - mapper.writeValue(out, computedChecksums); - getLog().info("Wrote protoc checksums to file " + checksumFile); - } - } - } - public void execute() throws MojoExecutionException { - try { - List command = new ArrayList(); - command.add(protocCommand); - command.add("--version"); - Exec exec = new Exec(this); - List out = new ArrayList(); - if (exec.run(command, out) == 127) { - getLog().error("protoc, not found at: " + protocCommand); - throw new MojoExecutionException("protoc failure"); - } else { - if (out.isEmpty()) { - getLog().error("stdout: " + out); - throw new MojoExecutionException( - "'protoc --version' did not return a version"); - } else { - if (!out.get(0).endsWith(protocVersion)) { - throw new MojoExecutionException( - "protoc version is '" + out.get(0) + "', expected version is '" - + protocVersion + "'"); - } - } - } - if (!output.mkdirs()) { - if (!output.exists()) { - throw new MojoExecutionException( - "Could not create directory: " + output); - } - } - - // Whether the import or source protoc files have changed. - ChecksumComparator comparator = new ChecksumComparator(checksumPath); - boolean importsChanged = false; - - command = new ArrayList(); - command.add(protocCommand); - command.add("--java_out=" + output.getCanonicalPath()); - if (imports != null) { - for (File i : imports) { - if (comparator.hasChanged(i)) { - importsChanged = true; - } - command.add("-I" + i.getCanonicalPath()); - } - } - // Filter to generate classes for just the changed source files. - List changedSources = new ArrayList<>(); - boolean sourcesChanged = false; - for (File f : FileSetUtils.convertFileSetToFiles(source)) { - // Need to recompile if the source has changed, or if any import has - // changed. - if (comparator.hasChanged(f) || importsChanged) { - sourcesChanged = true; - changedSources.add(f); - command.add(f.getCanonicalPath()); - } - } - - if (!sourcesChanged && !importsChanged) { - getLog().info("No changes detected in protoc files, skipping " - + "generation."); - } else { - if (getLog().isDebugEnabled()) { - StringBuilder b = new StringBuilder(); - b.append("Generating classes for the following protoc files: ["); - String prefix = ""; - for (File f : changedSources) { - b.append(prefix); - b.append(f.toString()); - prefix = ", "; - } - b.append("]"); - getLog().debug(b.toString()); - } - - exec = new Exec(this); - out = new ArrayList(); - List err = new ArrayList<>(); - if (exec.run(command, out, err) != 0) { - getLog().error("protoc compiler error"); - for (String s : out) { - getLog().error(s); - } - for (String s : err) { - getLog().error(s); - } - throw new MojoExecutionException("protoc failure"); - } - // Write the new checksum file on success. - comparator.writeChecksums(); - } - } catch (Throwable ex) { - throw new MojoExecutionException(ex.toString(), ex); - } - project.addCompileSourceRoot(output.getAbsolutePath()); + final ProtocRunner protoc = new ProtocRunner(project, imports, output, + source, protocCommand, protocVersion, checksumPath, this, false); + protoc.execute(); } } diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java new file mode 100644 index 00000000000..ef1588459b8 --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java @@ -0,0 +1,283 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.maven.plugin.protoc; + +import org.apache.hadoop.maven.plugin.util.Exec; +import org.apache.hadoop.maven.plugin.util.FileSetUtils; +import org.apache.maven.model.FileSet; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.project.MavenProject; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.TypeReference; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.CRC32; + +/** + * Common execution for both the main and test protoc mojos. + */ +public class ProtocRunner { + + private final MavenProject project; + private final File[] imports; + private final File output; + private final FileSet source; + private final String protocCommand; + private final String protocVersion; + private final String checksumPath; + private final boolean test; + private final AbstractMojo mojo; + + @SuppressWarnings("checkstyle:parameternumber") + public ProtocRunner(final MavenProject project, final File[] imports, + final File output, final FileSet source, final String protocCommand, + final String protocVersion, final String checksumPath, + final AbstractMojo mojo, final boolean test) { + this.project = project; + this.imports = Arrays.copyOf(imports, imports.length); + this.output = output; + this.source = source; + this.protocCommand = protocCommand; + this.protocVersion = protocVersion; + this.checksumPath = checksumPath; + this.mojo = mojo; + this.test = test; + } + + /** + * Compares include and source file checksums against previously computed + * checksums stored in a json file in the build directory. + */ + public class ChecksumComparator { + + private final Map storedChecksums; + private final Map computedChecksums; + + private final File checksumFile; + + ChecksumComparator(String checksumPath) throws IOException { + checksumFile = new File(checksumPath); + // Read in the checksums + if (checksumFile.exists()) { + ObjectMapper mapper = new ObjectMapper(); + storedChecksums = mapper + .readValue(checksumFile, new TypeReference>() { + }); + } else { + storedChecksums = new HashMap<>(0); + } + computedChecksums = new HashMap<>(); + } + + public boolean hasChanged(File file) throws IOException { + if (!file.exists()) { + throw new FileNotFoundException( + "Specified protoc include or source does not exist: " + file); + } + if (file.isDirectory()) { + return hasDirectoryChanged(file); + } else if (file.isFile()) { + return hasFileChanged(file); + } else { + throw new IOException("Not a file or directory: " + file); + } + } + + private boolean hasDirectoryChanged(File directory) throws IOException { + File[] listing = directory.listFiles(); + boolean changed = false; + if (listing == null) { + // not changed. + return false; + } + // Do not exit early, since we need to compute and save checksums + // for each file within the directory. + for (File f : listing) { + if (f.isDirectory()) { + if (hasDirectoryChanged(f)) { + changed = true; + } + } else if (f.isFile()) { + if (hasFileChanged(f)) { + changed = true; + } + } else { + mojo.getLog().debug("Skipping entry that is not a file or directory: " + + f); + } + } + return changed; + } + + private boolean hasFileChanged(File file) throws IOException { + long computedCsum = computeChecksum(file); + + // Return if the generated csum matches the stored csum + Long storedCsum = storedChecksums.get(file.getCanonicalPath()); + if (storedCsum == null || storedCsum.longValue() != computedCsum) { + // It has changed. + return true; + } + return false; + } + + private long computeChecksum(File file) throws IOException { + // If we've already computed the csum, reuse the computed value + final String canonicalPath = file.getCanonicalPath(); + if (computedChecksums.containsKey(canonicalPath)) { + return computedChecksums.get(canonicalPath); + } + // Compute the csum for the file + CRC32 crc = new CRC32(); + byte[] buffer = new byte[1024*64]; + try (BufferedInputStream in = + new BufferedInputStream(new FileInputStream(file))) { + while (true) { + int read = in.read(buffer); + if (read <= 0) { + break; + } + crc.update(buffer, 0, read); + } + } + // Save it in the generated map and return + final long computedCsum = crc.getValue(); + computedChecksums.put(canonicalPath, computedCsum); + return crc.getValue(); + } + + public void writeChecksums() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + try (BufferedOutputStream out = new BufferedOutputStream( + new FileOutputStream(checksumFile))) { + mapper.writeValue(out, computedChecksums); + mojo.getLog().info("Wrote protoc checksums to file " + checksumFile); + } + } + } + + public void execute() throws MojoExecutionException { + try { + List command = new ArrayList(); + command.add(protocCommand); + command.add("--version"); + Exec exec = new Exec(mojo); + List out = new ArrayList(); + if (exec.run(command, out) == 127) { + mojo.getLog().error("protoc, not found at: " + protocCommand); + throw new MojoExecutionException("protoc failure"); + } else { + if (out.isEmpty()) { + mojo.getLog().error("stdout: " + out); + throw new MojoExecutionException( + "'protoc --version' did not return a version"); + } else { + if (!out.get(0).endsWith(protocVersion)) { + throw new MojoExecutionException( + "protoc version is '" + out.get(0) + "', expected version is '" + + protocVersion + "'"); + } + } + } + if (!output.mkdirs()) { + if (!output.exists()) { + throw new MojoExecutionException( + "Could not create directory: " + output); + } + } + + // Whether the import or source protoc files have changed. + ChecksumComparator comparator = new ChecksumComparator(checksumPath); + boolean importsChanged = false; + + command = new ArrayList(); + command.add(protocCommand); + command.add("--java_out=" + output.getCanonicalPath()); + if (imports != null) { + for (File i : imports) { + if (comparator.hasChanged(i)) { + importsChanged = true; + } + command.add("-I" + i.getCanonicalPath()); + } + } + // Filter to generate classes for just the changed source files. + List changedSources = new ArrayList<>(); + boolean sourcesChanged = false; + for (File f : FileSetUtils.convertFileSetToFiles(source)) { + // Need to recompile if the source has changed, or if any import has + // changed. + if (comparator.hasChanged(f) || importsChanged) { + sourcesChanged = true; + changedSources.add(f); + command.add(f.getCanonicalPath()); + } + } + + if (!sourcesChanged && !importsChanged) { + mojo.getLog().info("No changes detected in protoc files, skipping " + + "generation."); + } else { + if (mojo.getLog().isDebugEnabled()) { + StringBuilder b = new StringBuilder(); + b.append("Generating classes for the following protoc files: ["); + String prefix = ""; + for (File f : changedSources) { + b.append(prefix); + b.append(f.toString()); + prefix = ", "; + } + b.append("]"); + mojo.getLog().debug(b.toString()); + } + + exec = new Exec(mojo); + out = new ArrayList(); + List err = new ArrayList<>(); + if (exec.run(command, out, err) != 0) { + mojo.getLog().error("protoc compiler error"); + for (String s : out) { + mojo.getLog().error(s); + } + for (String s : err) { + mojo.getLog().error(s); + } + throw new MojoExecutionException("protoc failure"); + } + // Write the new checksum file on success. + comparator.writeChecksums(); + } + } catch (Throwable ex) { + throw new MojoExecutionException(ex.toString(), ex); + } + if(test) { + project.addTestCompileSourceRoot(output.getAbsolutePath()); + } else { + project.addCompileSourceRoot(output.getAbsolutePath()); + } + } + +} diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocTestMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocTestMojo.java new file mode 100644 index 00000000000..c50f6cabed6 --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocTestMojo.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.maven.plugin.protoc; + +import org.apache.maven.model.FileSet; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; + +import java.io.File; + +/** + * Mojo to generate java test classes from .proto files using protoc. + * See package info for examples of use in a maven pom. + */ +@Mojo(name="test-protoc", defaultPhase = LifecyclePhase.GENERATE_TEST_SOURCES) +public class ProtocTestMojo extends AbstractMojo { + + @Parameter(defaultValue="${project}", readonly=true) + private MavenProject project; + + @Parameter + private File[] imports; + + @Parameter(defaultValue= + "${project.build.directory}/generated-test-sources/java") + private File output; + + @Parameter(required=true) + private FileSet source; + + @Parameter(defaultValue="protoc") + private String protocCommand; + + @Parameter(required=true) + private String protocVersion; + + @Parameter(defaultValue = + "${project.build.directory}/hadoop-maven-plugins-protoc-checksums.json") + private String checksumPath; + + public void execute() throws MojoExecutionException { + final ProtocRunner protoc = new ProtocRunner(project, imports, output, + source, protocCommand, protocVersion, checksumPath, this, true); + protoc.execute(); + } +} diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/package-info.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/package-info.java new file mode 100644 index 00000000000..c4c30ce7d2b --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/package-info.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + *

Plugins to generate Java files based on protocol buffers with the protoc + * command. + * + *

For generated files intended for primary build artifacts use like: + * 

+ *  <plugins>
+ *    ... SNIP ...
+ *    <plugin>
+ *      <groupId>org.apache.hadoop</groupId>
+ *      <artifactId>hadoop-maven-plugins</artifactId>
+ *      <executions>
+ *        ... SNIP ...
+ *        <execution>
+ *          <id>compile-protoc</id>
+ *          <goals>
+ *            <goal>protoc</goal>
+ *          </goals>
+ *          <configuration>
+ *            <protocVersion>${protobuf.version}</protocVersion>
+ *            <protocCommand>${protoc.path}</protocCommand>
+ *            <imports>
+ *              <param>${basedir}/src/main/proto</param>
+ *            </imports>
+ *            <source>
+ *              <directory>${basedir}/src/main/proto</directory>
+ *              <includes>
+ *                <include>HAServiceProtocol.proto</include>
+ *                ... SNIP ...
+ *                <include>RefreshCallQueueProtocol.proto</include>
+ *                <include>GenericRefreshProtocol.proto</include>
+ *              </includes>
+ *            </source>
+ *          </configuration>
+ *        </execution>
+ *        ... SNIP ...
+ *      </executions>
+ *      ... SNIP ...
+ *    </plugin>
+ *  </plugins>
+ *
+ * + * For generated files intended only for test, use like: + * 
+ *  <plugins>
+ *    ... SNIP ...
+ *    <plugin>
+ *      <groupId>org.apache.hadoop</groupId>
+ *      <artifactId>hadoop-maven-plugins</artifactId>
+ *      <executions>
+ *        ... SNIP ...
+ *        <execution>
+ *          <id>compile-test-protoc</id>
+ *          <goals>
+ *            <goal>test-protoc</goal>
+ *          </goals>
+ *          <configuration>
+ *            <protocVersion>${protobuf.version}</protocVersion>
+ *            <protocCommand>${protoc.path}</protocCommand>
+ *            <imports>
+ *              <param>${basedir}/src/test/proto</param>
+ *            </imports>
+ *            <source>
+ *              <directory>${basedir}/src/test/proto</directory>
+ *              <includes>
+ *                <include>test.proto</include>
+ *                <include>test_rpc_service.proto</include>
+ *              </includes>
+ *            </source>
+ *          </configuration>
+ *        </execution>
+ *        ... SNIP ...
+ *      </executions>
+ *      ... SNIP ...
+ *    </plugin>
+ *  </plugins>
+ *
+ * + */ +package org.apache.hadoop.maven.plugin.protoc; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index 52e276b4aa8..71014e4970f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -100,7 +100,6 @@ compile-protoc - generate-sources protoc @@ -128,7 +127,6 @@ server/SCM_Admin_protocol.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 668c50d9df5..a53ec57a358 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -154,7 +154,6 @@ compile-protoc - generate-sources protoc @@ -172,7 +171,6 @@ test_amrm_token.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index b2d3f32d5fe..f55c48831aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -262,7 +262,6 @@ compile-protoc - generate-sources protoc @@ -280,7 +279,6 @@ yarn_security_token.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index 1474c19f1d3..15f6511415d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -195,7 +195,6 @@ compile-protoc - generate-sources protoc @@ -215,7 +214,6 @@ yarn_server_timelineserver_recovery.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index b9e10ee9950..ee892d1a478 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -129,7 +129,6 @@ compile-protoc - generate-sources protoc @@ -153,7 +152,6 @@ collectornodemanager_protocol.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 6fbe39e68bb..acc1d50e638 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -291,7 +291,6 @@ compile-protoc - generate-sources protoc @@ -311,7 +310,6 @@ LocalizationProtocol.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 8e83a15cb2f..0fcd535f99f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -287,7 +287,6 @@ compile-protoc - generate-sources protoc @@ -307,14 +306,12 @@ yarn_server_resourcemanager_recovery.proto - ${project.build.directory}/generated-sources/java compile-test-protoc - generate-sources - protoc + test-protoc ${protobuf.version} @@ -327,10 +324,9 @@ ${basedir}/src/test/proto - test_client_tokens.proto + test_client_tokens.proto - ${project.build.directory}/generated-sources/java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml index 58bed0bc266..359c706598a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml @@ -135,10 +135,9 @@ hadoop-maven-plugins - compile-protoc - generate-sources + compile-test-protoc - protoc + test-protoc ${protobuf.version} @@ -154,7 +153,6 @@ test_token.proto - ${project.build.directory}/generated-sources/java