HADOOP-18487. protobuf 2.5 marked as provided.

The option protobuf.scope defines whether the protobuf 2.5.0
dependency is marked as provided or not.

* all declarations except those in yarn-csi are updated
* those modules which don't compile without their own explicit
  import (hadoop-hdfs-client, hadoop-hdfs-rbf)

It's actually interesting to see where/how that compile fails

hadoop-hdfs-client: ClientNamenodeProtocolTranslatorPB
hadoop-hdfs-rbf:RouterAdminProtocolTranslatorPB

both with "class file for com.google.protobuf.ServiceException not found",
even though *neither class uses it*

what they do have is references to ProtobufHelper.getRemoteException(),
which is overloaded to both the shaded ServiceException and the original one

Hypothesis: the javac overload resolution needs to look at the entire
class hierarchy before it can decide which one to use.

Proposed: add a new method
 ioe extractException(org.apache.hadoop.thirdparty.protobuf.ServiceException)
and move our own code to it. Without the overloading the classes should not
be needed

Change-Id: I70354abfe3f1fdc03c418dac88e60f8cc4929a33

hadoop-common-project/hadoop-common/pom.xml

@@ -254,7 +254,7 @@
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
-      <scope>compile</scope>
+      <scope>${protobuf.scope}</scope>
     </dependency>
     <dependency>
       <groupId>com.google.code.gson</groupId>

hadoop-hdfs-project/hadoop-hdfs-client/pom.xml

@@ -71,6 +71,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <scope>${protobuf.scope}</scope>
+    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml

@@ -146,7 +146,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
-      <scope>compile</scope>
+      <scope>${protobuf.scope}</scope>
     </dependency>
     <dependency>
       <groupId>javax.servlet</groupId>

hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml

@@ -76,6 +76,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>hadoop-federation-balance</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <scope>${protobuf.scope}</scope>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-log4j12</artifactId>

hadoop-hdfs-project/hadoop-hdfs/pom.xml

@@ -130,7 +130,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
-      <scope>compile</scope>
+      <scope>${protobuf.scope}</scope>
     </dependency>
     <dependency>
       <groupId>javax.servlet</groupId>

hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml

@@ -39,6 +39,7 @@
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
+      <scope>${protobuf.scope}</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>

hadoop-project/pom.xml

@@ -86,6 +86,8 @@
 
     <!--Protobuf version for backward compatibility-->
     <protobuf.version>2.5.0</protobuf.version>
+    <protobuf.scope>provided</protobuf.scope>
+
     <!-- ProtocolBuffer version, actually used in Hadoop -->
     <hadoop.protobuf.version>3.7.1</hadoop.protobuf.version>
     <protoc.path>${env.HADOOP_PROTOC_PATH}</protoc.path>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml

@@ -191,6 +191,7 @@
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
+      <scope>${protobuf.scope}</scope>
     </dependency>
 
     <dependency>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml

@@ -132,6 +132,7 @@
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
+      <scope>${protobuf.scope}</scope>
     </dependency>
     <dependency>
       <groupId>org.bouncycastle</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml

@@ -38,6 +38,7 @@
             <groupId>com.google.protobuf</groupId>
             <artifactId>protobuf-java</artifactId>
             <version>${hadoop.protobuf.version}</version>
+            <scope>${protobuf.scope}</scope>
         </dependency>
         <dependency>
             <groupId>io.netty</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml

@@ -89,6 +89,7 @@
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
+      <scope>${protobuf.scope}</scope>
     </dependency>
     <dependency>
       <groupId>junit</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml

@@ -81,6 +81,7 @@
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
+      <scope>${protobuf.scope}</scope>
     </dependency>
     <dependency>
       <groupId>commons-io</groupId>