Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18577 test well-formed-ness of our client jars. 

* Removes transitive hadoop deps from hbase-client that we don't use
* removes curator-recipes dep from hbase-client that we don't use
* removes jetty-util transitive from hbase-client that we don't use
* moves junit from compile to test scope for hbase-client
* adds an IT that checks the contents of jars
* excludes our server side web apps and related deps (jaxb, jersey, jetty) from shaded jars
* excludes proto source files from shaded jars
* relocates codahale metrics, commons-crypto, and curator
* finishes relocating jets3t
* moves logging backends to optional in shaded artifacts
* updates maven dependency plugin

Signed-off-by: Michael Stack <stack@apache.org>
This commit is contained in:
Sean Busbey 2017-08-10 16:16:20 -05:00
parent b2a304ff84
commit bc5478f947
6 changed files with 425 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hbase-client/pom.xml
View File

@ -119,6 +119,13 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop2-compat</artifactId>
<exclusions>
<!-- We don't need MR support classes here. -->
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
@ -193,13 +200,14 @@
<artifactId>metrics-core</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
@ -272,6 +280,10 @@
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>

186
hbase-shaded/hbase-shaded-check-invariants/pom.xml Normal file
View File

@ -0,0 +1,186 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>hbase</artifactId>
<groupId>org.apache.hbase</groupId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../..</relativePath>
</parent>
<artifactId>hbase-shaded-check-invariants</artifactId>
<packaging>pom</packaging>
<description>
Enforces our invariants for our shaded artifacts. e.g. shaded clients have
a specific set of transitive dependencies and shaded clients only contain
classes that are in particular packages. Does the enforcement through
the maven-enforcer-plugin and and integration test.
</description>
<name>Apache HBase Shaded Packaging Invariants</name>
<properties>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-server</artifactId>
<version>${project.version}</version>
</dependency>
<!-- parent pom defines these for children. :( :( :( -->
<dependency>
<groupId>com.github.stephenc.findbugs</groupId>
<artifactId>findbugs-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>provided</scope>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>extra-enforcer-rules</artifactId>
<version>1.0-beta-3</version>
</dependency>
</dependencies>
<executions>
<execution>
<id>enforce-banned-dependencies</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<skip>true</skip>
<rules>
<banTransitiveDependencies>
<!--
<message>
Our client-facing artifacts are not supposed to have additional dependencies
and one or more of them do. The output from the enforcer plugin should give
specifics.
</message>
-->
<excludes>
<!-- We leave logging stuff alone -->
<exclude>org.slf4j:*</exclude>
<exclude>log4j:*</exclude>
<exclude>commons-logging:*</exclude>
<!-- annotations that never change -->
<exclude>com.google.code.findbugs:*</exclude>
<exclude>com.github.stephenc.findbugs:*</exclude>
<!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
<exclude>org.apache.htrace:*</exclude>
</excludes>
</banTransitiveDependencies>
<banDuplicateClasses>
<findAllDuplicates>true</findAllDuplicates>
</banDuplicateClasses>
</rules>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>test-resources</id>
<phase>pre-integration-test</phase>
<goals>
<goal>testResources</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<!-- create a maven pom property that has all of our dependencies.
below in the integration-test phase we'll pass this list
of paths to our jar checker script.
-->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>put-client-artifacts-in-a-property</id>
<phase>pre-integration-test</phase>
<goals>
<goal>build-classpath</goal>
</goals>
<configuration>
<excludeScope>provided</excludeScope>
<excludeTransitive>true</excludeTransitive>
<outputProperty>hbase-client-artifacts</outputProperty>
</configuration>
</execution>
</executions>
</plugin>
<!--
Check that we actually relocated everything we included.
It's critical that we don't ship third party dependencies that haven't
been relocated under our pacakge space, since this will lead to
difficult to debug classpath errors for downstream. Unfortunately, that
means inspecting all the jars.
-->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<executions>
<execution>
<id>check-jar-contents</id>
<phase>integration-test</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>${shell-executable}</executable>
<workingDirectory>${project.build.testOutputDirectory}</workingDirectory>
<requiresOnline>false</requiresOnline>
<arguments>
<argument>ensure-jars-have-correct-contents.sh</argument>
<argument>${hbase-client-artifacts}</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

75
hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh Normal file
View File

@ -0,0 +1,75 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Usage: $0 [/path/to/some/example.jar:/path/to/another/example/created.jar]
#
# accepts a single command line argument with a colon separated list of
# paths to jars to check. Iterates through each such passed jar and checks
# all the contained paths to make sure they follow the below constructed
# safe list.
# we have to allow the directories that lead to the org/apache/hadoop dir
allowed_expr="(^org/$|^org/apache/$"
# We allow the following things to exist in our client artifacts:
# * classes in packages that start with org.apache.hadoop, which by
# convention should be in a path that looks like org/apache/hadoop
allowed_expr+="|^org/apache/hadoop/"
# * whatever in the "META-INF" directory
allowed_expr+="|^META-INF/"
# * the folding tables from jcodings
allowed_expr+="|^tables/"
# * Hadoop's and HBase's default configuration files, which have the form
# "_module_-default.xml"
allowed_expr+="|^[^-]*-default.xml$"
# * Hadoop's versioning properties files, which have the form
# "_module_-version-info.properties"
allowed_expr+="|^[^-]*-version-info.properties$"
# * Hadoop's application classloader properties file.
allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
# public suffix list used by httpcomponents
allowed_expr+="|^mozilla/$"
allowed_expr+="|^mozilla/public-suffix-list.txt$"
# Comes from commons-configuration, not sure if relocatable.
allowed_expr+="|^digesterRules.xml$"
allowed_expr+="|^properties.dtd$"
allowed_expr+="|^PropertyList-1.0.dtd$"
allowed_expr+=")"
declare -i bad_artifacts=0
declare -a bad_contents
IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1")
for artifact in "${artifact_list[@]}"; do
bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}"))
if [ ${#bad_contents[@]} -gt 0 ]; then
echo "[ERROR] Found artifact with unexpected contents: '${artifact}'"
echo " Please check the following and either correct the build or update"
echo " the allowed list with reasoning."
echo ""
for bad_line in "${bad_contents[@]}"; do
echo " ${bad_line}"
done
bad_artifacts=${bad_artifacts}+1
else
echo "[INFO] Artifact looks correct: '$(basename "${artifact}")'"
fi
done
# if there was atleast one bad artifact, exit with failure
if [ "${bad_artifacts}" -gt 0 ]; then
exit 1
fi

65
hbase-shaded/hbase-shaded-server/pom.xml
View File

@ -51,6 +51,71 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<exclusions>
<!-- Jaxb-api is a part of Java SE now -->
<exclusion>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</exclusion>
<!-- Jersey not used by our MR support -->
<exclusion>
<groupId>javax.ws.rs</groupId>
<artifactId>javax.ws.rs-api</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey.contribs</groupId>
<artifactId>jersey-guice</artifactId>
</exclusion>
<!-- Jetty not used by our MR support -->
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util-ajax</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-jsp</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson1</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

77
hbase-shaded/pom.xml
View File

@ -40,6 +40,7 @@
<modules>
<module>hbase-shaded-client</module>
<module>hbase-shaded-server</module>
<module>hbase-shaded-check-invariants</module>
</modules>
<dependencies>
<dependency>
@ -48,6 +49,23 @@
<version>${project.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<!-- log4j has to be non-optional for Hadoop 2 atleast -->
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</dependency>
<!-- put the log implementations to optional -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
@ -120,7 +138,6 @@
<createSourcesJar>false</createSourcesJar>
<shadedArtifactAttached>false</shadedArtifactAttached>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<dependencyReducedPomLocation>${project.build.directory}/dependency-reduced-pom.xml</dependencyReducedPomLocation>
<shadeTestJar>false</shadeTestJar>
<artifactSet>
<excludes>
@ -135,6 +152,10 @@
</artifactSet>
<relocations>
<!-- top level com not including sun-->
<relocation>
<pattern>com.codahale</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.com.codahale</shadedPattern>
</relocation>
<relocation>
<pattern>com.google</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.com.google</shadedPattern>
@ -215,6 +236,11 @@
<pattern>org.jets3t</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.org.jets3t</shadedPattern>
</relocation>
<!-- poorly named add-on package from jets3t dependency. TODO can we just exclude these? -->
<relocation>
<pattern>contribs.mx</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.contribs.mx</shadedPattern>
</relocation>
<relocation>
<pattern>org.objectweb</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.org.objectweb</shadedPattern>
@ -226,6 +252,10 @@
<pattern>org.apache.avro</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.org.apache.avro</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.curator</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.org.apache.curator</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.directory</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.org.apache.directory</shadedPattern>
@ -268,6 +298,10 @@
<pattern>org.apache.commons.configuration</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.org.apache.commons.configuration</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons.crypto</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.org.apache.commons.crypto</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons.daemon</pattern>
<shadedPattern>org.apache.hadoop.hbase.shaded.org.apache.commons.daemon</shadedPattern>
@ -339,6 +373,9 @@
<resources>
<resource>LICENSE.txt</resource>
<resource>ASL2.0</resource>
<resource>LICENSE-junit.txt</resource>
<!-- also this unneeded doc -->
<resource>overview.html</resource>
</resources>
</transformer>
<!-- Where notices exist, just concat them -->
@ -356,6 +393,44 @@
<exclude>META-INF/ECLIPSEF.RSA</exclude>
</excludes>
</filter>
<filter>
<!-- server side webapps that we don't need -->
<artifact>org.apache.hbase:hbase-server</artifact>
<excludes>
<exclude>hbase-webapps/*</exclude>
<exclude>hbase-webapps/**/*</exclude>
</excludes>
</filter>
<filter>
<!-- server side webapps that we don't need -->
<artifact>org.apache.hadoop:hadoop-yarn-common</artifact>
<excludes>
<exclude>webapps/*</exclude>
<exclude>webapps/**/*</exclude>
</excludes>
</filter>
<filter>
<!-- proto source files aren't needed -->
<artifact>*:*</artifact>
<excludes>
<exclude>*.proto</exclude>
<exclude>**/*.proto</exclude>
</excludes>
</filter>
<filter>
<!-- skip french localization -->
<artifact>org.apache.commons:commons-math3</artifact>
<excludes>
<exclude>assets/org/apache/commons/math3/**/*</exclude>
</excludes>
</filter>
<filter>
<!-- appears to be the result of a conflict in hadoop artifacts -->
<artifact>org.apache.hadoop:*</artifact>
<excludes>
<exclude>mapred-default.xml.orig</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>

7
pom.xml
View File

@ -567,6 +567,11 @@
-->
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven.dependency.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
@ -1444,6 +1449,7 @@
<maven.bundle.version>3.3.0</maven.bundle.version>
<maven.checkstyle.version>2.17</maven.checkstyle.version>
<maven.compiler.version>3.6.1</maven.compiler.version>
<maven.dependency.version>3.0.1</maven.dependency.version>
<maven.eclipse.version>2.10</maven.eclipse.version>
<maven.install.version>2.5.2</maven.install.version>
<maven.jar.version>3.0.2</maven.jar.version>
@ -1476,6 +1482,7 @@
<it.test.jar>hbase-it-${project.version}-tests.jar</it.test.jar>
<annotations.test.jar>hbase-annotations-${project.version}-tests.jar</annotations.test.jar>
<rsgroup.test.jar>hbase-rsgroup-${project.version}-tests.jar</rsgroup.test.jar>
<shell-executable>bash</shell-executable>
<surefire.version>2.19.1</surefire.version>
<surefire.provider>surefire-junit47</surefire.provider>
<!-- default: run small & medium, medium with 2 threads -->