hbase/dev-support/jdiffHBasePublicAPI.sh

#!/bin/bash
set -e

#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

################################################ ABOUT JDIFF #######################################################
#
#   What is JDiff? JDiff is a tool for comparing the public APIs of two separate Java codebases. Like diff, it will
#   give additions, changes, and removals. It will output an HTML report with the information.
#   To learn more, visit http://javadiff.sourceforge.net/.
#   JDiff is licensed under LGPL.

############################################# QUICK-START EXAMPLE ##################################################
#
#   Suppose we wanted to see the API diffs between HBase 0.92 and HBase 0.94. We could use this tool like so:
#   > ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.92 https://github.com/apache/hbase.git 0.94
#
#   This would generate a report in the local folder /tmp/jdiff/hbase_jdiff_report-p-0.92-c-0.94/
#   To view the report, simply examine /tmp/jdiff/hbase_jdiff_report-p-0.92-c-0.94/changes.html in your choice of
#   browser.
#
#   Note that this works because 0.92 and 0.94 have the source directory structure that is specified in the
#   hbase_jdiff_template.xml file. To compare 0.95 to 0.96, which have the post-singularity structure,  two other
#   template files (included) are used. The formats are autoated and is all taken care of automatically by the script.
#
#   On a local machine, JDiff reports have taken ~20-30 minutes to run. On Jenkins, it has taken over 35 minutes
#   in some cases. Your mileage may vary. Trunk and 0.95 take more time than 0.92 and 0.94.
#
#
############################################ SPECIFYING A LOCAL REPO ###############################################
#
#   The JDiff tool also works with local code. Instead of specifying a repo and a branch, you can specifying the
#   absolute path of the ./hbase folder and a name for code (e.g. experimental_94).
#
#   A local repo can be specified for none, one, or both of the sources.
#
############################################### EXAMPLE USE CASES ##################################################
#
#   Example 1: Generate a report to check if potential change doesn't break API compatibility with Apache HBase 0.94
#
#   In this case, you could compare the version you are using against a repo branch where your changes are.
#   >  ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.94 https://github.com/MY_REPO/hbase.git 0.94
#
#   Example 2: Generate a report to check if two branches of the same repo have any public API incompatibilities
#   >  ./jdiffHBasePublicAPI.sh https://github.com/MY_REPO/hbase.git $BRANCH_1  \
#   >   https://github.com/MY_REPO/hbase.git $BRANCH_2
#
#   Example 3: Have Example 1 done in a special directory in the user's home folder
#
#   >  export JDIFF_WORKING_DIRECTORY=~/jdiff_reports
#   >  ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.94 https://github.com/MY_REPO/hbase.git 0.94
#
#   Example 4: Check the API diff of a local change against an existing repo branch.
#   >  ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.95 /home/aleks/exp_hbase/hbase experiment_95
#
#   Example 5: Compare two local repos for public API changes
#   >  ./jdiffHBasePublicAPI.sh /home/aleks/stable_hbase/hbase stable_95 /home/aleks/exp_hbase/hbase experiment_95
#
#
############################################# READING A JDIFF REPORT ###############################################
#
#   The purpose of the JDiff report is show things that have changed between two versions of the public API. A user
#   would use this report to determine if committing a change would cause existing API clients to break. To do so,
#   there are specific things that one should look for in the report.
#
#   1. Identify the classes that constitute the public API. An example in 0.94 might be all classes in
#      org.apache.hadoop.hbase.client.*
#   2. After identifying those classes, go through each one and look for offending changes.
#      Those may include, but are not limited to:
#      1. Removed methods
#      2. Changed methods (including changes in return type and exception types)
#      3. Methods added to interfaces
#      4. Changed class inheritence information (may in innocuous but definitely worth validating)
#      5. Removed or renamed public static member variables and constants
#      6. Removed or renamed packages
#      7. Class moved to a different package

########################################### SETTING THE JDIFF WORKING DIRECTORY ####################################
#
#   By default, the working environment of jdiff is /tmp/jdiff. However, sometimes it is nice to have it place reports
#   and temp files elsewhere. In that case, please export JDIFF_WORKING_DIRECTORY into the bash environment and this
#   script will pick that up and use it.
#

scriptDirectory=$(dirname ${BASH_SOURCE[0]})
x=`echo $scriptDirectory | sed "s{\.{{g"`
DEV_SUPPORT_HOME="`pwd`$x"
. $scriptDirectory/jdiffHBasePublicAPI_common.sh

EXPECTED_ARGS=4

if [[ "$#" -ne "$EXPECTED_ARGS" ]]; then
  echo "This tool expects $EXPECTED_ARGS arguments, but received $#. Please check your command and try again.";
  echo "Usage: $0 <repoUrl or local source directory 1> <branch or source name 1> <repoUrl or local source directory 2> <branch or source name 2>"
  exit 1;
fi

echo "JDiff evaluation beginning:";
isGitRepo $1
FIRST_SOURCE_TYPE=$INPUT_FORMAT;
isGitRepo $3
SECOND_SOURCE_TYPE=$INPUT_FORMAT;

PREVIOUS_BRANCH=$2  ## We will still call it a branch even if it's not from a git repo.
CURRENT_BRANCH=$4

echo "We are going to compare source 1 which is a $FIRST_SOURCE_TYPE and source 2, which is a $SECOND_SOURCE_TYPE"


# Check that if either source is from a git repo, that the name is reasonable.
if [[ "$FIRST_SOURCE_TYPE" = "git_repo" ]]; then

  git check-ref-format --branch $2
fi

if [[ "$SECOND_SOURCE_TYPE" = "git_repo" ]]; then

  git check-ref-format --branch $4
fi

#If the JDIFF_WORKING_DIRECTORY is set, then we will output the report there. Otherwise, to the default location
if [[ "$JDIFF_WORKING_DIRECTORY" = "" ]]; then

  echo "JDIFF_WORKING_DIRECTORY not set. That's not an issue. We will default it to ./jidff"
  JDIFF_WORKING_DIRECTORY=/tmp/jdiff
else
  echo "JDIFF_WORKING_DIRECTORY set to $JDIFF_WORKING_DIRECTORY";
fi
mkdir -p $JDIFF_WORKING_DIRECTORY

# We will need this to reference the template we want to use
cd $JDIFF_WORKING_DIRECTORY
scenario_template_name=hbase_jdiff_p-$PREVIOUS_BRANCH-c-$CURRENT_BRANCH.xml


# Pull down JDiff tool and unpack it
if [ ! -d jdiff-1.1.1-with-incompatible-option ]; then
  curl -O http://cloud.github.com/downloads/tomwhite/jdiff/jdiff-1.1.1-with-incompatible-option.zip
  unzip jdiff-1.1.1-with-incompatible-option.zip
fi

JDIFF_HOME=`pwd`/jdiff-1.1.1-with-incompatible-option
cd $JDIFF_WORKING_DIRECTORY

# Pull down sources if necessary. Note that references to previous change are prefaced with p- in order to avoid collission of branch names
if [[ "$FIRST_SOURCE_TYPE" = "git_repo" ]]; then

  PREVIOUS_REPO=$1
  rm -rf p-$PREVIOUS_BRANCH
  mkdir -p p-$PREVIOUS_BRANCH
  cd p-$PREVIOUS_BRANCH
  git clone --depth 1 $PREVIOUS_REPO && cd hbase && git checkout origin/$PREVIOUS_BRANCH
  cd $JDIFF_WORKING_DIRECTORY
  HBASE_1_HOME=`pwd`/p-$PREVIOUS_BRANCH/hbase
else
  HBASE_1_HOME=$1
fi

echo "HBASE_1_HOME set to $HBASE_1_HOME"
echo "In HBASE_1_HOME, we have"
ls -la $HBASE_1_HOME

if [[ "$SECOND_SOURCE_TYPE" = "git_repo" ]]; then
  CURRENT_REPO=$3
  rm -rf $JDIFF_WORKING_DIRECTORY/c-$CURRENT_BRANCH
  mkdir -p $JDIFF_WORKING_DIRECTORY/c-$CURRENT_BRANCH
  cd $JDIFF_WORKING_DIRECTORY/c-$CURRENT_BRANCH
  git clone --depth 1 $CURRENT_REPO && cd hbase && git checkout origin/$CURRENT_BRANCH
  cd $JDIFF_WORKING_DIRECTORY
  HBASE_2_HOME=`pwd`/c-$CURRENT_BRANCH/hbase
else
  HBASE_2_HOME=$3
fi

echo "HBASE_2_HOME set to $HBASE_2_HOME"
echo "In HBASE_2_HOME, we have"
ls -la $HBASE_2_HOME

# Next step is to pull down the proper template based on the directory structure
isNewFormat $HBASE_1_HOME
export P_FORMAT=$BRANCH_FORMAT

isNewFormat $HBASE_2_HOME
export C_FORMAT=$BRANCH_FORMAT

if  [[ "$C_FORMAT" = "new" ]]; then

  if [[ "$P_FORMAT" = "new" ]]; then
    templateFile=$DEV_SUPPORT_HOME/hbase_jdiff_afterSingularityTemplate.xml
    echo "Previous format is of the new style. We'll be using template $templateFile";
  else
    templateFile=$DEV_SUPPORT_HOME/hbase_jdiff_acrossSingularityTemplate.xml
    echo "Previous format is of the old style. We'll be using template $templateFile";
  fi

else
  templateFile=$DEV_SUPPORT_HOME/hbase_jdiff_template.xml
  echo "Both formats are using the 94 and earlier style directory format. We'll be using template $templateFile"
fi

cp $templateFile $JDIFF_WORKING_DIRECTORY/$scenario_template_name

### Configure the jdiff script

### Note that PREVIOUS_BRANCH and CURRENT_BRANCH will be the absolute locations of the source.
echo "Configuring the jdiff script"
sed -i "s]hbase_jdiff_report]hbase_jdiff_report-p-$PREVIOUS_BRANCH-c-$CURRENT_BRANCH]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
sed -i "s]JDIFF_HOME_NAME]$JDIFF_HOME]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
sed -i "s]OLD_BRANCH_NAME]$HBASE_1_HOME]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
sed -i "s]NEW_BRANCH_NAME]$HBASE_2_HOME]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name

sed -i "s]V1]$PREVIOUS_BRANCH]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name
sed -i "s]V2]$CURRENT_BRANCH]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name

sed -i "s]JDIFF_FOLDER]$JDIFF_WORKING_DIRECTORY]g" $JDIFF_WORKING_DIRECTORY/$scenario_template_name

echo "Running jdiff";
ls -la $JDIFF_WORKING_DIRECTORY;
ant -f $JDIFF_WORKING_DIRECTORY/$scenario_template_name;

echo "jdiff operation complete. Report placed into $JDIFF_WORKING_DIRECTORY/hbase_jdiff_report-p-$PREVIOUS_BRANCH-c-$CURRENT_BRANCH/changes.html";