hbase/dev-support/jenkins-scripts/generate-hbase-website.sh

286 lines
9.7 KiB
Bash

#!/bin/bash
#
#/**
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
# * regarding copyright ownership. The ASF licenses this file
# * to you under the Apache License, Version 2.0 (the
# * "License"); you may not use this file except in compliance
# * with the License. You may obtain a copy of the License at
# *
# * http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
# This script is meant to run as part of a Jenkins job such as
# https://builds.apache.org/job/hbase_generate_website/
#
# It needs to be built on a Jenkins server with the label git-websites
#
# Allows specifying options for working directory, maven repo, and publishing to git
# run with --help for usage.
#
# If there is a build error, the Jenkins job is configured to send an email
declare CURRENT_HBASE_COMMIT
declare PUSHED
declare FILE
declare WEBSITE_COMMIT_MSG
declare -a FILES_TO_REMOVE
set -e
function usage {
echo "Usage: ${0} [options] /path/to/hbase/checkout"
echo ""
echo " --working-dir /path/to/use Path for writing logs and a local checkout of hbase-site repo."
echo " if given must exist."
echo " defaults to making a directory in /tmp."
echo " --local-repo /path/for/maven/.m2 Path for putting local maven repo."
echo " if given must exist."
echo " defaults to making a clean directory in --working-dir."
echo " --publish if given, will attempt to push results back to the hbase-site repo."
echo " --help show this usage message."
exit 1
}
# if no args specified, show usage
if [ $# -lt 1 ]; then
usage
fi
# Get arguments
declare component_dir
declare working_dir
declare local_repo
declare publish
while [ $# -gt 0 ]
do
case "$1" in
--working-dir) shift; working_dir=$1; shift;;
--local-repo) shift; local_repo=$1; shift;;
--publish) shift; publish="true";;
--) shift; break;;
-*) usage ;;
*) break;; # terminate while loop
esac
done
# should still have where component checkout is.
if [ $# -lt 1 ]; then
usage
fi
component_dir="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
if [ -z "${working_dir}" ]; then
echo "[DEBUG] defaulting to creating a directory in /tmp"
working_dir=/tmp
while [[ -e ${working_dir} ]]; do
working_dir=/tmp/hbase-generate-website-${RANDOM}.${RANDOM}
done
mkdir "${working_dir}"
else
# absolutes please
working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
if [ ! -d "${working_dir}" ]; then
echo "passed working directory '${working_dir}' must already exist."
exit 1
fi
fi
echo "You'll find logs and temp files in ${working_dir}"
if [ -z "${local_repo}" ]; then
echo "[DEBUG] defaulting to creating a local repo within '${working_dir}'"
local_repo="${working_dir}/.m2/repo"
# Nuke the local maven repo each time, to start with a known environment
rm -Rf "${local_repo}"
mkdir -p "${local_repo}"
else
# absolutes please
local_repo="$(cd "$(dirname "${local_repo}")"; pwd)/$(basename "${local_repo}")"
if [ ! -d "${local_repo}" ]; then
echo "passed directory for storing the maven repo '${local_repo}' must already exist."
exit 1
fi
fi
# Set up the environment
if [ -z "${JAVA_HOME}" ]; then
JAVA_HOME="${JDK_1_8_LATEST__HOME}"
export JAVA_HOME
export PATH="${JAVA_HOME}/bin:${PATH}"
fi
if [ -z "${MAVEN_HOME}" ]; then
MAVEN_HOME="${MAVEN_3_3_3_HOME}"
export MAVEN_HOME
export PATH="${MAVEN_HOME}/bin:${PATH}"
fi
export MAVEN_OPTS="${MAVEN_OPTS} -Dmaven.repo.local=${local_repo}"
# Verify the Maven version
mvn -version
# Verify the git version
git --version
cd "${working_dir}"
# Clean any leftover files in case we are reusing the workspace
rm -Rf -- *.patch *.patch.zip target *.txt hbase-site
# Save and print the SHA we are building
CURRENT_HBASE_COMMIT="$(cd "${component_dir}" && git show-ref --hash --dereference --verify refs/remotes/origin/HEAD)"
# Fail if it's empty
if [ -z "${CURRENT_HBASE_COMMIT}" ]; then
echo "Got back a blank answer for the current HEAD on the remote hbase repository. failing."
exit 1
fi
echo "Current HBase commit: $CURRENT_HBASE_COMMIT"
# Clone the hbase-site repo manually so it doesn't trigger spurious
# commits in Jenkins.
git clone --depth 1 --branch asf-site https://git-wip-us.apache.org/repos/asf/hbase-site.git
# Figure out if the commit of the hbase repo has already been built and bail if so.
declare -i PUSHED
PUSHED=$(cd hbase-site && git rev-list --grep "${CURRENT_HBASE_COMMIT}" --fixed-strings --count HEAD)
echo "[DEBUG] hash was found in $PUSHED commits for hbase-site repository."
if [ "${PUSHED}" -ne 0 ]; then
echo "$CURRENT_HBASE_COMMIT is already mentioned in the hbase-site commit log. Not building."
exit 0
else
echo "$CURRENT_HBASE_COMMIT is not yet mentioned in the hbase-site commit log. Assuming we don't have it yet."
fi
# Go to the hbase directory so we can build the site
cd "${component_dir}"
# This will only be set for builds that are triggered by SCM change, not manual builds
if [ -n "$CHANGE_ID" ]; then
echo -n " ($CHANGE_ID - $CHANGE_TITLE)"
fi
# Build and install HBase, then build the site
echo "Building HBase"
# TODO we have to do a local install first because for whatever reason, the maven-javadoc-plugin's
# forked compile phase requires that test-scoped dependencies be available, which
# doesn't work since we will not have done a test-compile phase (MJAVADOC-490). the first place this
# breaks for me is hbase-server trying to find hbase-http:test and hbase-zookeeper:test.
# But! some sunshine: because we're doing a full install before running site, we can skip all the
# compiling in the forked executions. We have to do it awkwardly because MJAVADOC-444.
if mvn \
-DskipTests \
-Dmaven.javadoc.skip=true \
--batch-mode \
-Denforcer.skip=true \
-Dcheckstyle.skip=true \
-Dfindbugs.skip=true \
--log-file="${working_dir}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt" \
clean install \
&& mvn site \
--batch-mode \
-Denforcer.skip=true \
-Dmaven.main.skip=true \
-Dmaven.test.skip=true \
-DskipTests \
--log-file="${working_dir}/hbase-site-log-${CURRENT_HBASE_COMMIT}.txt"; then
echo "Successfully built site."
else
status=$?
echo "Maven commands to build the site failed. check logs for details ${working_dir}/hbase-*-log-*.txt"
exit $status
fi
# Stage the site
echo "Staging HBase site"
mvn \
--batch-mode \
--log-file="${working_dir}/hbase-stage-log-${CURRENT_HBASE_COMMIT}.txt" \
site:stage
status=$?
if [ $status -ne 0 ] || [ ! -d target/staging ]; then
echo "Failure: mvn site:stage"
exit $status
fi
# Get ready to update the hbase-site repo with the new artifacts
cd "${working_dir}/hbase-site"
#Remove previously-generated files
FILES_TO_REMOVE=("hbase-*"
"apidocs"
"devapidocs"
"testapidocs"
"testdevapidocs"
"xref"
"xref-test"
"*book*"
"*.html"
"*.pdf*"
"css"
"js"
"images")
for FILE in "${FILES_TO_REMOVE[@]}"; do
if [ -e "${FILE}" ]; then
echo "Removing hbase-site/$FILE"
rm -Rf "${FILE}"
fi
done
# Copy in the newly-built artifacts
# TODO what do we do when the site build wants to remove something? Can't rsync because e.g. release-specific docs.
cp -pPR "${component_dir}"/target/staging/* .
# If the index.html is missing, bail because this is serious
if [ ! -f index.html ]; then
echo "The index.html is missing. Aborting."
exit 1
fi
echo "Adding all the files we know about"
git add .
# Create the commit message and commit the changes
WEBSITE_COMMIT_MSG="Published site at $CURRENT_HBASE_COMMIT."
echo "WEBSITE_COMMIT_MSG: $WEBSITE_COMMIT_MSG"
git commit -m "${WEBSITE_COMMIT_MSG}" -a
# Dump a little report
echo "This commit changed these files (excluding Modified files):"
git diff --name-status --diff-filter=ADCRTXUB origin/asf-site | tee "${working_dir}/hbase-file-diff-summary-${CURRENT_HBASE_COMMIT}.txt"
# Create a patch, which Jenkins can save as an artifact and can be examined for debugging
git format-patch --stdout origin/asf-site > "${working_dir}/${CURRENT_HBASE_COMMIT}.patch"
if [ ! -s "${working_dir}/${CURRENT_HBASE_COMMIT}.patch" ]; then
echo "Something went wrong when creating the patch of our updated site."
exit 1
fi
echo "Change set saved to patch ${working_dir}/${CURRENT_HBASE_COMMIT}.patch"
if [ -n "${publish}" ]; then
echo "Publishing changes to remote repo..."
if git push origin asf-site; then
"changes pushed."
else
echo "Failed to push to asf-site. Website not updated."
exit 1
fi
echo "Sending empty commit to work around INFRA-10751."
git commit --allow-empty -m "INFRA-10751 Empty commit"
# Push the empty commit
if git push origin asf-site; then
echo "empty commit pushed."
else
echo "Failed to push the empty commit to asf-site. Website may not update. Manually push an empty commit to fix this. (See INFRA-10751)"
exit 1
fi
echo "Pushed the changes to branch asf-site. Refresh http://hbase.apache.org/ to see the changes within a few minutes."
fi
# Zip up the patch so Jenkins can save it
cd "${working_dir}"
zip website.patch.zip "${CURRENT_HBASE_COMMIT}.patch"