HBASE-18548 Move sources of website gen and check jobs into source control
This commit is contained in:
Misty Stanley-Jones
parent
ded0842caf
commit
6114824b53
|
|
@ -0,0 +1,47 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This script is designed to run as a Jenkins job, such as at
|
||||
# https://builds.apache.org/view/All/job/HBase%20Website%20Link%20Checker/
|
||||
#
|
||||
# It generates artifacts which the Jenkins job then can mail out and/or archive.
|
||||
#
|
||||
# We download a specific version of linklint because the original has bugs and
|
||||
# is not well maintained.
|
||||
#
|
||||
# See http://www.linklint.org/doc/inputs.html for linklint options
|
||||
|
||||
# Clean up the workspace
|
||||
rm -rf *.zip > /dev/null
|
||||
rm -rf linklint > /dev/null
|
||||
rm -Rf link_report
|
||||
|
||||
# This version of linklint fixes some bugs in the now-unmaintained 2.3.5 version
|
||||
wget http://ingo-karkat.de/downloads/tools/linklint/linklint-2.3.5_ingo_020.zip
|
||||
unzip linklint-2.3.5_ingo_020.zip
|
||||
chmod +x linklint/linklint.pl
|
||||
|
||||
# Run the checker
|
||||
echo "Checking http://hbase.apache.org and saving report to link_report/"
|
||||
echo "Excluding /testapidocs/ because some tests use private classes not published in /apidocs/."
|
||||
# Check internal structure
|
||||
linklint/linklint.pl -http \
|
||||
-host hbase.apache.org \
|
||||
/@ \
|
||||
-skip /testapidocs/@ \
|
||||
-skip /testdevapidocs/@ \
|
||||
-net \
|
||||
-redirect \
|
||||
-no_query_string \
|
||||
-htmlonly \
|
||||
-timeout 30 \
|
||||
-delay 1 \
|
||||
-limit 100000 \
|
||||
-doc link_report
|
||||
|
||||
# Detect whether we had errors and act accordingly
|
||||
if ! grep -q 'ERROR' link_report/index.html; then
|
||||
echo "Errors found. Sending email."
|
||||
exit 1
|
||||
else
|
||||
echo "No errors found. Warnings might be present."
|
||||
fi
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This script is meant to run as part of a Jenkins job such as
|
||||
# https://builds.apache.org/job/hbase_generate_website/
|
||||
#
|
||||
# It needs to be built on a Jenkins server with the label git-websites
|
||||
#
|
||||
# It expects to have the hbase repo cloned to the directory hbase
|
||||
#
|
||||
# If there is a build error, the Jenkins job is configured to send an email
|
||||
|
||||
LOCAL_REPO=${WORKSPACE}/.m2/repo
|
||||
# Nuke the local maven repo each time, to start with a known environment
|
||||
rm -Rf "${LOCAL_REPO}"
|
||||
mkdir -p "${LOCAL_REPO}"
|
||||
|
||||
# Clean any leftover files in case we are reusing the workspace
|
||||
rm -Rf -- *.patch *.patch.zip hbase/target target *.txt hbase-site
|
||||
|
||||
# Set up the environment
|
||||
export JAVA_HOME=$JDK_1_8_LATEST__HOME
|
||||
export PATH=$JAVA_HOME/bin:$MAVEN_3_3_3_HOME/bin:$PATH
|
||||
export MAVEN_OPTS="-XX:MaxPermSize=256m -Dmaven.repo.local=${LOCAL_REPO}"
|
||||
|
||||
# Verify the Maven version
|
||||
mvn -version
|
||||
|
||||
# Save and print the SHA we are building
|
||||
CURRENT_HBASE_COMMIT="$(git log --pretty=format:%H -n1)"
|
||||
echo "Current HBase commit: $CURRENT_HBASE_COMMIT"
|
||||
|
||||
# Clone the hbase-site repo manually so it doesn't trigger spurious
|
||||
# commits in Jenkins.
|
||||
git clone --depth 1 --branch asf-site https://git-wip-us.apache.org/repos/asf/hbase-site.git
|
||||
|
||||
# Figure out the last commit we built the site from, and bail if the build
|
||||
# still represents the SHA of HBase master
|
||||
cd "${WORKSPACE}/hbase-site" || exit -1
|
||||
git log --pretty=%s | grep ${CURRENT_HBASE_COMMIT}
|
||||
PUSHED=$?
|
||||
echo "PUSHED is $PUSHED"
|
||||
|
||||
if [ $PUSHED -eq 0 ]; then
|
||||
echo "$CURRENT_HBASE_COMMIT is already mentioned in the hbase-site commit log. Not building."
|
||||
exit 0
|
||||
else
|
||||
echo "$CURRENT_HBASE_COMMIT is not yet mentioned in the hbase-site commit log. Assuming we don't have it yet. $PUSHED"
|
||||
fi
|
||||
|
||||
# Go to the hbase directory so we can build the site
|
||||
cd "${WORKSPACE}/hbase" || exit -1
|
||||
|
||||
# This will only be set for builds that are triggered by SCM change, not manual builds
|
||||
if [ "$CHANGE_ID" ]; then
|
||||
echo -n " ($CHANGE_ID - $CHANGE_TITLE)"
|
||||
fi
|
||||
|
||||
# Build and install HBase, then build the site
|
||||
echo "Building HBase"
|
||||
mvn \
|
||||
-DskipTests \
|
||||
-Dmaven.javadoc.skip=true \
|
||||
--batch-mode \
|
||||
-Dcheckstyle.skip=true \
|
||||
-Dfindbugs.skip=true \
|
||||
--log-file="${WORKSPACE}/hbase-build-log-${CURRENT_HBASE_COMMIT}.txt" \
|
||||
clean install \
|
||||
&& mvn clean site \
|
||||
--batch-mode \
|
||||
-DskipTests \
|
||||
--log-file="${WORKSPACE}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt"
|
||||
|
||||
status=$?
|
||||
if [ $status -ne 0 ]; then
|
||||
echo "Failure: mvn clean site"
|
||||
exit $status
|
||||
fi
|
||||
|
||||
# Stage the site
|
||||
echo "Staging HBase site"
|
||||
mvn \
|
||||
--batch-mode \
|
||||
--log-file="${WORKSPACE}/hbase-stage-log-${CURRENT_HBASE_COMMIT}.txt" \
|
||||
site:stage
|
||||
status=$?
|
||||
if [ $status -ne 0 ] || [ ! -d target/staging ]; then
|
||||
echo "Failure: mvn site:stage"
|
||||
exit $status
|
||||
fi
|
||||
|
||||
# Get ready to update the hbase-site repo with the new artifacts
|
||||
cd "${WORKSPACE}/hbase-site" || exit -1
|
||||
|
||||
#Remove previously-generated files
|
||||
FILES_TO_REMOVE=("hbase-*"
|
||||
"apidocs"
|
||||
"devapidocs"
|
||||
"testapidocs"
|
||||
"testdevapidocs"
|
||||
"xref"
|
||||
"xref-test"
|
||||
"*book*"
|
||||
"*.html"
|
||||
"*.pdf*"
|
||||
"css"
|
||||
"js"
|
||||
"images")
|
||||
|
||||
for FILE in "${FILES_TO_REMOVE[@]}"; do
|
||||
echo "Removing ${WORKSPACE}/hbase-site/$FILE"
|
||||
rm -Rf "${FILE}"
|
||||
done
|
||||
|
||||
# Copy in the newly-built artifacts
|
||||
cp -au "${WORKSPACE}"/hbase/target/staging/* .
|
||||
|
||||
# If the index.html is missing, bail because this is serious
|
||||
if [ ! -f index.html ]; then
|
||||
echo "The index.html is missing. Aborting."
|
||||
exit 1
|
||||
else
|
||||
# Add all the changes
|
||||
echo "Adding all the files we know about"
|
||||
git add .
|
||||
# Create the commit message and commit the changes
|
||||
WEBSITE_COMMIT_MSG="Published site at $CURRENT_HBASE_COMMIT."
|
||||
echo "WEBSITE_COMMIT_MSG: $WEBSITE_COMMIT_MSG"
|
||||
git commit -m "${WEBSITE_COMMIT_MSG}" -a
|
||||
# Dump a little report
|
||||
echo "This commit changed these files (excluding Modified files):"
|
||||
git diff --name-status --diff-filter=ADCRTXUB origin/asf-site
|
||||
# Create a patch, which Jenkins can save as an artifact and can be examined for debugging
|
||||
git format-patch --stdout origin/asf-site > "${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch"
|
||||
echo "Change set saved to patch ${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch"
|
||||
# Push the real commit
|
||||
git push origin asf-site || (echo "Failed to push to asf-site. Website not updated." && exit -1)
|
||||
# Create an empty commit to work around INFRA-10751
|
||||
git commit --allow-empty -m "INFRA-10751 Empty commit"
|
||||
# Push the empty commit
|
||||
git push origin asf-site || (echo "Failed to push the empty commit to asf-site. Website may not update. Manually push an empty commit to fix this. (See INFRA-10751)" && exit -1)
|
||||
echo "Pushed the changes to branch asf-site. Refresh http://hbase.apache.org/ to see the changes within a few minutes."
|
||||
git fetch origin
|
||||
git reset --hard origin/asf-site
|
||||
|
||||
# Zip up the patch so Jenkins can save it
|
||||
cd "${WORKSPACE}" || exit -1
|
||||
zip website.patch.zip "${CURRENT_HBASE_COMMIT}.patch"
|
||||
fi
|
||||
|
||||
#echo "Dumping current environment:"
|
||||
#env
|
||||
|
||||
|
|
@ -138,23 +138,23 @@ When you are satisfied with your changes, follow the procedure in
|
|||
[[website_publish]]
|
||||
=== Publishing the HBase Website and Documentation
|
||||
|
||||
HBase uses the ASF's `gitpubsub` mechanism.
|
||||
. After generating the website and documentation
|
||||
artifacts using `mvn clean site site:stage`, check out the `asf-site` repository.
|
||||
HBase uses the ASF's `gitpubsub` mechanism. A Jenkins job runs the
|
||||
`dev-support/jenkins-scripts/generate-hbase-website.sh` script, which runs the
|
||||
`mvn clean site site:stage` against the `master` branch of the `hbase`
|
||||
repository and commits the built artifacts to the `asf-site` branch of the
|
||||
`hbase-site` repository. When the commit is pushed, the website is redeployed
|
||||
automatically. If the script encounters an error, an email is sent to the
|
||||
developer mailing list. You can run the script manually or examine it to see the
|
||||
steps involved.
|
||||
|
||||
. Remove previously-generated content using the following command:
|
||||
+
|
||||
----
|
||||
rm -rf rm -rf *apidocs* *book* *.html *.pdf* css js
|
||||
----
|
||||
+
|
||||
WARNING: Do not remove the `0.94/` directory. To regenerate them, you must check out
|
||||
the 0.94 branch and run `mvn clean site site:stage` from there, and then copy the
|
||||
artifacts to the 0.94/ directory of the `asf-site` branch.
|
||||
[[website_check_links]]
|
||||
=== Checking the HBase Website for Broken Links
|
||||
|
||||
. Copy the contents of `target/staging` to the branch.
|
||||
|
||||
. Add and commit your changes, and submit a patch for review.
|
||||
A Jenkins job runs periodically to check HBase website for broken links, using
|
||||
the `dev-support/jenkins-scripts/check-website-links.sh` script. This script
|
||||
uses a tool called `linklint` to check for bad links and create a report. If
|
||||
broken links are found, an email is sent to the developer mailing list. You can
|
||||
run the script manually or examine it to see the steps involved.
|
||||
|
||||
=== HBase Reference Guide Style Guide and Cheat Sheet
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue