Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14642 Update website-publishing script

This commit is contained in:
Misty Stanley-Jones 2015-10-20 12:02:53 +10:00
parent c1f0442045
commit 51693b9cde
1 changed files with 123 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

234
dev-support/publish_hbase_website.sh
View File

@ -32,8 +32,8 @@ if [ "$#" == "0" ]; then
fi fi
# Process args # Process args
INTERACTIVE= INTERACTIVE=0
AUTO= AUTO=0
GIT_DIR= GIT_DIR=
SVN_DIR= SVN_DIR=
while getopts "hiag:s:" OPTION while getopts "hiag:s:" OPTION
@ -44,41 +44,50 @@ do
exit exit
;; ;;
i) i)
INTERACTIVE=1 if [ $AUTO -eq 1 ]; then
echo "Cannot specify both -i and -a."
echo -e "$USAGE"
exit 1
else
INTERACTIVE=1
AUTO=0
fi
;; ;;
a) a)
# We don't actually use this variable but require it to be # We don't actually use this variable but require it to be
# set explicitly because it will commit changes without asking # set explicitly because it will commit changes without asking
AUTO=1 if [ $INTERACTIVE -eq 1 ]; then
echo "Cannot specify both -i and -a."
echo -e "$USAGE"
exit 1
else
AUTO=1
INTERACTIVE=0
fi
;; ;;
g) g)
GIT_DIR=$OPTARG GIT_DIR=$OPTARG
if [ ! -d "$GIT_DIR/.git" ]; then
echo "Git directory $GIT_DIR does not exist or is not a Git repository."
exit 1
fi
;; ;;
s) s)
SVN_DIR=$OPTARG SVN_DIR=$OPTARG
if [ ! -d "$SVN_DIR/.svn" ]; then
echo "SVN directory $SVN_DIR does not exist or is not a Subversion repository."
exit 1
fi
;; ;;
esac esac
done done
if [ $INTERACTIVE ] && [ $AUTO ]; then
echo "Only one of -i or -a can be used."
echo -e $USAGE
exit 1
fi
# Set GIT_DIR and SVN_DIR to defaults if not given # Set GIT_DIR and SVN_DIR to defaults if not given
if [ ! $GIT_DIR ]; then if [ -z "$GIT_DIR" ]; then
GIT_DIR=~/git/hbase GIT_DIR="$HOME/git/hbase"
fi fi
if [ ! $SVN_DIR ]; then if [ -z "$SVN_DIR" ]; then
SVN_DIR=~/svn/hbase.apache.org/trunk SVN_DIR="$HOME/svn/hbase.apache.org/trunk"
fi
# Check that GIT_DIR and SVN_DIR exist
if [ ! -d $GIT_DIR -o ! -d $SVN_DIR ]; then
echo "Both the GIT and SVN directories must exist."
echo -e $USAGE
exit 1
fi fi
export MAVEN_OPTS=${MAVEN_OPTS:-"-Xmx3g -XX:MaxPermSize=256m"} export MAVEN_OPTS=${MAVEN_OPTS:-"-Xmx3g -XX:MaxPermSize=256m"}
@ -87,73 +96,102 @@ cd $GIT_DIR
# Get the latest # Get the latest
echo "Updating Git" echo "Updating Git"
git checkout master git checkout -q master
git pull git pull -q
status=$?
if [ $status -ne 0 ]; then
echo "git pull command failed."
echo " Please examine and run this script again after you have corrected the problem."
fi
GIT_SHA=$(git log --pretty=format:%H -n1)
# Generate the site to ~/git/hbase/target/site # Generate the site to ~/git/hbase/target/site
if [ $INTERACTIVE ]; then if [ $INTERACTIVE -eq 1 ]; then
read -p "Build the site? (y/n)" yn read -p "Build the site? (y/n)" yn
case $yn in case $yn in
[Yy]* ) [Yy]* )
mvn clean package javadoc:aggregate post-site site:stage -DskipTests -Dcheckstyle.skip echo "Building $GIT_SHA"
status=$? mvn clean site post-site site:stage
if [ $status -ne 0 ]; then status=$?
echo "The website does not build. Aborting." if [ $status -ne 0 ]; then
exit $status echo "The site does not build. Aborting."
fi exit $status
;; fi
[Nn]* ) ;;
echo "Not building the site." [Nn]* )
;; echo "Not building the site."
esac if [ -d target/staging ]; then
echo "Using site in $GIT_DIR/target/staging directory."
else
echo "Not building the site but $GIT_DIR/target/staging does not exist. Giving up."
exit 1
fi
;;
esac
else else
echo "Building the site in auto mode." echo "Building the site in auto mode."
mvn clean package javadoc:aggregate post-site site:stage -DskipTests -Dcheckstyle.skip mvn clean site post-site site:stage
status=$? status=$?
if [ $status != 0 ]; then if [ $status != 0 ]; then
echo "The website does not build. Aborting." echo "The site does not build. Aborting."
exit $status exit $status
fi fi
fi fi
# Refresh the local copy of the live website # Refresh the local copy of the live website
echo "Updating Subversion..." echo "Updating Subversion."
cd $SVN_DIR cd $SVN_DIR
# Be aware that this will restore all the files deleted a few lines down # Be aware that this will restore all the files deleted a few lines down
# if you are debugging this script # if you are debugging this script
# and need to run it multiple times without svn committing # and need to run it multiple times without svn committing
svn update > /dev/null svn update -q
# Get current size of svn directory and # files, for sanity checking before commit # Make a list of things that are in SVN but not the generated site
SVN_OLD_SIZE=`du -sm . |awk '{print $1}'` # and not auto-generated
SVN_OLD_NUMFILES=`find . -type f |wc -l |awk '{print $1}'`
# Make a list of things that are in SVN but not the generated site
# and not auto-generated
# -- we might need to delete these # -- we might need to delete these
echo "The following directories (if any) might be stale:" |tee /tmp/out.txt # But we never want to delete images or logos
find . -type d ! -path '*0.94*' ! -path '*apidocs*' \
! -path '*xref*' ! -path '*book*' \
-exec ls $GIT_DIR/target/staging/{} \; |grep 'No such' |tee -a /tmp/out.txt
echo "The following files (if any) might be stale:" |tee -a /tmp/out.txt
find . -type f ! -path '*0.94*' ! -path '*apidocs*' \
! -path '*xref*' ! -path '*book*' ! -name '*.svg' \
-exec ls $GIT_DIR/target/staging/{} \; |grep 'No such' |tee -a /tmp/out.txt
if [ $INTERACTIVE ]; then stale_dirs_exist=0
read -p "Exit to take care of them? (y/n)" yn echo "The following DIRECTORIES (if any) might be stale:" |tee /tmp/out.txt
case $yn in find . -type d ! -path '*0.94*' ! -path '*apidocs*' ! -path '*xref*' ! -path '*book*' ! -path '*svn*' | \
[Yy]* ) sed 's/\.\///g' | grep -v images | grep -v logos | grep -v img | grep -v css | \
while read i; do
if [ ! -d "$GIT_DIR/target/staging/$i" ]; then
echo $i
stale_dirs_exist=1
fi
done
stale_files_exist=0
echo "The following FILES (if any) might be stale:" |tee -a /tmp/out.txt
find . -type f ! -path '*0.94*' ! -path '*apidocs*' ! -path '*xref*' ! -path '*book*' ! -path '*svn*' | \
sed 's/\.\///g' | grep -v images | grep -v logos | grep -v img | grep -v css | grep -v hbase-default.xml | \
while read i; do
if [ ! -f "$GIT_DIR/target/staging/$i" ]; then
echo $i
stale_files_exist=1
fi
done
if [ $INTERACTIVE -eq 1 ]; then
if [ $stale_dirs_exist -eq 1 -o $stale_files_exist -eq 1 ]; then
read -p "Exit to take care of them? (y/n)" yn
case $yn in
[Yy]* )
exit exit
;; ;;
[Nn]* ) [Nn]* )
;; ;;
* ) echo "Please answer yes or no.";; * ) echo "Please answer yes or no.";;
esac esac
fi
else else
echo "Not taking care of stale directories and files in auto mode." |tee -a /tmp/out.txt if [ $stale_dirs_exist -eq 1 -o $stale_files_exist -eq 1 ]; then
echo "If necessary, handle them in a manual commit."|tee -a /tmp/out.txt echo "Stale files or directories exist, but not taking care of stale directories and files in auto mode." |tee -a /tmp/out.txt
echo "If necessary, handle them in a manual commit."|tee -a /tmp/out.txt
fi
fi fi
# Delete known auto-generated content from trunk # Delete known auto-generated content from trunk
@ -168,81 +206,55 @@ elif [ `uname` == "Linux" ]; then
COPYOPTS='-au' COPYOPTS='-au'
fi fi
cp $COPYOPTS $GIT_DIR/target/staging/* . cp $COPYOPTS $GIT_DIR/target/staging/. .
# Look for things we need to fix up in svn # Look for things we need to fix up in svn
echo "Untracked files: svn add" echo "Untracked files: svn add"
svn status |grep '?' |sed -e "s/[[:space:]]//g"|cut -d '?' -f 2|while read i; do svn status |grep '?' |sed -e "s/[[:space:]]//g"|cut -d '?' -f 2|while read i; do
svn add $i svn add $i
echo "Added $i"
done done
echo "Locally deleted files: svn del" echo "Locally deleted files: svn del"
svn status |grep '!' |sed -e "s/[[:space:]]//g"|cut -d '!' -f 2|while read i; do svn status |grep '!' |sed -e "s/[[:space:]]//g"|cut -d '!' -f 2|while read i; do
svn del $i svn del $i
echo "Deleted $i"
done done
# Display the proposed changes. I filtered out svn_added=$(svn status | grep ^A|wc -l|awk {'print $1'})
# modified because there are so many. svn_removed=$(svn status | grep ^D|wc -l|awk {'print $1'})
if [ $INTERACTIVE ]; then svn_modified=$(svn status |grep ^M|wc -l|awk {'print $1'})
svn status |grep -v '^M'|less -P "Enter 'q' to exit the list."
else
echo "The following changes will be made to SVN."
svn status
fi
# Get current size of svn directory, for sanity checking before commit echo -e "Published site at $GIT_SHA. \n\
SVN_NEW_SIZE=`du -sm . |awk '{print $1}'` \n\
SVN_NEW_NUMFILES=`find . -type f |wc -l |awk '{print $1}'` Added $svn_added files. \n\
Removed $svn_removed files. \n\
Modified $svn_modified files." |tee /tmp/commit.txt
# Get difference between new and old size and number of files if [ $INTERACTIVE -eq 1 ]; then
# We don't care about negatives so remove the sign
SVN_SIZE_DIFF=`expr $SVN_NEW_SIZE - $SVN_OLD_SIZE|sed 's/-//g'`
SVN_NUM_DIFF=`expr $SVN_NEW_NUMFILES - $SVN_OLD_NUMFILES|sed 's/-//g'`
# The whole site is only 500 MB so a difference of 10 MB is huge
# In this case, we should abort because something is wrong
# Leaving this commented out for now until we get some benchmarks
if [ $SVN_SIZE_DIFF > 10 -o $SVN_NUM_DIFF > 50 ]; then
echo "This commit would cause the website to change sizes by \
$SVN_DIFF MB and $SVN_NUM_DIFF files. There is likely a problem.
Aborting."
exit 1
fi
if [ $INTERACTIVE ]; then
read -p "Commit changes? This will publish the website. (y/n)" yn read -p "Commit changes? This will publish the website. (y/n)" yn
case $yn in case $yn in
[Yy]* ) [Yy]* )
echo "Published website using script in interactive mode. \ echo "Commit message:"
This commit changed the size of the website by $SVN_SIZE_DIFF MB \ cat /tmp/commit.txt
and the number of files by $SVN_NUM_DIFF files." |tee commit.txt
cat /tmp/out.txt >> /tmp/commit.txt
svn commit -F /tmp/commit.txt svn commit -F /tmp/commit.txt
exit exit
;; ;;
[Nn]* ) [Nn]* )
read -p "Revert SVN changes? (y/n)" revert read -p "Revert SVN changes? (y/n)" revert
case $revert in case $revert in
[Yy]* ) [Yy]* )
svn revert -R . svn revert -R .
svn update svn update
exit exit
;; ;;
[Nn]* ) [Nn]* )
exit exit
;; ;;
esac esac
;; ;;
esac esac
else else
echo "Published website using script in auto mode. This commit \ echo "Commit message:"
changed the size of the website by $SVN_SIZE_DIFF MB and the number of files \ cat /tmp/commit.txt
by $SVN_NUM_DIFF files." |tee /tmp/commit.txt
cat /tmp/out.txt >> /tmp/commit.txt
svn commit -q -F /tmp/commit.txt svn commit -q -F /tmp/commit.txt
fi fi