diff --git a/dev-support/Dockerfile b/dev-support/Dockerfile index 5d79988664e..389f24f0027 100644 --- a/dev-support/Dockerfile +++ b/dev-support/Dockerfile @@ -22,8 +22,14 @@ # dev-support/flaky-tests/flaky-reporting.Jenkinsfile FROM ubuntu:18.04 -ADD . /hbase/dev-support +COPY . /hbase/dev-support -RUN apt-get -y update \ - && apt-get -y install curl python-pip \ - && pip install -r /hbase/dev-support/python-requirements.txt +RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \ + && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \ + curl=7.58.0-2ubuntu3.8 \ + python2.7=2.7.17-1~18.04 \ + python-pip=9.0.1-2.3~ubuntu1.18.04.1 \ + python-setuptools=39.0.1-2 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +RUN pip install -r /hbase/dev-support/python-requirements.txt diff --git a/dev-support/create-release/hbase-rm/Dockerfile b/dev-support/create-release/hbase-rm/Dockerfile index 092b4183add..c78aa219f0b 100644 --- a/dev-support/create-release/hbase-rm/Dockerfile +++ b/dev-support/create-release/hbase-rm/Dockerfile @@ -15,28 +15,34 @@ # limitations under the License. # -# Image for building HBase releases. Based on Ubuntu 16.04. +# Image for building HBase releases. Based on Ubuntu 18.04. # # Includes: # * Java 8 FROM ubuntu:18.04 -# These arguments are just for reuse and not really meant to be customized. -ARG APT_INSTALL="apt-get install --no-install-recommends -y" - # Install extra needed repos and refresh. # # This is all in a single "RUN" command so that if anything changes, "apt update" is run to fetch # the most current package versions (instead of potentially using old versions cached by docker). -RUN apt-get clean && \ - apt-get update && \ - # Install openjdk 8. - $APT_INSTALL openjdk-8-jdk && \ - update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java && \ - # Install build / source control tools - $APT_INSTALL curl gnupg python-pip wget git maven subversion lsof \ - libcurl4-openssl-dev libxml2-dev && \ - pip install python-dateutil +RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \ + && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \ + curl=7.58.0-2ubuntu3.8 \ + git=1:2.17.1-1ubuntu0.5 \ + gnupg=2.2.4-1ubuntu1.2 \ + libcurl4-openssl-dev=7.58.0-2ubuntu3.8 \ + libxml2-dev=2.9.4+dfsg1-6.1ubuntu1.3 \ + lsof=4.89+dfsg-0.1 \ + maven=3.6.0-1~18.04.1 \ + openjdk-8-jdk=8u242-b08-0ubuntu3~18.04 \ + python-pip=9.0.1-2.3~ubuntu1.18.04.1 \ + subversion=1.9.7-4ubuntu1 \ + wget=1.19.4-1ubuntu2.2 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \ + && pip install \ + python-dateutil==2.8.1 WORKDIR /opt/hbase-rm/output diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 8f2e10440f9..3c1423d386c 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -152,6 +152,9 @@ RUN tar xzf /tmp/maven.tar.gz -C /opt && \ # ensure JVMs are available under `/usr/lib/jvm` and prefix each installation # as `java-` so as to conform with Yetus's assumptions. # +# when updating java or maven versions here, consider also updating +# `dev-support/hbase_docker/Dockerfile` as well. +# # hadolint ignore=DL3010 COPY --from=OPENJDK7_DOWNLOAD_IMAGE /tmp/zuluopenjdk7.tar.gz /tmp/zuluopenjdk7.tar.gz diff --git a/dev-support/hbase_docker/Dockerfile b/dev-support/hbase_docker/Dockerfile index 1a5dfa36976..7c7111c4570 100644 --- a/dev-support/hbase_docker/Dockerfile +++ b/dev-support/hbase_docker/Dockerfile @@ -14,37 +14,78 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:14.04 +FROM ubuntu:18.04 AS BASE_IMAGE +SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# Install Git, which is missing from the Ubuntu base images. -RUN apt-get update && apt-get install -y git +# hadolint ignore=DL3009 +RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \ + DEBIAN_FRONTEND=noninteractive apt-get -qq install --no-install-recommends -y \ + ca-certificates=20180409 \ + curl=7.58.0-2ubuntu3.8 \ + locales=2.27-3ubuntu1 -# Add the dependencies from the hbase_docker folder and delete ones we don't need. -WORKDIR /root -ADD . /root -RUN find . -not -name "*tar.gz" -delete +RUN locale-gen en_US.UTF-8 +ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 -# Install Java. -RUN mkdir -p /usr/java -RUN tar xzf *jdk* --strip-components 1 -C /usr/java -ENV JAVA_HOME /usr/java +FROM BASE_IMAGE AS MAVEN_DOWNLOAD_IMAGE +ENV MAVEN_VERSION='3.5.4' +ENV MAVEN_URL "https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz" +ENV MAVEN_SHA256 'ce50b1c91364cb77efe3776f756a6d92b76d9038b0a0782f7d53acf1e997a14d' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl --location --fail --silent --show-error --output /tmp/maven.tar.gz "${MAVEN_URL}" && \ + echo "${MAVEN_SHA256} */tmp/maven.tar.gz" | sha256sum -c - -# Install Maven. -RUN mkdir -p /usr/local/apache-maven -RUN tar xzf *maven* --strip-components 1 -C /usr/local/apache-maven -ENV MAVEN_HOME /usr/local/apache-maven +FROM BASE_IMAGE AS OPENJDK8_DOWNLOAD_IMAGE +ENV OPENJDK8_URL 'https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u232-b09/OpenJDK8U-jdk_x64_linux_hotspot_8u232b09.tar.gz' +ENV OPENJDK8_SHA256 '7b7884f2eb2ba2d47f4c0bf3bb1a2a95b73a3a7734bd47ebf9798483a7bcc423' +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk8.tar.gz "${OPENJDK8_URL}" && \ + echo "${OPENJDK8_SHA256} */tmp/adoptopenjdk8.tar.gz" | sha256sum -c - -# Add Java and Maven to the path. -ENV PATH /usr/java/bin:/usr/local/apache-maven/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +FROM BASE_IMAGE +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN DEBIAN_FRONTEND=noninteractive apt-get -qq install --no-install-recommends -y \ + git=1:2.17.1-1ubuntu0.5 \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# +# when updating java or maven versions here, consider also updating +# `dev-support/docker/Dockerfile` as well. +# + +# hadolint ignore=DL3010 +COPY --from=MAVEN_DOWNLOAD_IMAGE /tmp/maven.tar.gz /tmp/maven.tar.gz +RUN tar xzf /tmp/maven.tar.gz -C /opt && \ + ln -s "/opt/$(dirname "$(tar -tf /tmp/maven.tar.gz | head -n1)")" /opt/maven && \ + rm /tmp/maven.tar.gz + +# hadolint ignore=DL3010 +COPY --from=OPENJDK8_DOWNLOAD_IMAGE /tmp/adoptopenjdk8.tar.gz /tmp/adoptopenjdk8.tar.gz +RUN mkdir -p /usr/lib/jvm && \ + tar xzf /tmp/adoptopenjdk8.tar.gz -C /usr/lib/jvm && \ + ln -s "/usr/lib/jvm/$(basename "$(tar -tf /tmp/adoptopenjdk8.tar.gz | head -n1)")" /usr/lib/jvm/java-8-adoptopenjdk && \ + ln -s /usr/lib/jvm/java-8-adoptopenjdk /usr/lib/jvm/java-8 && \ + rm /tmp/adoptopenjdk8.tar.gz + +ENV MAVEN_HOME '/opt/maven' +ENV JAVA_HOME '/usr/lib/jvm/java-8' +ENV PATH '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +ENV PATH "${JAVA_HOME}/bin:${MAVEN_HOME}/bin:${PATH}" # Pull down HBase and build it into /root/hbase-bin. +WORKDIR /root RUN git clone https://gitbox.apache.org/repos/asf/hbase.git -b master RUN mvn clean install -DskipTests assembly:single -f ./hbase/pom.xml RUN mkdir -p hbase-bin -RUN tar xzf /root/hbase/hbase-assembly/target/*tar.gz --strip-components 1 -C /root/hbase-bin +RUN find /root/hbase/hbase-assembly/target -iname '*.tar.gz' -not -iname '*client*' \ + | head -n 1 \ + | xargs -I{} tar xzf {} --strip-components 1 -C /root/hbase-bin # Set HBASE_HOME, add it to the path, and start HBase. ENV HBASE_HOME /root/hbase-bin -ENV PATH /root/hbase-bin/bin:/usr/java/bin:/usr/local/apache-maven/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV PATH "/root/hbase-bin/bin:${PATH}" CMD ["/bin/bash", "-c", "start-hbase.sh; hbase shell"] diff --git a/dev-support/hbase_docker/README.md b/dev-support/hbase_docker/README.md index e42d20153a1..1750e809cc7 100644 --- a/dev-support/hbase_docker/README.md +++ b/dev-support/hbase_docker/README.md @@ -29,30 +29,15 @@ and launch the HBase shell when run. ## Usage -1. Download x64 .tar.gz files of the Oracle JDK and Apache Maven and place them - in this folder (i.e. both tarballs must be in the same folder as the - Dockerfile). Also note that the Dockerfile will properly pick up the tarballs - as long as the JDK file has "jdk" in its name and the Maven file contains - "maven". As an example, while developing this Dockerfile, my working directory - looked like this: - - ``` - $ ls -lh - total 145848 - -rw-r--r-- 1 root root 6956162 Sep 3 15:48 apache-maven-3.2.3-bin.tar.gz - -rw-r--r-- 1 root root 2072 Sep 3 15:48 Dockerfile - -rw-r--r-- 1 root root 142376665 Sep 3 15:48 jdk-7u67-linux-x64.tar.gz - -rw-r--r-- 1 root root 1844 Sep 3 15:56 README.md - ``` -2. Ensure that you have a recent version of Docker installed from +1. Ensure that you have a recent version of Docker installed from [docker.io](http://www.docker.io). -3. Set this folder as your working directory. -4. Type `docker build -t hbase_docker .` to build a Docker image called **hbase_docker**. +1. Set this folder as your working directory. +1. Type `docker build -t hbase_docker .` to build a Docker image called **hbase_docker**. This may take 10 minutes or more the first time you run the command since it will create a Maven repository inside the image as well as checkout the master branch of HBase. -5. When this completes successfully, you can run `docker run -it hbase_docker` +1. When this completes successfully, you can run `docker run -it hbase_docker` to access an HBase shell running inside of a container created from the **hbase_docker** image. Alternatively, you can type `docker run -it hbase_docker bash` to start a container without a running HMaster. Within this environment, - HBase is built in /root/hbase-bin. + HBase is built in `/root/hbase-bin`.