From 524593ee84e4b993a4f42ae573b09024b6b76c8f Mon Sep 17 00:00:00 2001 From: cnauroth Date: Fri, 24 Apr 2015 13:05:18 -0700 Subject: [PATCH] HADOOP-11843. Make setting up the build environment easier. Contributed by Niels Basjes. (cherry picked from commit 80935268f5fdd358070c6b68f89e8bd699785c54) Conflicts: hadoop-common-project/hadoop-common/CHANGES.txt --- BUILDING.txt | 39 +++++- dev-support/docker/Dockerfile | 67 ++++++++++ dev-support/docker/hadoop_env_checks.sh | 118 ++++++++++++++++++ .../hadoop-common/CHANGES.txt | 3 + start-build-env.sh | 50 ++++++++ 5 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 dev-support/docker/Dockerfile create mode 100644 dev-support/docker/hadoop_env_checks.sh create mode 100644 start-build-env.sh diff --git a/BUILDING.txt b/BUILDING.txt index b30b30e1586..b61b11ef683 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -15,6 +15,43 @@ Requirements: * Linux FUSE (Filesystem in Userspace) version 2.6 or above ( if compiling fuse_dfs ) * Internet connection for first build (to fetch all Maven and Hadoop dependencies) +---------------------------------------------------------------------------------- +The easiest way to get an environment with all the appropriate tools is by means +of the provided Docker config. +This requires a recent version of docker ( 1.4.1 and higher are known to work ). + +On Linux: + Install Docker and run this command: + + $ ./start-build-env.sh + +On Mac: + First make sure Homebrew has been installed ( http://brew.sh/ ) + $ brew install docker boot2docker + $ boot2docker init -m 4096 + $ boot2docker start + $ $(boot2docker shellinit) + $ ./start-build-env.sh + +The prompt which is then presented is located at a mounted version of the source tree +and all required tools for testing and building have been installed and configured. + +Note that from within this docker environment you ONLY have access to the Hadoop source +tree from where you started. So if you need to run + dev-support/test-patch.sh /path/to/my.patch +then the patch must be placed inside the hadoop source tree. + +Known issues: +- On Mac with Boot2Docker the performance on the mounted directory is currently extremely slow. + This is a known problem related to boot2docker on the Mac. + See: + https://github.com/boot2docker/boot2docker/issues/593 + This issue has been resolved as a duplicate, and they point to a new feature for utilizing NFS mounts + as the proposed solution: + https://github.com/boot2docker/boot2docker/issues/64 + An alternative solution to this problem is when you install Linux native inside a virtual machine + and run your IDE and Docker etc in side that VM. + ---------------------------------------------------------------------------------- Installing required packages for clean install of Ubuntu 14.04 LTS Desktop: @@ -29,7 +66,7 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop: * Native libraries $ sudo apt-get -y install build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev * ProtocolBuffer 2.5.0 (required) - $ sudo apt-get -y install libprotobuf-dev protobuf-compiler + $ sudo apt-get -y install protobuf-compiler Optional packages: diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile new file mode 100644 index 00000000000..81296dc46db --- /dev/null +++ b/dev-support/docker/Dockerfile @@ -0,0 +1,67 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +# FROM dockerfile/java:openjdk-7-jdk +FROM dockerfile/java:oracle-java7 + +WORKDIR /root + +# Install dependencies from packages +RUN apt-get update && apt-get install --no-install-recommends -y \ + git curl ant make maven \ + cmake gcc g++ protobuf-compiler \ + build-essential libtool \ + zlib1g-dev pkg-config libssl-dev \ + snappy libsnappy-dev \ + bzip2 libbz2-dev \ + libjansson-dev \ + fuse libfuse-dev \ + libcurl4-openssl-dev \ + python python2.7 + +# Install Forrest +RUN mkdir -p /usr/local/apache-forrest ; \ + curl -O http://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz ; \ + tar xzf *forrest* --strip-components 1 -C /usr/local/apache-forrest ; \ + echo 'forrest.home=/usr/local/apache-forrest' > build.properties + +# Install findbugs +RUN mkdir -p /opt/findbugs && \ + wget http://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ + -O /opt/findbugs.tar.gz && \ + tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs +ENV FINDBUGS_HOME /opt/findbugs + +# Install shellcheck +RUN apt-get install -y cabal-install +RUN cabal update && cabal install shellcheck --global + +# Fixing the Apache commons / Maven dependency problem under Ubuntu: +# See http://wiki.apache.org/commons/VfsProblems +RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar . + +# Avoid out of memory errors in builds +ENV MAVEN_OPTS -Xms256m -Xmx512m + +# Add a welcome message and environment checks. +ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh +RUN chmod 755 /root/hadoop_env_checks.sh +RUN echo '~/hadoop_env_checks.sh' >> /root/.bashrc + diff --git a/dev-support/docker/hadoop_env_checks.sh b/dev-support/docker/hadoop_env_checks.sh new file mode 100644 index 00000000000..910c802291a --- /dev/null +++ b/dev-support/docker/hadoop_env_checks.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------------------------------------------------------- +function showWelcome { +cat <