HADOOP-14671. Upgrade Apache Yetus to 0.8.0

Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
(cherry picked from commit 4b8c2b1c1c)

 Conflicts:
	dev-support/docker/Dockerfile
	hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/RELEASENOTES.2.10.0.md
	hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.2/CHANGELOG.3.0.2.md
	hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.2/RELEASENOTES.3.1.2.md
This commit is contained in:
Allen Wittenauer 2018-09-01 16:05:33 -07:00 committed by Akira Ajisaka
parent 34e507cb8c
commit 1bcde89983
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
183 changed files with 4384 additions and 1329 deletions

View File

@ -608,8 +608,8 @@ function makearelease
#shellcheck disable=SC2038
find . -name rat.txt | xargs -I% cat % > "${ARTIFACTS_DIR}/hadoop-${HADOOP_VERSION}${RC_LABEL}-rat.txt"
# Stage CHANGES and RELEASENOTES files
for i in CHANGES RELEASENOTES; do
# Stage CHANGELOG and RELEASENOTES files
for i in CHANGELOG RELEASENOTES; do
run cp -p \
"${BASEDIR}/hadoop-common-project/hadoop-common/src/site/markdown/release/${HADOOP_VERSION}"/${i}*.md \
"${ARTIFACTS_DIR}/${i}.md"

View File

@ -77,7 +77,7 @@ WANTED="$1"
shift
ARGV=("$@")
HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.4.0}
HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.8.0}
BIN=$(yetus_abs "${BASH_SOURCE-$0}")
BINDIR=$(dirname "${BIN}")

View File

@ -22,6 +22,8 @@ FROM ubuntu:xenial
WORKDIR /root
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
#####
# Disable suggests/recommends
#####
@ -38,118 +40,125 @@ ENV DEBCONF_TERSE true
# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
# Ubuntu Java. See Java section below!
######
RUN apt-get -q update && apt-get -q install -y \
apt-utils \
build-essential \
bzip2 \
curl \
doxygen \
fuse \
g++ \
gcc \
git \
gnupg-agent \
libbz2-dev \
libcurl4-openssl-dev \
libfuse-dev \
libprotobuf-dev \
libprotoc-dev \
libsnappy-dev \
libssl-dev \
libtool \
libzstd1-dev \
locales \
make \
pinentry-curses \
pkg-config \
python \
python2.7 \
python-pip \
python-pkg-resources \
python-setuptools \
python-wheel \
rsync \
software-properties-common \
snappy \
sudo \
zlib1g-dev
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends \
apt-utils \
build-essential \
bzip2 \
clang \
curl \
doxygen \
fuse \
g++ \
gcc \
git \
gnupg-agent \
libbz2-dev \
libcurl4-openssl-dev \
libfuse-dev \
libprotobuf-dev \
libprotoc-dev \
libsasl2-dev \
libsnappy-dev \
libssl-dev \
libtool \
locales \
make \
pinentry-curses \
pkg-config \
python \
python2.7 \
python-pip \
python-pkg-resources \
python-setuptools \
python-wheel \
rsync \
software-properties-common \
snappy \
sudo \
valgrind \
zlib1g-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
#######
# OpenJDK 8
#######
RUN apt-get -q install -y openjdk-8-jdk
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
#######
# OpenJDK 9
# w/workaround for
# https://bugs.launchpad.net/ubuntu/+source/openjdk-9/+bug/1593191
#######
RUN apt-get -o Dpkg::Options::="--force-overwrite" \
-q install -y \
openjdk-9-jdk-headless
#######
# Set default Java
#######
#
# By default, OpenJDK sets the default Java to the highest version.
# We want the opposite, soooooo....
#
RUN update-java-alternatives --set java-1.8.0-openjdk-amd64
RUN update-alternatives --get-selections | grep -i jdk | \
while read line; do \
alternative=$(echo $line | awk '{print $1}'); \
path=$(echo $line | awk '{print $3}'); \
newpath=$(echo $path | sed -e 's/java-9/java-8/'); \
update-alternatives --set $alternative $newpath; \
done
######
# Install cmake 3.1.0 (3.5.1 ships with Xenial)
######
RUN mkdir -p /opt/cmake && \
curl -L -s -S \
RUN mkdir -p /opt/cmake \
&& curl -L -s -S \
https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \
-o /opt/cmake.tar.gz && \
tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake
-o /opt/cmake.tar.gz \
&& tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake
ENV CMAKE_HOME /opt/cmake
ENV PATH "${PATH}:/opt/cmake/bin"
######
# Install Google Protobuf 2.5.0 (2.6.0 ships with Xenial)
######
RUN mkdir -p /opt/protobuf-src && \
curl -L -s -S \
# hadolint ignore=DL3003
RUN mkdir -p /opt/protobuf-src \
&& curl -L -s -S \
https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
-o /opt/protobuf.tar.gz && \
tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src
RUN cd /opt/protobuf-src && ./configure --prefix=/opt/protobuf && make install
-o /opt/protobuf.tar.gz \
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
&& cd /opt/protobuf-src \
&& ./configure --prefix=/opt/protobuf \
&& make install \
&& cd /root \
&& rm -rf /opt/protobuf-src
ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin"
######
# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
######
RUN apt-get -q update && apt-get -q install -y maven
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends maven \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ENV MAVEN_HOME /usr
######
# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
# Ant is needed for findbugs
######
RUN apt-get -q update && apt-get -q install -y findbugs ant
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends findbugs ant \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ENV FINDBUGS_HOME /usr
####
# Install shellcheck (0.4.6, the latest as of 2017-09-26)
####
RUN add-apt-repository -y ppa:hvr/ghc
RUN apt-get -q update && apt-get -q install -y shellcheck ghc-8.0.2
# hadolint ignore=DL3008
RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2 \
&& apt-get -q update \
&& apt-get -q install -y --no-install-recommends shellcheck \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
####
# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
####
RUN apt-get -q update && apt-get -q install -y bats
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends bats \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
####
# Install pylint at fixed version (2.0.0 removed python2 support)
@ -163,17 +172,30 @@ RUN pip2 install \
####
# Install dateutil.parser
####
RUN pip2 install python-dateutil
RUN pip2 install python-dateutil==2.7.3
###
# Install node.js 5.x for web UI framework
# (instead of 4.2.6 shipped with Xenial)
###
RUN curl -sL https://deb.nodesource.com/setup_5.x | bash - && \
apt-get install -y nodejs && \
npm install -g npm@5.10.0 && \
npm install -g bower@1.7.7 && \
npm install -g ember-cli@1.13.14
# hadolint ignore=DL3008, DL3016
RUN apt-get -q update \
&& apt-get install -y --no-install-recommends nodejs npm \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& ln -s /usr/bin/nodejs /usr/bin/node \
&& npm install npm@latest -g \
&& npm install -g jshint
###
# Install hadolint
####
RUN curl -L -s -S \
https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
-o /bin/hadolint \
&& chmod a+rx /bin/hadolint \
&& shasum -a 512 /bin/hadolint | \
awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'
###
# Avoid out of memory errors in builds
@ -189,8 +211,13 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true
# YETUS CUT HERE
###
# Add a welcome message and environment checks.
ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh
RUN chmod 755 /root/hadoop_env_checks.sh
RUN echo '~/hadoop_env_checks.sh' >> /root/.bashrc
# Hugo static website generator (for new hadoop site and Ozone docs)
RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.30.2/hugo_0.30.2_Linux-64bit.deb \
&& dpkg --install hugo.deb \
&& rm hugo.deb
# Add a welcome message and environment checks.
COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
RUN chmod 755 /root/hadoop_env_checks.sh
# hadolint ignore=SC2016
RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc

View File

@ -1042,6 +1042,8 @@
<argument>--projecttitle</argument>
<argument>Apache Hadoop</argument>
<argument>--usetoday</argument>
<argument>--fileversions</argument>
<argument>--dirversions</argument>
<argument>--version</argument>
<argument>${project.version}</argument>
</arguments>

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 0.15.4 - Unreleased (as of 2017-08-28)
## Release 0.15.4 - Unreleased (as of 2018-09-01)

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 0.17.3 - Unreleased (as of 2017-08-28)
## Release 0.17.3 - Unreleased (as of 2018-09-01)

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 0.18.4 - Unreleased (as of 2017-08-28)
## Release 0.18.4 - Unreleased (as of 2018-09-01)

View File

@ -84,7 +84,7 @@
| [HADOOP-5921](https://issues.apache.org/jira/browse/HADOOP-5921) | JobTracker does not come up because of NotReplicatedYetException | Major | . | Amareshwari Sriramadasu | Amar Kamat |
| [HADOOP-6017](https://issues.apache.org/jira/browse/HADOOP-6017) | NameNode and SecondaryNameNode fail to restart because of abnormal filenames. | Blocker | . | Raghu Angadi | Tsz Wo Nicholas Sze |
| [HADOOP-5920](https://issues.apache.org/jira/browse/HADOOP-5920) | TestJobHistory fails some times. | Major | . | Amareshwari Sriramadasu | Amar Kamat |
| [HDFS-26](https://issues.apache.org/jira/browse/HDFS-26) | HADOOP-5862 for version .20 (Namespace quota exceeded message unclear) | Major | . | Boris Shkolnik | Boris Shkolnik |
| [HDFS-26](https://issues.apache.org/jira/browse/HDFS-26) | HADOOP-5862 for version .20 (Namespace quota exceeded message unclear) | Major | . | Boris Shkolnik | Boris Shkolnik |
| [HDFS-438](https://issues.apache.org/jira/browse/HDFS-438) | Improve help message for quotas | Minor | . | Raghu Angadi | Raghu Angadi |
| [MAPREDUCE-2](https://issues.apache.org/jira/browse/MAPREDUCE-2) | ArrayOutOfIndex error in KeyFieldBasedPartitioner on empty key | Major | . | Amar Kamat | Amar Kamat |
| [MAPREDUCE-130](https://issues.apache.org/jira/browse/MAPREDUCE-130) | Delete the jobconf copy from the log directory of the JobTracker when the job is retired | Major | . | Devaraj Das | Amar Kamat |

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 0.20.203.1 - Unreleased (as of 2017-08-28)
## Release 0.20.203.1 - Unreleased (as of 2018-09-01)

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 0.20.3 - Unreleased (as of 2017-08-28)
## Release 0.20.3 - Unreleased (as of 2018-09-01)
### INCOMPATIBLE CHANGES:

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 0.21.1 - Unreleased (as of 2017-08-28)
## Release 0.21.1 - Unreleased (as of 2018-09-01)
### INCOMPATIBLE CHANGES:

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 0.22.1 - Unreleased (as of 2017-08-28)
## Release 0.22.1 - Unreleased (as of 2018-09-01)
### INCOMPATIBLE CHANGES:

View File

@ -97,6 +97,7 @@
| [HDFS-5557](https://issues.apache.org/jira/browse/HDFS-5557) | Write pipeline recovery for the last packet in the block may cause rejection of valid replicas | Critical | . | Kihwal Lee | Kihwal Lee |
| [HDFS-5526](https://issues.apache.org/jira/browse/HDFS-5526) | Datanode cannot roll back to previous layout version | Blocker | datanode | Tsz Wo Nicholas Sze | Kihwal Lee |
| [HDFS-5558](https://issues.apache.org/jira/browse/HDFS-5558) | LeaseManager monitor thread can crash if the last block is complete but another block is not. | Major | . | Kihwal Lee | Kihwal Lee |
| [HDFS-5357](https://issues.apache.org/jira/browse/HDFS-5357) | TestFileSystemAccessService failures in JDK7 | Major | . | Robert Parker | Robert Parker |
### TESTS:

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 0.23.2 - Unreleased (as of 2017-08-28)
## Release 0.23.2 - Unreleased (as of 2018-09-01)
### INCOMPATIBLE CHANGES:

View File

@ -35,7 +35,7 @@
| [MAPREDUCE-4845](https://issues.apache.org/jira/browse/MAPREDUCE-4845) | ClusterStatus.getMaxMemory() and getUsedMemory() exist in MR1 but not MR2 | Major | client | Sandy Ryza | Sandy Ryza |
| [YARN-285](https://issues.apache.org/jira/browse/YARN-285) | RM should be able to provide a tracking link for apps that have already been purged | Major | . | Derek Dagit | Derek Dagit |
| [MAPREDUCE-4899](https://issues.apache.org/jira/browse/MAPREDUCE-4899) | Provide a plugin to the Yarn Web App Proxy to generate tracking links for M/R appllications given the ID | Major | . | Derek Dagit | Derek Dagit |
| [MAPREDUCE-4810](https://issues.apache.org/jira/browse/MAPREDUCE-4810) | Add admin command options for ApplicationMaster | Minor | applicationmaster | Jason Lowe | Jerry Chen |
| [MAPREDUCE-4810](https://issues.apache.org/jira/browse/MAPREDUCE-4810) | Add admin command options for ApplicationMaster | Minor | applicationmaster | Jason Lowe | Haifeng Chen |
### BUG FIXES:
@ -84,7 +84,7 @@
| [HADOOP-9181](https://issues.apache.org/jira/browse/HADOOP-9181) | Set daemon flag for HttpServer's QueuedThreadPool | Major | . | Liang Xie | Liang Xie |
| [YARN-320](https://issues.apache.org/jira/browse/YARN-320) | RM should always be able to renew its own tokens | Blocker | resourcemanager | Daryn Sharp | Daryn Sharp |
| [YARN-325](https://issues.apache.org/jira/browse/YARN-325) | RM CapacityScheduler can deadlock when getQueueInfo() is called and a container is completing | Blocker | capacityscheduler | Jason Lowe | Arun C Murthy |
| [MAPREDUCE-4848](https://issues.apache.org/jira/browse/MAPREDUCE-4848) | TaskAttemptContext cast error during AM recovery | Major | mr-am | Jason Lowe | Jerry Chen |
| [MAPREDUCE-4848](https://issues.apache.org/jira/browse/MAPREDUCE-4848) | TaskAttemptContext cast error during AM recovery | Major | mr-am | Jason Lowe | Haifeng Chen |
| [HADOOP-9097](https://issues.apache.org/jira/browse/HADOOP-9097) | Maven RAT plugin is not checking all source files | Critical | build | Tom White | Thomas Graves |
| [HDFS-4385](https://issues.apache.org/jira/browse/HDFS-4385) | Maven RAT plugin is not checking all source files | Critical | build | Thomas Graves | Thomas Graves |
| [MAPREDUCE-4934](https://issues.apache.org/jira/browse/MAPREDUCE-4934) | Maven RAT plugin is not checking all source files | Critical | build | Thomas Graves | Thomas Graves |

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 0.24.0 - Unreleased (as of 2017-08-28)
## Release 0.24.0 - Unreleased (as of 2018-09-02)
### INCOMPATIBLE CHANGES:

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 1.1.3 - Unreleased (as of 2017-08-28)
## Release 1.1.3 - Unreleased (as of 2018-09-02)

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 1.2.2 - Unreleased (as of 2017-08-28)
## Release 1.2.2 - Unreleased (as of 2018-09-02)

View File

@ -18,7 +18,7 @@
-->
# Apache Hadoop Changelog
## Release 1.3.0 - Unreleased (as of 2017-08-28)
## Release 1.3.0 - Unreleased (as of 2018-09-02)
### INCOMPATIBLE CHANGES:

Some files were not shown because too many files have changed in this diff Show More