HADOOP-15256. Create docker images for latest stable hadoop3 build. Contributed by Elek, Marton.
This commit is contained in:
Anu Engineer
commit
f309d08bb1
|
|
@ -0,0 +1 @@
|
|||
__pycache__
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM apache/hadoop-runner
|
||||
ARG HADOOP_URL=https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=hadoop/common/hadoop-3.0.0/hadoop-3.0.0.tar.gz
|
||||
WORKDIR /opt
|
||||
RUN sudo rm -rf /opt/hadoop && wget $HADOOP_URL -O hadoop.tar.gz && tar zxf hadoop.tar.gz && rm hadoop.tar.gz && mv hadoop* hadoop && rm -rf /opt/hadoop/share/doc
|
||||
WORKDIR /opt/hadoop
|
||||
ADD log4j.properties /opt/hadoop/etc/hadoop/log4j.properties
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
Please use the included docker-compose.yaml to test it:
|
||||
|
||||
```
|
||||
docker-compose build
|
||||
docker-compose up -d
|
||||
```
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
set -e
|
||||
mkdir -p build
|
||||
if [ ! -d "$DIR/build/apache-rat-0.12" ]; then
|
||||
wget "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz" -O "$DIR/build/apache-rat.tar.gz"
|
||||
cd $DIR/build
|
||||
tar zvxf apache-rat.tar.gz
|
||||
cd -
|
||||
fi
|
||||
java -jar $DIR/build/apache-rat-0.12/apache-rat-0.12.jar $DIR -e public -e apache-rat-0.12 -e .git -e .gitignore
|
||||
docker build -t apache/hadoop:3 .
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
CORE-SITE.XML_fs.default.name=hdfs://namenode
|
||||
CORE-SITE.XML_fs.defaultFS=hdfs://namenode
|
||||
HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020
|
||||
HDFS-SITE.XML_dfs.replication=1
|
||||
MAPRED-SITE.XML_mapreduce.framework.name=yarn
|
||||
YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager
|
||||
YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false
|
||||
YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600
|
||||
YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false
|
||||
YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=*
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=*
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings=
|
||||
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "2"
|
||||
services:
|
||||
namenode:
|
||||
build: .
|
||||
hostname: namenode
|
||||
command: ["hdfs", "namenode"]
|
||||
ports:
|
||||
- 9870:9870
|
||||
env_file:
|
||||
- ./config
|
||||
environment:
|
||||
ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
|
||||
datanode:
|
||||
build: .
|
||||
command: ["hdfs", "datanode"]
|
||||
env_file:
|
||||
- ./config
|
||||
resourcemanager:
|
||||
build: .
|
||||
hostname: resourcemanager
|
||||
command: ["yarn", "resourcemanager"]
|
||||
ports:
|
||||
- 8088:8088
|
||||
env_file:
|
||||
- ./config
|
||||
volumes:
|
||||
- ./test.sh:/opt/test.sh
|
||||
nodemanager:
|
||||
build: .
|
||||
command: ["yarn", "nodemanager"]
|
||||
env_file:
|
||||
- ./config
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
log4j.appender.stdout.layout: org.apache.log4j.PatternLayout
|
||||
log4j.rootLogger: INFO, stdout
|
||||
log4j.appender.stdout: org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.stdout.layout.ConversionPattern: %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
|
||||
Loading…
Reference in New Issue