Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-1019. Use apache/hadoop-runner image to test ozone secure cluster. Contributed by Xiaoyu Yao.

This commit is contained in:
Xiaoyu Yao 2019-02-26 16:41:09 +01:00 committed by Márton Elek
parent 41dc49c312
commit 6d097eb488
No known key found for this signature in database
GPG Key ID: D51EA8F00EE79B28
5 changed files with 242 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -34,6 +34,8 @@ RUN useradd --uid 1000 hadoop --gid 100 --home /opt/hadoop
RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
RUN chown hadoop /opt
ADD scripts /opt/
ADD scripts/krb5.conf /etc/
RUN yum install -y krb5-workstation
WORKDIR /opt/hadoop

43
scripts/envtoconf.py
View File

@ -16,22 +16,23 @@
# limitations under the License.
#
"""convert environment variables to config"""
import os
import re
from shutil import copyfile
import argparse
import sys
import transformation
class Simple:
class Simple(object):
"""Simple conversion"""
def __init__(self, args):
parser = argparse.ArgumentParser()
parser.add_argument("--destination", help="Destination directory", required=True)
self.args = parser.parse_args(args=args)
# copy the default files to file.raw in desitnation directory
# copy the default files to file.raw in destination directory
self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']
self.output_dir = self.args.destination
@ -39,16 +40,19 @@ class Simple:
self.configurables = {}
def destination_file_path(self, name, extension):
"""destination file path"""
return os.path.join(self.output_dir, "{}.{}".format(name, extension))
def write_env_var(self, name, extension, key, value):
"""Write environment variables"""
with open(self.destination_file_path(name, extension) + ".raw", "a") as myfile:
myfile.write("{}: {}\n".format(key, value))
def process_envs(self):
"""Process environment variables"""
for key in os.environ.keys():
p = re.compile("[_\\.]")
parts = p.split(key)
pattern = re.compile("[_\\.]")
parts = pattern.split(key)
extension = None
name = parts[0].lower()
if len(parts) > 1:
@ -57,46 +61,53 @@ class Simple:
if extension and "!" in extension:
splitted = extension.split("!")
extension = splitted[0]
format = splitted[1]
config_key = key[len(name) + len(extension) + len(format) + 3:].strip()
fmt = splitted[1]
config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
else:
format = extension
fmt = extension
if extension and extension in self.known_formats:
if name not in self.configurables.keys():
with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
myfile.write("")
self.configurables[name] = (extension, format)
self.configurables[name] = (extension, fmt)
self.write_env_var(name, extension, config_key, os.environ[key])
else:
for configurable_name in self.configurables.keys():
for configurable_name in self.configurables:
if key.lower().startswith(configurable_name.lower()):
self.write_env_var(configurable_name, self.configurables[configurable_name], key[len(configurable_name) + 1:], os.environ[key])
self.write_env_var(configurable_name,
self.configurables[configurable_name],
key[len(configurable_name) + 1:],
os.environ[key])
def transform(self):
for configurable_name in self.configurables.keys():
"""transform"""
for configurable_name in self.configurables:
name = configurable_name
extension, format = self.configurables[name]
extension, fmt = self.configurables[name]
destination_path = self.destination_file_path(name, extension)
with open(destination_path + ".raw", "r") as myfile:
content = myfile.read()
transformer_func = getattr(transformation, "to_" + format)
transformer_func = getattr(transformation, "to_" + fmt)
content = transformer_func(content)
with open(destination_path, "w") as myfile:
myfile.write(content)
def main(self):
"""main"""
# add the
self.process_envs()
# copy file.ext.raw to file.ext in the destination directory, and transform to the right format (eg. key: value ===> XML)
# copy file.ext.raw to file.ext in the destination directory, and
# transform to the right format (eg. key: value ===> XML)
self.transform()
def main():
"""main"""
Simple(sys.argv[1:]).main()

21
scripts/krb5.conf
View File

@ -1,11 +1,9 @@
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
@ -14,12 +12,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##
[logging]
default = FILE:/var/log/krb5libs.log
kdc = FILE:/var/log/krb5kdc.log
admin_server = FILE:/var/log/kadmind.log
default = FILE:/var/log/krb5libs.log
kdc = FILE:/var/log/krb5kdc.log
admin_server = FILE:/var/log/kadmind.log
[libdefaults]
dns_canonicalize_hostname = false

50
scripts/starter.sh
View File

@ -20,8 +20,6 @@ set -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
$DIR/envtoconf.py --destination /opt/hadoop/etc/hadoop
if [ -n "$SLEEP_SECONDS" ]; then
echo "Sleeping for $SLEEP_SECONDS seconds"
sleep $SLEEP_SECONDS
@ -58,17 +56,26 @@ if [ ! -z "$WAITFOR" ]; then
fi
fi
if [ -n "$KERBEROS_ENABLED" ]; then
echo "Setting up kerberos!!"
KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
if [ -n "$SLEEP_SECONDS" ]; then
echo "Sleeping for $(SLEEP_SECONDS) seconds"
sleep "$SLEEP_SECONDS"
fi
if [ -z "$KEYTAB_DIR"]; then
KEYTAB_DIR=/etc/security/keytabs
fi
while true
do
STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://$ISSUER_SERVER/keytab/test/test)
if [ $STATUS -eq 200 ]; then
set +e
STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)
set -e
if [ "$STATUS" -eq 200 ]; then
echo "Got 200, KDC service ready!!"
break
else
@ -77,26 +84,30 @@ if [ -n "$KERBEROS_ENABLED" ]; then
sleep 5
done
export HOST_NAME=`hostname -f`
HOST_NAME=$(hostname -f)
export HOST_NAME
for NAME in ${KERBEROS_KEYTABS}; do
echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $CONF_DIR/$NAME.keytab"
wget http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME -O $CONF_DIR/$NAME.keytab
echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $KEYTAB_DIR/$NAME.keytab"
wget "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$KEYTAB_DIR/$NAME.keytab"
klist -kt "$KEYTAB_DIR/$NAME.keytab"
KERBEROS_ENABLED=true
done
cat $DIR/krb5.conf | sed "s/SERVER/$KERBEROS_SERVER/g" | sudo tee /etc/krb5.conf
sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf
fi
#To avoid docker volume permission problems
sudo chmod o+rwx /data
"$DIR"/envtoconf.py --destination /opt/hadoop/etc/hadoop
if [ -n "$ENSURE_NAMENODE_DIR" ]; then
CLUSTERID_OPTS=""
if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; then
CLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"
fi
if [ ! -d "$ENSURE_NAMENODE_DIR" ]; then
/opt/hadoop/bin/hdfs namenode -format -force $CLUSTERID_OPTS
/opt/hadoop/bin/hdfs namenode -format -force "$CLUSTERID_OPTS"
fi
fi
@ -113,7 +124,6 @@ if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
fi
fi
if [ -n "$ENSURE_OM_INITIALIZED" ]; then
if [ ! -f "$ENSURE_OM_INITIALIZED" ]; then
# Improve om and scm start up options
@ -121,21 +131,6 @@ if [ -n "$ENSURE_OM_INITIALIZED" ]; then
fi
fi
# The KSM initialization block will go away eventually once
# we have completed renaming KSM to OzoneManager (OM).
#
if [ -n "$ENSURE_KSM_INITIALIZED" ]; then
if [ ! -f "$ENSURE_KSM_INITIALIZED" ]; then
# To make sure SCM is running in dockerized environment we will sleep
# Could be removed after HDFS-13203
echo "Waiting 15 seconds for SCM startup"
sleep 15
/opt/hadoop/bin/ozone ksm -createObjectStore
fi
fi
# Supports byteman script to instrument hadoop process with byteman script
#
#
@ -158,5 +153,4 @@ if [ -n "$BYTEMAN_SCRIPT" ] || [ -n "$BYTEMAN_SCRIPT_URL" ]; then
echo "Process is instrumented with adding $AGENT_STRING to HADOOP_OPTS"
fi
$@
"$@"

53
scripts/transformation.py
View File

@ -16,13 +16,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""This module transform properties into different format"""
def render_yaml(yaml_root, prefix=""):
"""render yaml"""
result = ""
if isinstance(yaml_root, dict):
if len(prefix)>0:
result +="\n"
if prefix:
result += "\n"
for key in yaml_root:
result += "{}{}: {}".format(prefix, key, render_yaml(yaml_root[key], prefix + " "))
result += "{}{}: {}".format(prefix, key, render_yaml(
yaml_root[key], prefix + " "))
elif isinstance(yaml_root, list):
result += "\n"
for item in yaml_root:
@ -33,6 +37,7 @@ def render_yaml(yaml_root, prefix=""):
def to_yaml(content):
"""transform to yaml"""
props = process_properties(content)
keys = props.keys()
@ -41,7 +46,7 @@ def to_yaml(content):
parts = key.split(".")
node = yaml_props
prev_part = None
parent_node = None
parent_node = {}
for part in parts[:-1]:
if part.isdigit():
if isinstance(node, dict):
@ -67,54 +72,64 @@ def to_yaml(content):
return render_yaml(yaml_props)
def to_yml(content):
"""transform to yml"""
return to_yaml(content)
def to_properties(content):
"""transform to properties"""
result = ""
props = process_properties(content)
for key in props.keys():
result += "{}: {}\n".format(key, props[key])
for key, val in props.items():
result += "{}: {}\n".format(key, val)
return result
def to_env(content):
"""transform to environment variables"""
result = ""
props = process_properties(content)
for key in props.keys():
result += "{}={}\n".format(key, props[key])
for key, val in props:
result += "{}={}\n".format(key, val)
return result
def to_sh(content):
"""transform to shell"""
result = ""
props = process_properties(content)
for key in props.keys():
result += "export {}=\"{}\"\n".format(key, props[key])
for key, val in props:
result += "export {}=\"{}\"\n".format(key, val)
return result
def to_cfg(content):
"""transform to config"""
result = ""
props = process_properties(content)
for key in props.keys():
result += "{}={}\n".format(key, props[key])
for key, val in props:
result += "{}={}\n".format(key, val)
return result
def to_conf(content):
"""transform to configuration"""
result = ""
props = process_properties(content)
for key in props.keys():
result += "export {}={}\n".format(key, props[key])
for key, val in props:
result += "export {}={}\n".format(key, val)
return result
def to_xml(content):
"""transform to xml"""
result = "<configuration>\n"
props = process_properties(content)
for key in props.keys():
result += "<property><name>{0}</name><value>{1}</value></property>\n".format(key, props[key])
for key in props:
result += "<property><name>{0}</name><value>{1}</value></property>\n". \
format(key, props[key])
result += "</configuration>"
return result
@ -125,9 +140,9 @@ def process_properties(content, sep=': ', comment_char='#'):
"""
props = {}
for line in content.split("\n"):
l = line.strip()
if l and not l.startswith(comment_char):
key_value = l.split(sep)
sline = line.strip()
if sline and not sline.startswith(comment_char):
key_value = sline.split(sep)
key = key_value[0].strip()
value = sep.join(key_value[1:]).strip().strip('"')
props[key] = value