From 559cb99bf69f9af4b101bd3f0c1d6d5f99f02a6f Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Tue, 4 Dec 2018 14:13:39 -0800 Subject: [PATCH] Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)" This reverts commit 920401b2966c2dcd7cd1b3dfd7a764f08412323f. --- .../resources/assemblies/hadoop-yarn-dist.xml | 8 - .../installation/install.conf | 74 --- .../installation/install.sh | 116 ----- .../package/calico/calico-node.service | 50 -- .../installation/package/calico/calicoctl.cfg | 22 - .../installation/package/docker/daemon.json | 23 - .../package/docker/docker.service | 35 -- .../installation/package/etcd/etcd.service | 40 -- .../package/hadoop/container-executor.cfg | 41 -- .../package/submarine/submarine.sh | 25 - .../installation/scripts/calico.sh | 224 --------- .../installation/scripts/docker.sh | 166 ------- .../installation/scripts/download-server.sh | 42 -- .../installation/scripts/environment.sh | 213 --------- .../installation/scripts/etcd.sh | 152 ------ .../installation/scripts/hadoop.sh | 117 ----- .../installation/scripts/menu.sh | 444 ------------------ .../installation/scripts/nvidia-docker.sh | 99 ---- .../installation/scripts/nvidia.sh | 120 ----- .../installation/scripts/submarine.sh | 38 -- .../installation/scripts/utils.sh | 123 ----- 21 files changed, 2172 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf delete mode 100755 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 9c401e8a7cd..a2ea08c6027 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -65,14 +65,6 @@ 0755 - - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation - /share/hadoop/yarn/submarine-installer - - **/* - - 0755 - hadoop-yarn/conf etc/hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf deleted file mode 100644 index 82dcf61b73b..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -##### install config ##### - -# DNS -LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf -YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address - -# etcd hosts list -ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) - -# docker registry ip:port -DOCKER_REGISTRY="10.120.196.232:5000" - -# Start the http download service on the specified server, -# Will download all the dependencies in the http server, -# Run the install script on other servers. -# Automatically download dependencies from http, -# Solve the problem that all servers are slow to download online. -# At the same time, you can also manually download the dependencies to the downloads directory. -# Offline installation of the system -DOWNLOAD_SERVER_IP="10.120.196.236" -DOWNLOAD_SERVER_PORT="19000" - -# yarn container-executor config -# How to compile container-executor: -# Go to the hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager -# path and enter the command: mvn package -Pnative -DskipTests -# Only the nodemanager is compiled. The path of the compiled container-executor is: -# ./target/native/target/usr/local/bin/container-executor -YARN_CONTAINER_EXECUTOR_PATH="Please enter the full path to the container-executor here" - -# Keep the same configuration as 'yarn.nodemanager.linux-container-executor.cgroups.hierarchy' -# in yarn-site.xml, default '/hadoop-yarn' -YARN_HIERARCHY="/hadoop-yarn" - -# Keep the same configuration as 'yarn.nodemanager.local-dirs' in yarn-site.xml -YARN_NODEMANAGER_LOCAL_DIRS="/home/hadoop/disk/1/yarn/local,/home/hadoop/disk/2/yarn/local,/home/hadoop/disk/3/yarn/local" - -# Keep the same configuration as 'yarn.nodemanager.log-dirs' in yarn-site.xml -YARN_NODEMANAGER_LOG_DIRS="/home/hadoop/disk/1/yarn/logs,/home/hadoop/disk/2/yarn/logs,/home/hadoop/disk/3/yarn/logs" - -##### System component download url address, Generally do not need to be modified ##### -DOCKER_REPO="https://yum.dockerproject.org/repo/main/centos/7/Packages" -DOCKER_ENGINE_RPM="docker-engine-1.12.5-1.el7.centos.x86_64.rpm" -DOCKER_ENGINE_SELINUX_RPM="docker-engine-selinux-1.12.5-1.el7.centos.noarch.rpm" - -NVIDIA_DETECT_URL="https://raw.githubusercontent.com/liuxunorg/submarine-installer/packages/nvidia/nvidia-detect" -NVIDIA_DOCKER_RPM="nvidia-docker-1.0.1-1.x86_64.rpm" -NVIDIA_DOCKER_RPM_URL="https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/${NVIDIA_DOCKER_RPM}" - -ETCD_TAR_GZ="etcd-v3.3.9-linux-amd64.tar.gz" -ETCD_DOWNLOAD_URL="https://github.com/etcd-io/etcd/releases/download/v3.3.9/${ETCD_TAR_GZ}" - -CALICO_IPV4POOL_CIDR="192.20.0.0" -CALICO_NETWORK_NAME="calico-network" -CALICO_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico" -CALICO_IPAM_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico-ipam" -CALICOCTL_DOWNLOAD_URL="https://github.com/projectcalico/calicoctl/releases/download/v3.2.3/calicoctl" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh deleted file mode 100755 index 90b12c2820c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# description: sumbarine install scripts. - -ROOT=$(cd "$(dirname "$0")"; pwd) -SUBMARINE_INSTALLER_VERSION="v0.7" -PACKAGE_DIR=${ROOT}/package -SCRIPTS_DIR=${ROOT}/scripts -INSTALL_TEMP_DIR=${ROOT}/temp -DOWNLOAD_DIR=${ROOT}/downloads -DATE=`date +%Y%m%d-%H:%M:%S` -LOG=${ROOT}/logs/install.log.`date +%Y%m%d%H%M%S` -LOCAL_HOST_IP_LIST=() -LOCAL_HOST_IP="" -OPERATING_SYSTEM="" -DOWNLOAD_HTTP="" - -# import shell script -. ${ROOT}/install.conf -. ${ROOT}/scripts/calico.sh -. ${ROOT}/scripts/docker.sh -. ${ROOT}/scripts/download-server.sh -. ${ROOT}/scripts/environment.sh -. ${ROOT}/scripts/etcd.sh -. ${ROOT}/scripts/hadoop.sh -. ${ROOT}/scripts/menu.sh -. ${ROOT}/scripts/nvidia.sh -. ${ROOT}/scripts/nvidia-docker.sh -. ${ROOT}/scripts/submarine.sh -. ${ROOT}/scripts/utils.sh - -#================================= Main ======================================== -mkdir $ROOT/logs/ -p -mkdir $INSTALL_TEMP_DIR -p -mkdir $DOWNLOAD_DIR -p - -source /etc/os-release -OPERATING_SYSTEM=$ID - -get_ip_list -ipCount=${#LOCAL_HOST_IP_LIST[@]} -if [[ $ipCount -eq 1 ]]; then - LOCAL_HOST_IP=${LOCAL_HOST_IP_LIST[0]} -else - echo -e "Detect the network card IP in the server, \e[31m[${LOCAL_HOST_IP_LIST[@]}]\e[0m" - echo -n -e "please enter a valid IP address: " - - read ipInput - if ! valid_ip $ipInput; then - echo -e "you input \e[31m$ipInput\e[0m address format is incorrect! " | tee -a $LOG - exit_install - else - LOCAL_HOST_IP=$ipInput - fi -fi - -echo -n -e "Please confirm whether the IP address of this machine is \e[31m${LOCAL_HOST_IP}\e[0m?[y|n]" -read myselect -if [[ "$myselect" != "y" && "$myselect" != "Y" ]]; then - exit_install -fi - -check_install_conf - -if [[ -n "$DOWNLOAD_SERVER_IP" && -n "$DOWNLOAD_SERVER_PORT" && "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then - DOWNLOAD_HTTP="http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}" -fi - -check_install_user - -# 清理安装临时目录 -rm $INSTALL_TEMP_DIR/* -rf >>$LOG 2>&1 - -menu_index="0" -for ((j=1;;j++)) -do - menu - case "$menu_index" in - "0") - menu_index="$menu_choice" - ;; - "1"|"2"|"3"|"4"|"5") -# echo "aaaa=$menu_index-$menu_choice" - menu_process - if [[ $? = 1 ]]; then - echo "Press any key to return menu!" - read - fi - ;; - "a") - exit_install - ;; - "q") - exit_install - ;; - *) - menu_index="0" - menu_choice="0" - menu - ;; - esac -done diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service deleted file mode 100644 index 744c4191f94..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[Unit] -Description=calico node -After=docker.service -Requires=docker.service - -[Service] -User=root -PermissionsStartOnly=true -ExecStart=/usr/bin/docker run --net=host --privileged --name=calico-node \ - -e ETCD_ENDPOINTS=ETCD_ENDPOINTS_REPLACE \ - -e CALICO_LIBNETWORK_ENABLED=true \ - -e CALICO_NETWORKING_BACKEND=bird \ - -e CALICO_DISABLE_FILE_LOGGING=true \ - -e CALICO_IPV4POOL_CIDR=CALICO_IPV4POOL_CIDR_REPLACE/16 \ - -e CALICO_IPV4POOL_IPIP=always \ - -e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \ - -e FELIX_IPV6SUPPORT=false \ - -e FELIX_LOGSEVERITYSCREEN=info \ - -e FELIX_IPINIPMTU=1440 \ - -e FELIX_HEALTHENABLED=true \ - -e IP= \ - -v /etc/calico/ssl:/etc/calico/ssl \ - -v /var/run/calico:/var/run/calico \ - -v /lib/modules:/lib/modules \ - -v /run/docker/plugins:/run/docker/plugins \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /var/log/calico:/var/log/calico \ - calico/node:v2.6.2 -ExecStop=/usr/bin/docker rm -f calico-node -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg deleted file mode 100644 index 44ada867a62..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: calicoApiConfig -metadata: -spec: - datastoreType: "etcdv2" - etcdEndpoints: ETCD_ENDPOINTS_REPLACE \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json deleted file mode 100644 index 052fe732b86..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - "insecure-registries": ["DOCKER_REGISTRY_REPLACE"], - "cluster-store":"CLUSTER_STORE_REPLACE", - "cluster-advertise":"LOCAL_HOST_IP_REPLACE:2375", - "dns": ["YARN_DNS_HOST_REPLACE", "LOCAL_DNS_HOST_REPLACE"], - "hosts": ["tcp://LOCAL_HOST_IP_REPLACE:2375","unix:///var/run/docker.sock"] -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service deleted file mode 100644 index f444f995d57..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[Unit] -Description=Docker Application Container Engine -Documentation=http://docs.docker.io - -[Service] -Environment="PATH=/usr/bin:/bin:/sbin:/usr/bin:/usr/sbin" -ExecStart=/usr/bin/dockerd --log-level=error -ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT -ExecReload=/bin/kill -s HUP $MAINPID -Restart=on-failure -RestartSec=5 -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity -Delegate=yes -KillMode=process - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service deleted file mode 100644 index 5a44ce2cfa3..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[Unit] -Description=etcd -Documentation=https://github.com/coreos/etcd - -[Service] -Type=notify -Restart=always -RestartSec=5s -LimitNOFILE=40000 -TimeoutStartSec=0 - -ExecStart=/usr/bin/etcd \ - --name=ETCD_NODE_NAME_REPLACE \ - --data-dir=/var/lib/etcd \ - --listen-client-urls=http://LOCAL_HOST_REPLACE:2379,http://127.0.0.1:2379 \ - --listen-peer-urls=http://LOCAL_HOST_REPLACE:2380 \ - --advertise-client-urls=http://LOCAL_HOST_REPLACE:2379 \ - --initial-advertise-peer-urls=http://LOCAL_HOST_REPLACE:2380 \ - --initial-cluster=INITIAL_CLUSTER_REPLACE \ - --initial-cluster-token=etcd-token \ - --initial-cluster-state=new - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg deleted file mode 100644 index 9d9f0cade0a..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -yarn.nodemanager.local-dirs=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE -yarn.nodemanager.linux-container-executor.group=yarn -yarn.nodemanager.log-dirs=YARN_NODEMANAGER_LOG_DIRS_REPLACE -banned.users=root -allowed.system.users=yarn -min.user.id=500 - -[docker] -module.enabled=true -docker.binary=/usr/bin/docker -docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE,DAC_READ_SEARCH,SYS_PTRACE,SYS_ADMIN -docker.allowed.networks=bridge,host,none,CALICO_NETWORK_NAME_REPLACE -docker.allowed.ro-mounts=/etc/group,/etc/passwd,/etc/krb5.conf,YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,regex:^nvidia_driver_.*$ -docker.allowed.rw-mounts=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,YARN_NODEMANAGER_LOG_DIRS_REPLACE -docker.privileged-containers.enabled=false -docker.trusted.registries=local,centos,hortonworks,DOCKER_REGISTRY_REPLACE -docker.allowed.volume-drivers=nvidia-docker -docker.allowed.devices=regex:^/dev/nvidia.*$ - -[gpu] -module.enabled=true - -[cgroups] -root=/sys/fs/cgroup -yarn-hierarchy=YARN_HIERARCHY_REPLACE \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh deleted file mode 100644 index ffbc38f75a5..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Solution: Grant user yarn the access to /sys/fs/cgroup/cpu,cpuacct, -# which is the subfolder of cgroup mount destination. -chown :yarn -R /sys/fs/cgroup/cpu,cpuacct -chmod g+rwx -R /sys/fs/cgroup/cpu,cpuacct - -# If GPUs are used，the access to cgroup devices folder is neede as well -chown :yarn -R /sys/fs/cgroup/devices -chmod g+rwx -R /sys/fs/cgroup/devices diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh deleted file mode 100644 index bd40fcfe3a4..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description download calico bin -## @audience public -## @stability stable -function download_calico_bin() -{ - # submarin http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_CALICOCTL_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calicoctl - MY_CALICO_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico - MY_CALICO_IPAM_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico-ipam - else - MY_CALICOCTL_DOWNLOAD_URL=${CALICOCTL_DOWNLOAD_URL} - MY_CALICO_DOWNLOAD_URL=${CALICO_DOWNLOAD_URL} - MY_CALICO_IPAM_DOWNLOAD_URL=${CALICO_IPAM_DOWNLOAD_URL} - fi - - mkdir -p ${DOWNLOAD_DIR}/calico - - if [[ -f ${DOWNLOAD_DIR}/calico/calico ]]; then - echo "${DOWNLOAD_DIR}/calico/calico is exist." - else - echo "download ${MY_CALICO_DOWNLOAD_URL} ..." - wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_DOWNLOAD_URL} - fi - - if [[ -f ${DOWNLOAD_DIR}/calico/calicoctl ]]; then - echo "${DOWNLOAD_DIR}/calico is exist." - else - echo "download ${MY_CALICOCTL_DOWNLOAD_URL} ..." - wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICOCTL_DOWNLOAD_URL} - fi - - if [[ -f ${DOWNLOAD_DIR}/calico/calico-ipam ]]; then - echo "${DOWNLOAD_DIR}/calico/calico-ipam is exist." - else - echo "download ${MY_CALICO_IPAM_DOWNLOAD_URL} ..." - wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_IPAM_DOWNLOAD_URL} - fi -} - -## @description install calico bin -## @audience public -## @stability stable -function install_calico_bin() -{ - download_calico_bin - - cp -f ${DOWNLOAD_DIR}/calico/calico /usr/bin/calico - cp -f ${DOWNLOAD_DIR}/calico/calicoctl /usr/bin/calicoctl - cp -f ${DOWNLOAD_DIR}/calico/calico-ipam /usr/bin/calico-ipam - - chmod +x /usr/bin/calico - chmod +x /usr/bin/calicoctl - chmod +x /usr/bin/calico-ipam -} - -## @description install calico config -## @audience public -## @stability stable -function install_calico_config() -{ - mkdir -p /etc/calico - - cp -rf ${PACKAGE_DIR}/calico ${INSTALL_TEMP_DIR}/ - - # 1. replace etcdEndpoints - # etcdEndpoints: https://10.196.69.173:2379,https://10.196.69.174:2379,https://10.196.69.175:2379 - etcdEndpoints='' - index=0 - etcdHostsSize=${#ETCD_HOSTS[@]} - for item in ${ETCD_HOSTS[@]} - do - index=$(($index+1)) - etcdEndpoints="${etcdEndpoints}http:\/\/${item}:2379" - if [[ ${index} -lt ${etcdHostsSize} ]]; then - etcdEndpoints=${etcdEndpoints}"," - fi - done - # echo "etcdEndpoints=${etcdEndpoints}" - sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calicoctl.cfg >>$LOG - - if [[ ! -d /etc/calico ]]; then - mkdir /etc/calico - else - rm -rf /etc/calico/* - fi - - cp -f $INSTALL_TEMP_DIR/calico/calicoctl.cfg /etc/calico/calicoctl.cfg - - sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG - sed -i "s/CALICO_IPV4POOL_CIDR_REPLACE/${CALICO_IPV4POOL_CIDR}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG - cp $INSTALL_TEMP_DIR/calico/calico-node.service /etc/systemd/system/ >>$LOG - - systemctl daemon-reload - systemctl enable calico-node.service -} - -## @description modify kernel network config -## @audience public -## @stability stable -function kernel_network_config() -{ - if [ `grep -c "net.ipv4.conf.all.rp_filter=1" /etc/sysctl.conf` -eq '0' ]; then - echo "net.ipv4.conf.all.rp_filter=1" >>/etc/sysctl.conf - fi - - if [ `grep -c "net.ipv4.ip_forward=1" /etc/sysctl.conf` -eq '0' ]; then - echo "net.ipv4.ip_forward=1" >>/etc/sysctl.conf - fi - - sysctl -p -} - -## @description check if the calico-network exist -## @audience public -## @stability stable -function calico_network_exist() -{ - local dockerNetwokInfo=`docker network ls --filter NAME=${CALICO_NETWORK_NAME}` - echo ${dockerNetwokInfo} | grep ${CALICO_NETWORK_NAME} -} - -## @description verification calico -## @audience public -## @stability stable -function verification_calico() -{ - echo " ===== Check if the network between 2 containers can be connected =====" - local claicoNetworkExist=`calico_network_exist` - if [[ "$claicoNetworkExist" = "" ]]; then - echo "Create a calico network" - docker network create --driver calico --ipam-driver calico-ipam ${CALICO_NETWORK_NAME} - else - echo "calico network ${CALICO_NETWORK_NAME} is exist." - fi - - local verifyA="verify-calico-network-A" - local verifyAInfo=`containers_exist ${verifyA}` - if [[ -n "$verifyAInfo" ]]; then - echo "Delete existing container ${verifyA}." - docker stop ${verifyA} - docker rm ${verifyA} - fi - echo "Create containers verify-calico-network-A" - docker run --net ${CALICO_NETWORK_NAME} --name ${verifyA} -tid busybox - - local verifyB="verify-calico-network-B" - local verifyBInfo=`containers_exist ${verifyB}` - if [[ -n "$verifyBInfo" ]]; then - echo "Delete existing container ${verifyB}." - docker stop ${verifyB} - docker rm ${verifyB} - fi - echo "Create containers verify-calico-network-B" - docker run --net ${CALICO_NETWORK_NAME} --name ${verifyB} -tid busybox - - echo -e "\033[33m${verifyA} ping ${verifyB}\033[0m" - docker exec ${verifyA} ping ${verifyB} -c 5 -} - -## @description install calico -## @audience public -## @stability stable -function install_calico() -{ - kernel_network_config - install_calico_bin - install_calico_config - start_calico - verification_calico -} - -## @description uninstall calico -## @audience public -## @stability stable -function uninstall_calico() -{ - echo "stop calico-node.service" - systemctl stop calico-node.service - - echo "rm /usr/bin/calico ..." - rm /usr/bin/calicoctl - rm /usr/bin/calico - rm /usr/bin/calico-ipam - - rm -rf /etc/calico/ - rm /etc/systemd/system/calico-node.service - systemctl daemon-reload -} - -## @description start calico -## @audience public -## @stability stable -function start_calico() -{ - systemctl restart calico-node.service - systemctl status calico-node.service -} - -## @description stop calico -## @audience public -## @stability stable -function stop_calico() -{ - systemctl stop calico-node.service - systemctl status calico-node.service -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh deleted file mode 100644 index 26f19cc9a9f..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description download docker rmp -## @audience public -## @stability stable -function download_docker_rpm() -{ - # download http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_DOCKER_ENGINE_SELINUX_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_SELINUX_RPM}" - MY_DOCKER_ENGINE_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_RPM}" - else - MY_DOCKER_ENGINE_SELINUX_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_SELINUX_RPM} - MY_DOCKER_ENGINE_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_RPM} - fi - - # download docker rpm - if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} ]]; then - echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} is exist." - else - echo "download ${MY_DOCKER_ENGINE_SELINUX_RPM} ..." - wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_SELINUX_RPM} - fi - - if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} ]]; then - echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} is exist." - else - echo "download ${MY_DOCKER_ENGINE_RPM} ..." - wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_RPM} - fi -} - -## @description install docker bin -## @audience public -## @stability stable -function install_docker_bin() -{ - download_docker_rpm - - yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} - yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} -} - -## @description uninstall docker bin -## @audience public -## @stability stable -function uninstall_docker_bin() -{ - download_docker_rpm - - yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} - yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} -} - -## @description install docker config -## @audience public -## @stability stable -function install_docker_config() -{ - rm -rf ${INSTALL_TEMP_DIR}/docker - cp -rf ${PACKAGE_DIR}/docker ${INSTALL_TEMP_DIR}/ - - # replace cluster-store - # "cluster-store":"etcd://10.196.69.173:2379,10.196.69.174:2379,10.196.69.175:2379" - # char '/' need to escape '\/' - clusterStore="etcd:\/\/" - index=1 - etcdHostsSize=${#ETCD_HOSTS[@]} - for item in ${ETCD_HOSTS[@]} - do - clusterStore="${clusterStore}${item}:2379" - if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then - clusterStore=${clusterStore}"," - fi - index=$(($index+1)) - done - echo "clusterStore=${clusterStore}" - sed -i "s/CLUSTER_STORE_REPLACE/${clusterStore}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG - - sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG - sed -i "s/LOCAL_HOST_IP_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG - sed -i "s/YARN_DNS_HOST_REPLACE/${YARN_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG - sed -i "s/LOCAL_DNS_HOST_REPLACE/${LOCAL_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG - - # Delete the ASF license comment in the daemon.json file, otherwise it will cause a json format error. - sed -i '1,16d' $INSTALL_TEMP_DIR/docker/daemon.json - - if [ ! -d "/etc/docker" ]; then - mkdir /etc/docker - fi - - cp $INSTALL_TEMP_DIR/docker/daemon.json /etc/docker/ - cp $INSTALL_TEMP_DIR/docker/docker.service /etc/systemd/system/ >>$LOG -} - -## @description install docker -## @audience public -## @stability stable -function install_docker() -{ - install_docker_bin - install_docker_config - - systemctl daemon-reload - systemctl enable docker.service -} - -## @description unstall docker -## @audience public -## @stability stable -function uninstall_docker() -{ - echo "stop docker service" - systemctl stop docker - - echo "remove docker" - uninstall_docker_bin - - rm /etc/docker/daemon.json >>$LOG - rm /etc/systemd/system/docker.service >>$LOG - - systemctl daemon-reload -} - -## @description start docker -## @audience public -## @stability stable -function start_docker() -{ - systemctl restart docker - systemctl status docker - docker info -} - -## @description stop docker -## @audience public -## @stability stable -function stop_docker() -{ - systemctl stop docker - systemctl status docker -} - -## @description check if the containers exist -## @audience public -## @stability stable -function containers_exist() -{ - local dockerContainersInfo=`docker ps -a --filter NAME=$1` - echo ${dockerContainersInfo} | grep $1 -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh deleted file mode 100644 index 62af8279fab..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description start download server -## @audience public -## @stability stable -function start_download_server() -{ - if [[ "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then - echo -e "\033[31mERROR: Only $DOWNLOAD_SERVER_IP can start the download service.\033[0m" - return 1 - fi - - echo -e "You can put the install package file in the \033[34m${DOWNLOAD_DIR}\033[0m folder first, Or automatic download." - echo -n "Do you want to start download http server?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - download_etcd_bin - download_calico_bin - download_docker_rpm - download_nvidia_driver - download_nvidia_docker_bin - download_yarn_container_executor - - python -m SimpleHTTPServer ${DOWNLOAD_SERVER_PORT} - fi -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh deleted file mode 100644 index 17f59b2359c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description check operation System -## @audience public -## @stability stable -function check_operationSystem() -{ - echo -e "The submarine assembly support \033[32m[centos-release-7-3.1611.el7.centos.x86_64]\033[0m or higher operating system version." - - case ${OPERATING_SYSTEM} in - centos) - local operationSystemVersion=`rpm --query centos-release` - echo -e "The current operating system version is \e[31m[${operationSystemVersion}]\e[0m" | tee -a $LOG - ;; - *) - echo -e "\033[31mWARN: The submarine assembly Unsupported [${OPERATING_SYSTEM}] operating system\033[0m" - ;; - esac -} - -## @description update operation System Kernel -## @audience public -## @stability stable -function update_operationSystemKernel() -{ - echo "If the server is unable to connect to the network, execute the following command yourself: - wget http://vault.centos.org/7.3.1611/os/x86_64/Packages/kernel-headers-3.10.0-514.el7.x86_64.rpm - rpm -ivh kernel-headers-3.10.0-514.el7.x86_64.rpm" - - echo -n "Do you want to kernel upgrades?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - echo "Now try to use the yum command for kernel upgrades ..." - yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) - - local kernelVersion=`uname -r` - echo -e "After the upgrade, the operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG - fi -} - -## @description check operation system kernel -## @audience public -## @stability stable -function check_operationSystemKernel() -{ -case ${OPERATING_SYSTEM} in -centos) - local kernelVersion=`uname -r` - - echo -e "Submarine support operating system kernel version is \033[32m 3.10.0-514.el7.x86_64 \033[0m" | tee -a $LOG - echo -e "Current operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG - - update_operationSystemKernel - ;; -*) - echo -e "\033[31m WARN: The submarine assembly Unsupported operating system [${OPERATING_SYSTEM}] \033[0m" - ;; -esac -} - -## @description get gcc version -## @audience public -## @stability stable -function get_gcc_version() -{ - local gccVersion=`gcc --version` - version=${gccVersion%Copyright*} - echo $version -} - -## @description install gcc -## @audience public -## @stability stable -function install_gcc() -{ - echo -n "Do you want to install gcc?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]]; then - echo "Execute the yum install gcc make g++ command" - yum install gcc make g++ - - local gccVersion=`gcc --version` - echo -e "After the install, the gcc version is \e[31m${gccVersion}\e[0m" | tee -a $LOG - fi -} - -## @description check gcc Version -## @audience public -## @stability stable -function check_gccVersion() -{ - local gccVersionInfo=`gcc --version` - local gccVersion=${gccVersionInfo%Copyright*} - - if [[ "$gccVersion" = "" ]]; then - echo "The gcc was not installed on the system. Automated installation ..." - install_gcc - else - echo -e "Submarine gcc version need \033[34mgcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11)\033[0m or higher." - echo -e "Current gcc version was \033[34m${gccVersion}\033[0m" - fi -} - -## @description check GPU -## @audience public -## @stability stable -function check_GPU() -{ - gpuInfo=`lspci | grep -i nvidia` - - if [[ "$gpuInfo" = "" ]]; then - echo -e "\033[31mERROR: The system did not detect the GPU graphics card.\033[0m" - else - echo -e "\033[32mINFO: The system detect the GPU graphics card.\033[0m" - fi -} - -## @description check user group -## @audience public -## @stability stable -function check_userGroup() -{ - echo -e "check hadoop user group ..." - - echo -e "Hadoop runs the required user [hdfs, mapred, yarn] and groups [hdfs, mapred, yarn, hadoop] installed by ambari." - echo -e "If you are not using ambari for hadoop installation, -then you can add the user and group by root by executing the following command: -\033[34madduser hdfs -adduser mapred -adduser yarn -addgroup hadoop -usermod -aG hdfs,hadoop hdfs -usermod -aG mapred,hadoop mapred -usermod -aG yarn,hadoop yarn -usermod -aG hdfs,hadoop hadoop -groupadd docker -usermod -aG docker yarn -usermod -aG docker hadoop\033[0m\n" - - echo -e "check docker user group ..." - # check user group - DOCKER_USER_GROUP='docker' - egrep "^${DOCKER_USER_GROUP}" /etc/group >& /dev/null - if [[ $? -ne 0 ]]; then - echo -e "user group ${DOCKER_USER_GROUP} does not exist, Please execute the following command:" - echo -e "\033[34mgroupadd $DOCKER_USER_GROUP\033[0m" - fi - - # check user - USER_GROUP=(yarn hadoop) - for user in ${USER_GROUP[@]} - do - egrep "^${user}" /etc/passwd >& /dev/null - if [[ $? -ne 0 ]]; then - echo -e "User ${user} does not exist, Please execute the following command:" - echo -e "\033[34madduser ${user}\033[0m" - echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m" - fi - - echo -e "Please execute the following command:" - echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m" - done -} - -## @description Some preparatory work for nvidia driver installation -## @audience public -## @stability stable -function prepare_nvidia_environment() -{ - echo "prepare nvidia environment ..." - - yum -y update - yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) - - yum -y install epel-release - yum -y install dkms - - echo -e "\033[34m ===== Please manually execute the following command ===== -# 1. Disable nouveau -# Add the content 'rd.driver.blacklist=nouveau nouveau.modeset=0' -# to the 'GRUB_CMDLINE_LINUX' configuration item in the /etc/default/grub file. -root:> vi /etc/default/grub -vi:> GRUB_CMDLINE_LINUX=\"rd.driver.blacklist=nouveau nouveau.modeset=0 ...\" - -# 2. Generate configuration -root:> grub2-mkconfig -o /boot/grub2/grub.cfg - -# 3. Open (new) /etc/modprobe.d/blacklist.conf, add content 'blacklist nouveau' -root:> vi /etc/modprobe.d/blacklist.conf -vi:> blacklist nouveau - -# 4. Update configuration and reboot -root:> mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r)-nouveau.img -root:> dracut /boot/initramfs-$(uname -r).img $(uname -r) -root:> reboot -\033[0m" -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh deleted file mode 100644 index 48df80ee0e0..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description download etcd bin -## @audience public -## @stability stable -function download_etcd_bin() -{ - # my download http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_ETCD_DOWNLOAD_URL="${DOWNLOAD_HTTP}/downloads/etcd/${ETCD_TAR_GZ}" - else - MY_ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL} - fi - - if [[ -f "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ}" ]]; then - echo "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} is exist." - else - echo "download ${MY_ETCD_DOWNLOAD_URL} ..." - wget -P ${DOWNLOAD_DIR}/etcd ${MY_ETCD_DOWNLOAD_URL} - fi -} - -## @description install etcd bin -## @audience public -## @stability stable -function install_etcd_bin() -{ - download_etcd_bin - - # install etcd bin - mkdir -p ${INSTALL_TEMP_DIR} - rm -rf ${INSTALL_TEMP_DIR}/etcd-*-linux-amd6 - tar zxvf ${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} -C ${INSTALL_TEMP_DIR} - - cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcd /usr/bin - cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcdctl /usr/bin - - mkdir -p /var/lib/etcd - chmod -R a+rw /var/lib/etcd -} - -## @description install etcd config -## @audience public -## @stability stable -function install_etcd_config() -{ - # config etcd.service - rm -rf ${INSTALL_TEMP_DIR}/etcd - cp -rf ${PACKAGE_DIR}/etcd ${INSTALL_TEMP_DIR}/ - - # 1. Replace name with ETCD_NODE_NAME_REPLACE based on the location of the local IP in $ETCD_HOSTS - indexEtcdList=$(indexByEtcdHosts ${LOCAL_HOST_IP}) - # echo ${indexEtcdList} - etcdNodeName="etcdnode${indexEtcdList}" - # echo ${etcdNodeName} - sed -i "s/ETCD_NODE_NAME_REPLACE/${etcdNodeName}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG - - # 2. Replace local IP address - sed -i "s/LOCAL_HOST_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG - - # 3. Replace the initial-cluster parameter - # --initial-cluster=etcdnode1=http://10.196.69.173:2380,etcdnode2=http://10.196.69.174:2380,etcdnode3=http://10.196.69.175:2380 \ - initialCluster='' - index=0 - etcdHostsSize=${#ETCD_HOSTS[@]} - for item in ${ETCD_HOSTS[@]} - do - # char '/' need to escape '\/' - initialCluster="${initialCluster}etcdnode${index}=http:\/\/${item}:2380" - if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then - initialCluster=${initialCluster}"," - fi - index=$(($index+1)) - done - #echo "initialCluster=${initialCluster}" - sed -i "s/INITIAL_CLUSTER_REPLACE/${initialCluster}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG - - cp $INSTALL_TEMP_DIR/etcd/etcd.service /etc/systemd/system/ >>$LOG -} - -## @description install etcd -## @audience public -## @stability stable -function install_etcd() -{ - index=$(indexByEtcdHosts ${LOCAL_HOST_IP}) - if [ -z "$index" ]; then - echo -e "STOP: This host\033[31m[${LOCAL_HOST_IP}]\033[0m is not in the ETCD server list\033[31m[${ETCD_HOSTS[@]}]\033[0m" - return 1 - fi - - install_etcd_bin - - install_etcd_config - - systemctl daemon-reload - systemctl enable etcd.service -} - -## @description uninstall etcd -## @audience public -## @stability stable -function uninstall_etcd() -{ - echo "stop etcd.service" - systemctl stop etcd.service - - echo "rm etcd ..." - rm /usr/bin/etcd - rm /usr/bin/etcdctl - rm -rf /var/lib/etcd - rm /etc/systemd/system/etcd.service - - systemctl daemon-reload -} - -## @description start etcd -## @audience public -## @stability stable -function start_etcd() -{ - systemctl restart etcd.service - - echo " ===== Check the status of the etcd service =====" - echo " exec etcdctl cluster-health" - etcdctl cluster-health - echo " exec etcdctl cluster-health" - etcdctl member list -} - -## @description stop etcd -## @audience public -## @stability stable -function stop_etcd() -{ - systemctl stop etcd.service -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh deleted file mode 100644 index 9df5d7e5b21..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description install yarn -## @audience public -## @stability stable -function install_yarn() -{ - install_yarn_container_executor - install_yarn_config -} - -## @description uninstall yarn -## @audience public -## @stability stable -function uninstall_yarn() -{ - rm -rf /etc/yarn/sbin/Linux-amd64-64/* - rm -rf /etc/yarn/sbin/etc/hadoop/* -} - -## @description download yarn container executor -## @audience public -## @stability stable -function download_yarn_container_executor() -{ - # my download http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_YARN_CONTAINER_EXECUTOR_PATH="${DOWNLOAD_HTTP}/downloads/hadoop/container-executor" - else - MY_YARN_CONTAINER_EXECUTOR_PATH=${YARN_CONTAINER_EXECUTOR_PATH} - fi - - if [ ! -d "${DOWNLOAD_DIR}/hadoop" ]; then - mkdir -p ${DOWNLOAD_DIR}/hadoop - fi - - if [[ -f "${DOWNLOAD_DIR}/hadoop/container-executor" ]]; then - echo "${DOWNLOAD_DIR}/hadoop/container-executor is exist." - else - if [[ -n "$DOWNLOAD_HTTP" ]]; then - echo "download ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..." - wget -P ${DOWNLOAD_DIR}/hadoop ${MY_YARN_CONTAINER_EXECUTOR_PATH} - else - echo "copy ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..." - cp ${MY_YARN_CONTAINER_EXECUTOR_PATH} ${DOWNLOAD_DIR}/hadoop/ - fi - fi -} - -## @description install yarn container executor -## @audience public -## @stability stable -function install_yarn_container_executor() -{ - echo "install yarn container executor file ..." - - download_yarn_container_executor - - if [ ! -d "/etc/yarn/sbin/Linux-amd64-64" ]; then - mkdir -p /etc/yarn/sbin/Linux-amd64-64 - fi - if [ -f "/etc/yarn/sbin/Linux-amd64-64/container-executor" ]; then - rm /etc/yarn/sbin/Linux-amd64-64/container-executor - fi - - cp -f ${DOWNLOAD_DIR}/hadoop/container-executor /etc/yarn/sbin/Linux-amd64-64 - - sudo chmod 6755 /etc/yarn/sbin/Linux-amd64-64 - sudo chown :yarn /etc/yarn/sbin/Linux-amd64-64/container-executor - sudo chmod 6050 /etc/yarn/sbin/Linux-amd64-64/container-executor -} - -## @description install yarn config -## @audience public -## @stability stable -function install_yarn_config() -{ - echo "install yarn config file ..." - - cp -R ${PACKAGE_DIR}/hadoop ${INSTALL_TEMP_DIR}/ - - find="/" - replace="\/" - escape_yarn_nodemanager_local_dirs=${YARN_NODEMANAGER_LOCAL_DIRS//$find/$replace} - escape_yarn_nodemanager_log_dirs=${YARN_NODEMANAGER_LOG_DIRS//$find/$replace} - escape_yarn_hierarchy=${YARN_HIERARCHY//$find/$replace} - - sed -i "s/YARN_NODEMANAGER_LOCAL_DIRS_REPLACE/${escape_yarn_nodemanager_local_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG - sed -i "s/YARN_NODEMANAGER_LOG_DIRS_REPLACE/${escape_yarn_nodemanager_log_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG - sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG - sed -i "s/CALICO_NETWORK_NAME_REPLACE/${CALICO_NETWORK_NAME}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG - sed -i "s/YARN_HIERARCHY_REPLACE/${escape_yarn_hierarchy}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG - - # Delete the ASF license comment in the container-executor.cfg file, otherwise it will cause a cfg format error. - sed -i '1,16d' $INSTALL_TEMP_DIR/hadoop/container-executor.cfg - - if [ ! -d "/etc/yarn/sbin/etc/hadoop" ]; then - mkdir -p /etc/yarn/sbin/etc/hadoop - fi - - cp -f $INSTALL_TEMP_DIR/hadoop/container-executor.cfg /etc/yarn/sbin/etc/hadoop/ -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh deleted file mode 100644 index 68f91c1260d..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh +++ /dev/null @@ -1,444 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description main menu -## @audience public -## @stability stable -main_menu() -{ -cat< [prepare system environment] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.prepare operation system\e[0m" -echo -e " \e[32m2.prepare operation system kernel\e[0m" -echo -e " \e[32m3.prepare GCC version\e[0m" -echo -e " \e[32m4.check GPU\e[0m" -echo -e " \e[32m5.prepare user&group\e[0m" -echo -e " \e[32m6.prepare nvidia environment\e[0m" -echo -e "" -echo -e " \e[32mb.back main menu\e[0m" -cat< [install component] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.instll etcd\e[0m" -echo -e " \e[32m2.instll docker\e[0m" -echo -e " \e[32m3.instll calico network\e[0m" -echo -e " \e[32m4.instll nvidia driver\e[0m" -echo -e " \e[32m5.instll nvidia docker\e[0m" -echo -e " \e[32m6.instll yarn container-executor\e[0m" -echo -e " \e[32m7.instll submarine autorun script\e[0m" -echo -e "" -echo -e " \e[32mb.back main menu\e[0m" -cat< [uninstll component] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.uninstll etcd\e[0m" -echo -e " \e[32m2.uninstll docker\e[0m" -echo -e " \e[32m3.uninstll calico network\e[0m" -echo -e " \e[32m4.uninstll nvidia driver\e[0m" -echo -e " \e[32m5.uninstll nvidia docker\e[0m" -echo -e " \e[32m6.uninstll yarn container-executor\e[0m" -echo -e " \e[32m7.uninstll submarine autorun script\e[0m" -echo -e "" -echo -e " \e[32mb.back main menu\e[0m" -cat< [stop component] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.start etcd\e[0m" -echo -e " \e[32m2.start docker\e[0m" -echo -e " \e[32m3.start calico network\e[0m" -echo -e "" -echo -e " \e[32mb.back main menu\e[0m" -cat< [stop component] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.stop etcd\e[0m" -echo -e " \e[32m2.stop docker\e[0m" -echo -e " \e[32m3.stop calico network\e[0m" -echo -e "" -echo -e " \e[32mb.back main menu\e[0m" -cat< nvidia-docker run --rm ${DOCKER_REGISTRY}/nvidia/cuda:9.0-devel nvidia-smi -# If you don't see the list of graphics cards above, the NVIDIA driver installation failed. ===== -\033[0m" - - echo -e "\033[32m===== Please manually execute the following command =====\033[0m" - echo -e "\033[32m# Test with tf.test.is_gpu_available() -shell:> nvidia-docker run -it ${DOCKER_REGISTRY}/tensorflow/tensorflow:1.9.0-gpu bash -# In docker container -container:> python -python:> import tensorflow as tf -python:> tf.test.is_gpu_available() -python:> exit() -\033[0m" -} - -## @description uninstall nvidia docker -## @audience public -## @stability stable -function uninstall_nvidia_docker() -{ - echo "This method is not implemented." -} - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh deleted file mode 100644 index 7b2b1de0d56..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -nvidia_run_file="" - -## @description get nvidia version -## @audience public -## @stability stable -function get_nvidia_version() -{ - chmod +x ${DOWNLOAD_DIR}/nvidia/nvidia-detect - local nvidia_detect_info=`${DOWNLOAD_DIR}/nvidia/nvidia-detect -v` - echo $nvidia_detect_info | sed "s/^.*This device requires the current \([0-9.]*\).*/\1/" -} - -## @description download nvidia driver -## @audience public -## @stability stable -function download_nvidia_driver() -{ - # submarin http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_NVIDIA_DETECT_URL="${DOWNLOAD_HTTP}/downloads/nvidia/nvidia-detect" - else - MY_NVIDIA_DETECT_URL=${NVIDIA_DETECT_URL} - fi - - # download nvidia_detect - if [[ -f "${DOWNLOAD_DIR}/nvidia/nvidia-detect" ]]; then - echo "${DOWNLOAD_DIR}/nvidia/nvidia-detect is exist." - else - echo "download ${MY_NVIDIA_DETECT_URL} ..." - wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DETECT_URL} - fi - - echo "execution nvidia-detect to check the graphics card ..." - local nvidiaVersion=`get_nvidia_version` - echo -e "detect nvidia version is \033[31m${nvidiaVersion}\033[0m" - - # download NVIDIA driver - if [[ "$nvidiaVersion" = "" ]]; then - echo -e "\033[31mERROR: No graphics card device detected.\033[0m" - return 1 - else - nvidia_run_file="NVIDIA-Linux-x86_64-${nvidiaVersion}.run" - - # submarin http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_NVIDIA_DRIVER_RUN_URL="${DOWNLOAD_HTTP}/downloads/nvidia/${nvidia_run_file}" - else - # http://us.download.nvidia.com/XFree86/Linux-x86_64/390.87/NVIDIA-Linux-x86_64-390.87.run - MY_NVIDIA_DRIVER_RUN_URL="http://us.download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/${nvidia_run_file}" - fi - - if [[ -f ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} ]]; then - echo "NVIDIA driver files already exist in the ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} directory." - echo "===== Please make sure the ${DOWNLOAD_DIR}/nvidia/nvidia/${nvidia_run_file} file is complete and can be used normally. =====" - else - echo "Download the NVIDIA driver from the ${MY_NVIDIA_DRIVER_RUN_URL}" - wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DRIVER_RUN_URL} - fi - fi -} - -## @description install nvidia -## @audience public -## @stability stable -function install_nvidia() -{ - download_nvidia_driver - - # Confirm that the system disables nouveau - local disable_nouveau_info=`lsmod | grep nouveau` - if [[ "$disable_nouveau_info" = "" ]]; then - echo "===== Start installing the NVIDIA driver =====" - echo -e "Some options during the installation -Would you like to register the kernel module sources with DKMS? - This will allow DKMS to automatically build a new module, if you install a different kernel later. \033[33m[Yes]\033[0m -Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m -centos Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m -Would you like to run the nvidia-xconfig utility to automatically update your X configuration file... \033[33m[No]\033[0m" - sleep 2 - sh ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} - else - echo -e "ERROR: Nouveau is not disabled" - return 1 - fi - - echo -e "\033[32m===== execute nvidia-smi. You should be able to see the list of graphics cards =====\033[0m" - sleep 1 - nvidia-smi -} - -## @description uninstall nvidia -## @audience public -## @stability stable -function uninstall_nvidia() -{ - if [ ! -f "/usr/bin/nvidia-uninstall" ]; then - echo -e "\033[31mERROR: /usr/bin/nvidia-uninstall file is not exist!\033[0m" - return 1 - fi - - echo -e "execute /usr/bin/nvidia-uninstall" - /usr/bin/nvidia-uninstall -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh deleted file mode 100644 index b73d5e77c28..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description install submarine autorun script -## @audience public -## @stability stable -function install_submarine() -{ - cp ${PACKAGE_DIR}/submarine/submarine.sh /etc/rc.d/init.d/submarine.sh - chmod +x /etc/rc.d/init.d/submarine.sh - chmod +x /etc/rc.d/rc.local - - if [ `grep -c "/etc/rc.d/init.d/submarine.sh" /etc/rc.d/rc.local` -eq '0' ]; then - echo "/etc/rc.d/init.d/submarine.sh">> /etc/rc.d/rc.local - fi -} - -## @description uninstall submarine autorun script -## @audience public -## @stability stable -function uninstall_submarine() -{ - rm /etc/rc.d/init.d/submarine.sh -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh deleted file mode 100644 index 7b3c2a94ce3..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description check install user -## @audience public -## @stability stable -function check_install_user() -{ - if [[ $(id -u) -ne 0 ]];then - echo "This script must be run with a ROOT user!" - exit # don't call exit_install() - fi -} - -## @description exit install -## @audience public -## @stability stable -function exit_install() -{ - echo "Exit the installation!" | tee -a $LOG - exit $1 -} - -## @description Check if the IP address format is correct -## @audience public -## @stability stable -function valid_ip() -{ - local ip=$1 - local stat=1 - - if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then - OIFS=$IFS - IFS='.' - ip=($ip) - IFS=$OIFS - - if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]; then - stat=$? - fi - fi - - return $stat -} - -## @description Check if the configuration file configuration is correct -## @audience public -## @stability stable -function check_install_conf() -{ - echo "Check if the configuration file configuration is correct ..." | tee -a $LOG - - # check etcd conf - hostCount=${#ETCD_HOSTS[@]} - if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2 - echo "Number of nodes = [$hostCount], must be configured to be greater than or equal to 3 servers! " | tee -a $LOG - exit_install - fi - for ip in ${ETCD_HOSTS[@]} - do - if ! valid_ip $ip; then - echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG - exit_install - fi - done - echo "Check if the configuration file configuration is correct [ Done ]" | tee -a $LOG -} - -## @description index by EtcdHosts list -## @audience public -## @stability stable -function indexByEtcdHosts() { - index=0 - while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do - if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then - echo $index - return - fi - let "index++" - done - echo "" -} - -## @description get local IP -## @audience public -## @stability stable -function getLocalIP() -{ - local _ip _myip _line _nl=$'\n' - while IFS=$': \t' read -a _line ;do - [ -z "${_line%inet}" ] && - _ip=${_line[${#_line[1]}>4?1:2]} && - [ "${_ip#127.0.0.1}" ] && _myip=$_ip - done< <(LANG=C /sbin/ifconfig) - printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip -} - -## @description get ip list -## @audience public -## @stability stable -function get_ip_list() -{ - array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2) - - for ip in ${array[@]} - do - LOCAL_HOST_IP_LIST+=(${ip}) - done -}