From bbf5844968b34d84287f3f850307da0fbb953161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Wed, 10 Jul 2019 14:27:55 +0200 Subject: [PATCH] HDDS-1525. Mapreduce failure when using Hadoop 2.7.5 Closes #1065 --- .../hadoop/ozone/client/rpc/RpcClient.java | 8 +- .../ozone-mr/{docker-config => common-config} | 44 ------- .../{ozonefs => ozone-mr/hadoop27}/.env | 8 +- .../ozone-mr/hadoop27/docker-compose.yaml | 102 +++++++++++++++++ .../compose/ozone-mr/hadoop27/docker-config | 19 +++ .../{ozonefs => ozone-mr/hadoop27}/test.sh | 17 ++- .../src/main/compose/ozone-mr/hadoop31/.env | 22 ++++ .../{ => hadoop31}/docker-compose.yaml | 28 ++--- .../compose/ozone-mr/hadoop31/docker-config | 19 +++ .../main/compose/ozone-mr/hadoop31/test.sh | 45 ++++++++ .../main/compose/ozone-mr/{ => hadoop32}/.env | 4 +- .../ozone-mr/hadoop32/docker-compose.yaml | 104 +++++++++++++++++ .../compose/ozone-mr/hadoop32/docker-config | 19 +++ .../compose/ozone-mr/{ => hadoop32}/test.sh | 8 +- .../main/compose/ozonefs/docker-compose.yaml | 100 ---------------- .../src/main/compose/ozonefs/docker-config | 38 ------ .../src/main/compose/ozonefs/hadoopo3fs.robot | 56 --------- hadoop-ozone/dist/src/main/compose/testlib.sh | 21 +++- .../main/smoketest/ozonefs/hadoopo3fs.robot | 2 +- hadoop-ozone/ozonefs-lib-legacy/pom.xml | 30 ++++- .../org/apache/hadoop/fs/ozone/BasicOzFs.java | 45 ++++++++ .../fs/ozone/BasicOzoneClientAdapterImpl.java | 56 ++++++++- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 52 +++++++-- .../hadoop/fs/ozone/FileStatusAdapter.java | 108 ++++++++++++++++++ .../hadoop/fs/ozone/FilteredClassLoader.java | 6 +- .../hadoop/fs/ozone/OzoneClientAdapter.java | 20 ++-- 26 files changed, 675 insertions(+), 306 deletions(-) rename hadoop-ozone/dist/src/main/compose/ozone-mr/{docker-config => common-config} (63%) rename hadoop-ozone/dist/src/main/compose/{ozonefs => ozone-mr/hadoop27}/.env (75%) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-config rename hadoop-ozone/dist/src/main/compose/{ozonefs => ozone-mr/hadoop27}/test.sh (65%) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/.env rename hadoop-ozone/dist/src/main/compose/ozone-mr/{ => hadoop31}/docker-compose.yaml (87%) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-config create mode 100755 hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh rename hadoop-ozone/dist/src/main/compose/ozone-mr/{ => hadoop32}/.env (90%) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-config rename hadoop-ozone/dist/src/main/compose/ozone-mr/{ => hadoop32}/test.sh (90%) delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonefs/docker-config delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonefs/hadoopo3fs.robot create mode 100644 hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java create mode 100644 hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index ce7150d445e..f47677a2def 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -25,7 +25,6 @@ import org.apache.hadoop.crypto.CryptoInputStream; import org.apache.hadoop.crypto.CryptoOutputStream; import org.apache.hadoop.crypto.key.KeyProvider; -import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; @@ -109,7 +108,7 @@ * to execute client calls. This uses RPC protocol for communication * with the servers. */ -public class RpcClient implements ClientProtocol, KeyProviderTokenIssuer { +public class RpcClient implements ClientProtocol { private static final Logger LOG = LoggerFactory.getLogger(RpcClient.class); @@ -1137,9 +1136,4 @@ public URI getKeyProviderUri() throws IOException { public String getCanonicalServiceName() { return (dtService != null) ? dtService.toString() : null; } - - @Override - public Token getDelegationToken(String renewer) throws IOException { - return getDelegationToken(renewer == null ? null : new Text(renewer)); - } } diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config similarity index 63% rename from hadoop-ozone/dist/src/main/compose/ozone-mr/docker-config rename to hadoop-ozone/dist/src/main/compose/ozone-mr/common-config index 216e2314f4d..b83f3323fab 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config @@ -28,8 +28,6 @@ OZONE-SITE.XML_hdds.datanode.dir=/data/hdds HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012 -CORE-SITE.xml_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem -CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs CORE-SITE.xml_fs.defaultFS=o3fs://bucket1.vol1/ MAPRED-SITE.XML_mapreduce.framework.name=yarn @@ -39,7 +37,6 @@ MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME MAPRED-SITE.XML_mapreduce.map.memory.mb=4096 MAPRED-SITE.XML_mapreduce.reduce.memory.mb=4096 MAPRED-SITE.XML_mapred.child.java.opts=-Xmx2g -MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-current-0.5.0-SNAPSHOT.jar YARN-SITE.XML_yarn.app.mapreduce.am.staging-dir=/user YARN_SITE.XML_yarn.timeline-service.enabled=true @@ -87,44 +84,3 @@ LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop=INFO LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR - -#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. -#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm - -#LOG4J2.PROPERTIES_* are for Ozone Audit Logging -LOG4J2.PROPERTIES_monitorInterval=30 -LOG4J2.PROPERTIES_filter=read,write -LOG4J2.PROPERTIES_filter.read.type=MarkerFilter -LOG4J2.PROPERTIES_filter.read.marker=READ -LOG4J2.PROPERTIES_filter.read.onMatch=DENY -LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL -LOG4J2.PROPERTIES_filter.write.type=MarkerFilter -LOG4J2.PROPERTIES_filter.write.marker=WRITE -LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL -LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL -LOG4J2.PROPERTIES_appenders=console, rolling -LOG4J2.PROPERTIES_appender.console.type=Console -LOG4J2.PROPERTIES_appender.console.name=STDOUT -LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout -LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n -LOG4J2.PROPERTIES_appender.rolling.type=RollingFile -LOG4J2.PROPERTIES_appender.rolling.name=RollingFile -LOG4J2.PROPERTIES_appender.rolling.fileName=${sys:hadoop.log.dir}/om-audit-${hostName}.log -LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz -LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout -LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n -LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies -LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy -LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400 -LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy -LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB -LOG4J2.PROPERTIES_loggers=audit -LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger -LOG4J2.PROPERTIES_logger.audit.name=OMAudit -LOG4J2.PROPERTIES_logger.audit.level=INFO -LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling -LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile -LOG4J2.PROPERTIES_rootLogger.level=INFO -LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout -LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT - diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/.env b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/.env similarity index 75% rename from hadoop-ozone/dist/src/main/compose/ozonefs/.env rename to hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/.env index 8753b1dc9bb..1ec33aefa99 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonefs/.env +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/.env @@ -14,5 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -HDDS_VERSION=${hdds.version} -HADOOP_RUNNER_VERSION=${docker.ozone-runner.version} \ No newline at end of file +HDDS_VERSION=@hdds.version@ +#TODO: swich to apache/hadoop. Older versions are not supported by apache/hadoop, yet. +# See: HADOOP-16092 for more details. +HADOOP_IMAGE=flokkr/hadoop +HADOOP_VERSION=2.7.7 +HADOOP_RUNNER_VERSION=@docker.ozone-runner.version@ diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml new file mode 100644 index 00000000000..a23566b8a47 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + datanode: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + volumes: + - ../../..:/opt/hadoop + ports: + - 9864 + command: ["/opt/hadoop/bin/ozone","datanode"] + env_file: + - docker-config + - ../common-config + om: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + hostname: om + volumes: + - ../../..:/opt/hadoop + ports: + - 9874:9874 + environment: + WAITFOR: scm:9876 + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + env_file: + - docker-config + - ../common-config + command: ["/opt/hadoop/bin/ozone","om"] + s3g: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + hostname: s3g + volumes: + - ../../..:/opt/hadoop + ports: + - 9878:9878 + env_file: + - ./docker-config + - ../common-config + command: ["/opt/hadoop/bin/ozone","s3g"] + scm: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + hostname: scm + volumes: + - ../../..:/opt/hadoop + ports: + - 9876:9876 + env_file: + - docker-config + - ../common-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["/opt/hadoop/bin/ozone","scm"] + rm: + image: ${HADOOP_IMAGE}:${HADOOP_VERSION} + hostname: rm + volumes: + - ../../..:/opt/ozone + ports: + - 8088:8088 + env_file: + - ./docker-config + - ../common-config + environment: + HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-legacy-@project.version@.jar + command: ["yarn", "resourcemanager"] + nm: + image: ${HADOOP_IMAGE}:${HADOOP_VERSION} + hostname: nm + volumes: + - ../../..:/opt/ozone + env_file: + - ./docker-config + - ../common-config + environment: + HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-legacy-@project.version@.jar + WAIT_FOR: rm:8088 + command: ["yarn","nodemanager"] +# Optional section: comment out this part to get DNS resolution for all the containers. +# dns: +# image: andyshinn/dnsmasq:2.76 +# ports: +# - 53:53/udp +# - 53:53/tcp +# volumes: +# - "/var/run/docker.sock:/var/run/docker.sock" +# command: +# - "-k" +# - "-d" diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-config new file mode 100644 index 00000000000..9e9cc04359c --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-config @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.xml_fs.o3fs.impl=org.apache.hadoop.fs.ozone.BasicOzoneFileSystem +CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.BasicOzFs +MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-legacy-@project.version@.jar diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/test.sh similarity index 65% rename from hadoop-ozone/dist/src/main/compose/ozonefs/test.sh rename to hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/test.sh index 5ca756957c3..a2ab5d8826a 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonefs/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/test.sh @@ -19,20 +19,25 @@ COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR # shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" +source "$COMPOSE_DIR/../../testlib.sh" start_docker_env -execute_robot_test scm ozonefs/ozonefs.robot +execute_robot_test scm createmrenv.robot -## TODO: As of now the o3fs tests are unstable. +#rm is the container name (resource manager) and not the rm command +execute_command_in_container rm sudo apk add --update py-pip +execute_command_in_container rm sudo pip install robotframework -export OZONE_HOME=/opt/ozone +# reinitialize the directories to use +export OZONE_DIR=/opt/ozone +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../../testlib.sh" -#execute_robot_test hadoop32 ozonefs/hadoopo3fs.robot +execute_robot_test rm ozonefs/hadoopo3fs.robot -#execute_robot_test hadoop31 ozonefs/hadoopo3fs.robot +execute_robot_test rm -v hadoop.version:2.7.7 mapreduce.robot stop_docker_env diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/.env b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/.env new file mode 100644 index 00000000000..c6ef057b518 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/.env @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HDDS_VERSION=@hdds.version@ +#TODO: swich to apache/hadoop. Older versions are not supported by apache/hadoop, yet. +# See: HADOOP-16092 for more details. +HADOOP_IMAGE=flokkr/hadoop +HADOOP_VERSION=3.1.2 +HADOOP_RUNNER_VERSION=@docker.ozone-runner.version@ diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml similarity index 87% rename from hadoop-ozone/dist/src/main/compose/ozone-mr/docker-compose.yaml rename to hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml index a0d138f0bfb..af1c9609185 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml @@ -19,17 +19,18 @@ services: datanode: image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} volumes: - - ../..:/opt/hadoop + - ../../..:/opt/hadoop ports: - 9864 command: ["/opt/hadoop/bin/ozone","datanode"] env_file: - docker-config + - ../common-config om: image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} hostname: om volumes: - - ../..:/opt/hadoop + - ../../..:/opt/hadoop ports: - 9874:9874 environment: @@ -37,26 +38,29 @@ services: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION env_file: - docker-config + - ../common-config command: ["/opt/hadoop/bin/ozone","om"] s3g: image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} hostname: s3g volumes: - - ../..:/opt/hadoop + - ../../..:/opt/hadoop ports: - 9878:9878 env_file: - ./docker-config + - ../common-config command: ["/opt/hadoop/bin/ozone","s3g"] scm: image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} hostname: scm volumes: - - ../..:/opt/hadoop + - ../../..:/opt/hadoop ports: - 9876:9876 env_file: - docker-config + - ../common-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION command: ["/opt/hadoop/bin/ozone","scm"] @@ -64,11 +68,12 @@ services: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} hostname: rm volumes: - - ../..:/opt/ozone + - ../../..:/opt/ozone ports: - 8088:8088 env_file: - ./docker-config + - ../common-config environment: HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-current-@project.version@.jar command: ["yarn", "resourcemanager"] @@ -76,20 +81,11 @@ services: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} hostname: nm volumes: - - ../..:/opt/ozone + - ../../..:/opt/ozone env_file: - ./docker-config + - ../common-config environment: HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-current-@project.version@.jar WAIT_FOR: rm:8088 command: ["yarn","nodemanager"] - dns: - image: andyshinn/dnsmasq:2.76 - ports: - - 53:53/udp - - 53:53/tcp - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - command: - - "-k" - - "-d" diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-config new file mode 100644 index 00000000000..f826c750372 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-config @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.xml_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem +CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs +MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-current-@project.version@.jar diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh new file mode 100755 index 00000000000..03caea39f75 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../../testlib.sh" + +start_docker_env + +execute_robot_test scm createmrenv.robot + + +#rm is the container name (resource manager) and not the rm command +execute_command_in_container rm sudo apk add --update py-pip +execute_command_in_container rm sudo pip install robotframework + +# reinitialize the directories to use +export OZONE_DIR=/opt/ozone +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../../testlib.sh" + +execute_robot_test rm ozonefs/hadoopo3fs.robot + +execute_robot_test rm -v hadoop.version:3.1.2 mapreduce.robot + + +stop_docker_env + +generate_report diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/.env b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/.env similarity index 90% rename from hadoop-ozone/dist/src/main/compose/ozone-mr/.env rename to hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/.env index 938101f8b35..d2c5aad9f98 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/.env +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/.env @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -HDDS_VERSION=${hdds.version} +HDDS_VERSION=@hdds.version@ HADOOP_IMAGE=apache/hadoop HADOOP_VERSION=3 -HADOOP_RUNNER_VERSION=${docker.ozone-runner.version} +HADOOP_RUNNER_VERSION=@docker.ozone-runner.version@ diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml new file mode 100644 index 00000000000..755b279749c --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + datanode: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + volumes: + - ../../..:/opt/hadoop + ports: + - 9864 + command: ["/opt/hadoop/bin/ozone","datanode"] + env_file: + - docker-config + - ../common-config + om: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + hostname: om + volumes: + - ../../..:/opt/hadoop + ports: + - 9874:9874 + environment: + WAITFOR: scm:9876 + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + env_file: + - docker-config + - ../common-config + command: ["/opt/hadoop/bin/ozone","om"] + s3g: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + hostname: s3g + volumes: + - ../../..:/opt/hadoop + ports: + - 9878:9878 + env_file: + - ./docker-config + - ../common-config + command: ["/opt/hadoop/bin/ozone","s3g"] + scm: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + hostname: scm + volumes: + - ../../..:/opt/hadoop + ports: + - 9876:9876 + env_file: + - docker-config + - ../common-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["/opt/hadoop/bin/ozone","scm"] + rm: + image: ${HADOOP_IMAGE}:${HADOOP_VERSION} + hostname: rm + volumes: + - ../../..:/opt/ozone + ports: + - 8088:8088 + env_file: + - ./docker-config + - ../common-config + environment: + HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-current-@project.version@.jar + command: ["yarn", "resourcemanager"] + nm: + image: ${HADOOP_IMAGE}:${HADOOP_VERSION} + hostname: nm + volumes: + - ../../..:/opt/ozone + env_file: + - ./docker-config + - ../common-config + environment: + HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-current-@project.version@.jar + WAIT_FOR: rm:8088 + command: ["yarn","nodemanager"] +# Optional section: comment out this part to get DNS resolution for all the containers. +# Add 127.0.0.1 (or the ip of your docker machine) to the resolv.conf to get local DNS resolution +# For all the containers (including resource managers and Node manager UI) +# dns: +# image: andyshinn/dnsmasq:2.76 +# ports: +# - 53:53/udp +# - 53:53/tcp +# volumes: +# - "/var/run/docker.sock:/var/run/docker.sock" +# command: +# - "-k" +# - "-d" diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-config new file mode 100644 index 00000000000..f826c750372 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-config @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.xml_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem +CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs +MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-current-@project.version@.jar diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/test.sh similarity index 90% rename from hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh rename to hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/test.sh index 892a540b467..b1910a57e1f 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/test.sh @@ -19,7 +19,7 @@ COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR # shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" +source "$COMPOSE_DIR/../../testlib.sh" start_docker_env @@ -27,8 +27,12 @@ execute_robot_test scm createmrenv.robot # reinitialize the directories to use export OZONE_DIR=/opt/ozone + # shellcheck source=/dev/null -source "$COMPOSE_DIR/../testlib.sh" +source "$COMPOSE_DIR/../../testlib.sh" + +execute_robot_test rm ozonefs/hadoopo3fs.robot + execute_robot_test rm mapreduce.robot stop_docker_env diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml deleted file mode 100644 index 179eb47d240..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - datanode: - image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - ports: - - 9864 - command: ["/opt/hadoop/bin/ozone","datanode"] - env_file: - - ./docker-config - om: - image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} - hostname: om - volumes: - - ../..:/opt/hadoop - ports: - - 9874 - environment: - ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","om"] - scm: - image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} - volumes: - - ../..:/opt/hadoop - ports: - - 9876 - env_file: - - ./docker-config - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["/opt/hadoop/bin/ozone","scm"] - hadoop32: - image: flokkr/hadoop:3.1.0 - volumes: - - ../..:/opt/ozone - env_file: - - ./docker-config - environment: - HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-current-@project.version@.jar - CORE-SITE.XML_fs.o3fs.impl: org.apache.hadoop.fs.ozone.OzoneFileSystem - command: ["watch","-n","100000","ls"] - hadoop31: - image: flokkr/hadoop:3.1.0 - volumes: - - ../..:/opt/ozone - env_file: - - ./docker-config - environment: - HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-legacy-@project.version@.jar - CORE-SITE.XML_fs.o3fs.impl: org.apache.hadoop.fs.ozone.OzoneFileSystem - command: ["watch","-n","100000","ls"] - hadoop29: - image: flokkr/hadoop:2.9.0 - volumes: - - ../..:/opt/ozone - env_file: - - ./docker-config - environment: - HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-legacy-@project.version@.jar - CORE-SITE.XML_fs.o3fs.impl: org.apache.hadoop.fs.ozone.BasicOzoneFileSystem - command: ["watch","-n","100000","ls"] - hadoop27: - image: flokkr/hadoop:2.7.3 - volumes: - - ../..:/opt/ozone - env_file: - - ./docker-config - environment: - HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-legacy-@project.version@.jar - CORE-SITE.XML_fs.o3fs.impl: org.apache.hadoop.fs.ozone.BasicOzoneFileSystem - command: ["watch","-n","100000","ls"] - spark: - image: flokkr/spark - volumes: - - ../..:/opt/ozone - env_file: - - ./docker-config - environment: - HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-legacy-@project.version@.jar - CORE-SITE.XML_fs.o3fs.impl: org.apache.hadoop.fs.ozone.BasicOzoneFileSystem - command: ["watch","-n","100000","ls"] diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config deleted file mode 100644 index 5da65050d01..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem -OZONE-SITE.XML_ozone.om.address=om -OZONE-SITE.XML_ozone.om.http-address=om:9874 -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True -OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.handler.type=distributed -OZONE-SITE.XML_ozone.scm.client.address=scm -OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s -OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -OZONE-SITE.XML_ozone.replication=1 -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout -LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender -LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR -LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN -LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.grpc.client.GrpcClientProtocolClient=WARN diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/hadoopo3fs.robot b/hadoop-ozone/dist/src/main/compose/ozonefs/hadoopo3fs.robot deleted file mode 100644 index 6da9d607082..00000000000 --- a/hadoop-ozone/dist/src/main/compose/ozonefs/hadoopo3fs.robot +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Test ozone fs usage from Hdfs and Spark -Library OperatingSystem -Library String -Resource ../../smoketest/env-compose.robot -Resource ../../smoketest/commonlib.robot - -*** Variables *** -${DATANODE_HOST} datanode - -*** Keywords *** - -Test hadoop dfs - [arguments] ${prefix} - ${random} = Generate Random String 5 [NUMBERS] - ${result} = Execute on host ${prefix} hdfs dfs -put /opt/hadoop/NOTICE.txt o3fs://bucket1.vol1/${prefix}-${random} - ${result} = Execute on host ${prefix} hdfs dfs -ls o3fs://bucket1.vol1/ - Should contain ${result} ${prefix}-${random} - -*** Test Cases *** - -Create bucket and volume to test - ${result} = Run tests on host scm createbucketenv.robot - -Test hadoop 3.1 - Test hadoop dfs hadoop31 - -Test hadoop 3.2 - Test hadoop dfs hadoop31 - -Test hadoop 2.9 - Test hadoop dfs hadoop29 - -Test hadoop 2.7 - Test hadoop dfs hadoop27 - -Test spark 2.3 - ${legacyjar} = Execute on host spark bash -c 'find /opt/ozone/share/ozone/lib/ -name *legacy*.jar' - ${postfix} = Generate Random String 5 [NUMBERS] - ${result} = Execute on host spark /opt/spark/bin/spark-submit --jars ${legacyjar} --class org.apache.spark.examples.DFSReadWriteTest /opt/spark//examples/jars/spark-examples_2.11-2.3.0.jar /opt/spark/README.md o3fs://bucket1.vol1/spark-${postfix} - diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index fab446000e7..410e059d19b 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -72,14 +72,19 @@ start_docker_env(){ ## @param robot test file or directory relative to the smoketest dir execute_robot_test(){ CONTAINER="$1" - TEST="$2" + shift 1 #Remove first argument which was the container name + # shellcheck disable=SC2206 + ARGUMENTS=($@) + TEST="${ARGUMENTS[${#ARGUMENTS[@]}-1]}" #Use last element as the test name + unset 'ARGUMENTS[${#ARGUMENTS[@]}-1]' #Remove the last element, remainings are the custom parameters TEST_NAME=$(basename "$TEST") TEST_NAME="$(basename "$COMPOSE_DIR")-${TEST_NAME%.*}" set +e OUTPUT_NAME="$COMPOSE_ENV_NAME-$TEST_NAME-$CONTAINER" OUTPUT_PATH="$RESULT_DIR_INSIDE/robot-$OUTPUT_NAME.xml" docker-compose -f "$COMPOSE_FILE" exec -T "$CONTAINER" mkdir -p "$RESULT_DIR_INSIDE" - docker-compose -f "$COMPOSE_FILE" exec -e SECURITY_ENABLED="${SECURITY_ENABLED}" -T "$CONTAINER" python -m robot --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST" + # shellcheck disable=SC2068 + docker-compose -f "$COMPOSE_FILE" exec -T -e SECURITY_ENABLED="${SECURITY_ENABLED}" "$CONTAINER" python -m robot ${ARGUMENTS[@]} --log NONE -N "$TEST_NAME" --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST" FULL_CONTAINER_NAME=$(docker-compose -f "$COMPOSE_FILE" ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}') docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/" @@ -87,6 +92,18 @@ execute_robot_test(){ } + +## @description Execute specific command in docker container +## @param container name +## @param specific command to execute +execute_command_in_container(){ + set -e + # shellcheck disable=SC2068 + docker-compose -f "$COMPOSE_FILE" exec -T $@ + set +e +} + + ## @description Stops a docker-compose based test environment (with saving the logs) stop_docker_env(){ docker-compose -f "$COMPOSE_FILE" logs > "$RESULT_DIR/docker-$OUTPUT_NAME.log" diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot index 1915e366512..8d12a526ea4 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot @@ -29,4 +29,4 @@ Test hadoop dfs ${random} = Generate Random String 5 [NUMBERS] ${result} = Execute hdfs dfs -put /opt/hadoop/NOTICE.txt o3fs://bucket1.vol1/${PREFIX}-${random} ${result} = Execute hdfs dfs -ls o3fs://bucket1.vol1/ - Should contain ${PREFIX}-${random} + Should contain ${result} ${PREFIX}-${random} diff --git a/hadoop-ozone/ozonefs-lib-legacy/pom.xml b/hadoop-ozone/ozonefs-lib-legacy/pom.xml index 14fcac36c02..fedd26b7847 100644 --- a/hadoop-ozone/ozonefs-lib-legacy/pom.xml +++ b/hadoop-ozone/ozonefs-lib-legacy/pom.xml @@ -64,8 +64,12 @@ slf4j-api,slf4j-log4j12,log4j-api,log4j-core,log4j,hadoop-ozone-filesystem + + ${project.build.directory}/dependency-maven-plugin-markers-lib + + include-ozonefs @@ -74,9 +78,33 @@ prepare-package target/classes - hadoop-ozone-filesystem + hadoop-ozone-filesystem,hadoop-ozone-common + compile META-INF/*.SF + + ${project.build.directory}/dependency-maven-plugin-markers-direct + + + + + + include-token + + unpack-dependencies + + prepare-package + + target/classes + hadoop-ozone-common,hadoop-hdds-common + compile + + org/apache/hadoop/ozone/security/OzoneTokenIdentifier.class,org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.class,org/apache/hadoop/ozone/protocol/proto/OzoneManagerProtocolProtos*,org/apache/hadoop/hdds/protocol/proto/HddsProtos* + + META-INF/*.SF + + ${project.build.directory}/dependency-maven-plugin-markers-token + diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java new file mode 100644 index 00000000000..52a8ede1b6d --- /dev/null +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.ozone; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DelegateToFileSystem; +import org.apache.hadoop.ozone.OzoneConsts; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * ozone implementation of AbstractFileSystem. + * This impl delegates to the OzoneFileSystem + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class BasicOzFs extends DelegateToFileSystem { + + public BasicOzFs(URI theUri, Configuration conf) + throws IOException, URISyntaxException { + super(theUri, new BasicOzoneFileSystem(), conf, + OzoneConsts.OZONE_URI_SCHEME, false); + } + +} diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index fa862d9677b..c76a210dbf3 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -28,6 +29,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -235,30 +238,55 @@ public boolean deleteObject(String keyName) { } } - public OzoneFileStatus getFileStatus(String pathKey) throws IOException { + public FileStatusAdapter getFileStatus(String key, URI uri, + Path qualifiedPath, String userName) + throws IOException { try { incrementCounter(Statistic.OBJECTS_QUERY); - return bucket.getFileStatus(pathKey); + OzoneFileStatus status = bucket.getFileStatus(key); + makeQualified(status, uri, qualifiedPath, userName); + return toFileStatusAdapter(status); + } catch (OMException e) { if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) { throw new - FileNotFoundException(pathKey + ": No such file or directory!"); + FileNotFoundException(key + ": No such file or directory!"); } throw e; } } + public void makeQualified(FileStatus status, URI uri, Path path, + String username) { + if (status instanceof OzoneFileStatus) { + ((OzoneFileStatus) status) + .makeQualified(uri, path, + username, username); + } + + } + @Override public Iterator listKeys(String pathKey) { incrementCounter(Statistic.OBJECTS_LIST); return new IteratorAdapter(bucket.listKeys(pathKey)); } - public List listStatus(String keyName, boolean recursive, - String startKey, long numEntries) throws IOException { + public List listStatus(String keyName, boolean recursive, + String startKey, long numEntries, URI uri, + Path workingDir, String username) throws IOException { try { incrementCounter(Statistic.OBJECTS_LIST); - return bucket.listStatus(keyName, recursive, startKey, numEntries); + List statuses = bucket + .listStatus(keyName, recursive, startKey, numEntries); + + List result = new ArrayList<>(); + for (OzoneFileStatus status : statuses) { + Path qualifiedPath = status.getPath().makeQualified(uri, workingDir); + makeQualified(status, uri, qualifiedPath, username); + result.add(toFileStatusAdapter(status)); + } + return result; } catch (OMException e) { if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) { throw new FileNotFoundException(e.getMessage()); @@ -372,4 +400,20 @@ public BasicKeyInfo next() { } } } + + private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status) { + return new FileStatusAdapter( + status.getLen(), + status.getPath(), + status.isDirectory(), + status.getReplication(), + status.getBlockSize(), + status.getModificationTime(), + status.getAccessTime(), + status.getPermission().toShort(), + status.getOwner(), + status.getGroup(), + status.getPath() + ); + } } diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index da3f1ac78f5..660eb9a2e2e 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -24,11 +24,12 @@ import java.net.URISyntaxException; import java.util.EnumSet; import java.util.Iterator; -import java.util.List; import java.util.LinkedList; +import java.util.List; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -42,7 +43,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; @@ -497,13 +497,18 @@ public FileStatus[] listStatus(Path f) throws IOException { statistics.incrementReadOps(1); LOG.trace("listStatus() path:{}", f); int numEntries = LISTING_PAGE_SIZE; - LinkedList statuses = new LinkedList<>(); - List tmpStatusList; + LinkedList statuses = new LinkedList<>(); + List tmpStatusList; String startKey = ""; do { tmpStatusList = - adapter.listStatus(pathToKey(f), false, startKey, numEntries); + adapter.listStatus(pathToKey(f), false, startKey, numEntries, uri, + workingDir, getUsername()) + .stream() + .map(this::convertFileStatus) + .collect(Collectors.toList()); + if (!tmpStatusList.isEmpty()) { if (startKey.isEmpty()) { statuses.addAll(tmpStatusList); @@ -517,10 +522,7 @@ public FileStatus[] listStatus(Path f) throws IOException { // exhausted. } while (tmpStatusList.size() == numEntries); - for (OzoneFileStatus status : statuses) { - status.makeQualified(uri, status.getPath().makeQualified(uri, workingDir), - getUsername(), getUsername()); - } + return statuses.toArray(new FileStatus[0]); } @@ -624,9 +626,9 @@ public FileStatus getFileStatus(Path f) throws IOException { LOG.trace("getFileStatus() path:{}", f); Path qualifiedPath = f.makeQualified(uri, workingDir); String key = pathToKey(qualifiedPath); - - return adapter.getFileStatus(key) - .makeQualified(uri, qualifiedPath, getUsername(), getUsername()); + FileStatus status = convertFileStatus( + adapter.getFileStatus(key, uri, qualifiedPath, getUsername())); + return status; } /** @@ -759,4 +761,30 @@ public boolean isNumber(String number) { } return true; } + + private FileStatus convertFileStatus( + FileStatusAdapter fileStatusAdapter) { + + Path symLink = null; + try { + fileStatusAdapter.getSymlink(); + } catch (Exception ex) { + //NOOP: If not symlink symlink remains null. + } + + return new FileStatus( + fileStatusAdapter.getLength(), + fileStatusAdapter.isDir(), + fileStatusAdapter.getBlockReplication(), + fileStatusAdapter.getBlocksize(), + fileStatusAdapter.getModificationTime(), + fileStatusAdapter.getAccessTime(), + new FsPermission(fileStatusAdapter.getPermission()), + fileStatusAdapter.getOwner(), + fileStatusAdapter.getGroup(), + symLink, + fileStatusAdapter.getPath() + ); + + } } diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java new file mode 100644 index 00000000000..91597839340 --- /dev/null +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.ozone; + +import org.apache.hadoop.fs.Path; + +/** + * Class to hold the internal information of a FileStatus. + *

+ * As FileStatus class is not compatible between 3.x and 2.x hadoop we can + * use this adapter to hold all the required information. Hadoop 3.x FileStatus + * information can be converted to this class, and this class can be used to + * create hadoop 2.x FileStatus. + *

+ * FileStatus (Hadoop 3.x) --> FileStatusAdapter --> FileStatus (Hadoop 2.x) + */ +public final class FileStatusAdapter { + + private final long length; + private final Path path; + private final boolean isdir; + private final short blockReplication; + private final long blocksize; + private final long modificationTime; + private final long accessTime; + private final short permission; + private final String owner; + private final String group; + private final Path symlink; + + @SuppressWarnings("checkstyle:ParameterNumber") + public FileStatusAdapter(long length, Path path, boolean isdir, + short blockReplication, long blocksize, long modificationTime, + long accessTime, short permission, String owner, + String group, Path symlink) { + this.length = length; + this.path = path; + this.isdir = isdir; + this.blockReplication = blockReplication; + this.blocksize = blocksize; + this.modificationTime = modificationTime; + this.accessTime = accessTime; + this.permission = permission; + this.owner = owner; + this.group = group; + this.symlink = symlink; + } + + public Path getPath() { + return path; + } + + public boolean isDir() { + return isdir; + } + + public short getBlockReplication() { + return blockReplication; + } + + public long getBlocksize() { + return blocksize; + } + + public long getModificationTime() { + return modificationTime; + } + + public long getAccessTime() { + return accessTime; + } + + public short getPermission() { + return permission; + } + + public String getOwner() { + return owner; + } + + public String getGroup() { + return group; + } + + public Path getSymlink() { + return symlink; + } + + public long getLength() { + return length; + } + +} diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FilteredClassLoader.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FilteredClassLoader.java index 2643cab41b8..a90797efdd8 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FilteredClassLoader.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/FilteredClassLoader.java @@ -52,17 +52,19 @@ public class FilteredClassLoader extends URLClassLoader { public FilteredClassLoader(URL[] urls, ClassLoader parent) { super(urls, null); + delegatedClasses.add("org.apache.hadoop.crypto.key.KeyProvider"); delegatedClasses.add("org.apache.hadoop.fs.ozone.OzoneClientAdapter"); + delegatedClasses.add("org.apache.hadoop.fs.ozone.FileStatusAdapter"); delegatedClasses.add("org.apache.hadoop.security.token.Token"); delegatedClasses.add("org.apache.hadoop.fs.ozone.BasicKeyInfo"); delegatedClasses.add("org.apache.hadoop.fs.ozone.OzoneFSOutputStream"); delegatedClasses.add("org.apache.hadoop.fs.ozone.OzoneFSStorageStatistics"); delegatedClasses.add("org.apache.hadoop.fs.ozone.Statistic"); delegatedClasses.add("org.apache.hadoop.fs.Seekable"); - + delegatedClasses.add("org.apache.hadoop.io.Text"); + delegatedClasses.add("org.apache.hadoop.fs.Path"); delegatedClasses.addAll(StringUtils.getTrimmedStringCollection( System.getenv("HADOOP_OZONE_DELEGATED_CLASSES"))); - this.delegate = parent; systemClassLoader = getSystemClassLoader(); diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java index 390925c0780..0ae8c8ff1d8 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java @@ -17,18 +17,17 @@ */ package org.apache.hadoop.fs.ozone; -import org.apache.hadoop.crypto.key.KeyProvider; -import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; -import org.apache.hadoop.security.token.Token; - -import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; - import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.util.Iterator; import java.util.List; +import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; +import org.apache.hadoop.security.token.Token; + /** * Lightweight adapter to separate hadoop/ozone classes. *

@@ -53,8 +52,9 @@ OzoneFSOutputStream createFile(String key, boolean overWrite, Iterator listKeys(String pathKey); - List listStatus(String keyName, boolean recursive, - String startKey, long numEntries) throws IOException; + List listStatus(String keyName, boolean recursive, + String startKey, long numEntries, URI uri, + Path workingDir, String username) throws IOException; Token getDelegationToken(String renewer) throws IOException; @@ -65,5 +65,7 @@ Token getDelegationToken(String renewer) String getCanonicalServiceName(); - OzoneFileStatus getFileStatus(String pathKey) throws IOException; + FileStatusAdapter getFileStatus(String key, URI uri, + Path qualifiedPath, String userName) throws IOException; + }