HDDS-177. Create a releasable ozonefs artifact
Contributed by Marton, Elek.
This commit is contained in:
Anu Engineer
parent
ca14fec02f
commit
e16e5b307d
|
|
@ -145,6 +145,8 @@ run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-$
|
|||
run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" .
|
||||
run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" .
|
||||
run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" .
|
||||
mkdir -p "./share/hadoop/ozonefs"
|
||||
cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar"
|
||||
# Optional documentation, could be missing
|
||||
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ksm/
|
||||
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/
|
||||
|
|
@ -153,5 +155,5 @@ cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdd
|
|||
mkdir -p ./share/hadoop/mapreduce
|
||||
mkdir -p ./share/hadoop/yarn
|
||||
echo
|
||||
echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}"
|
||||
echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone"
|
||||
echo
|
||||
|
|
|
|||
|
|
@ -265,6 +265,11 @@
|
|||
<artifactId>hadoop-ozone-docs</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-ozone-filesystem</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<plugins>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,71 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
version: "3"
|
||||
services:
|
||||
namenode:
|
||||
image: apache/hadoop-runner
|
||||
hostname: namenode
|
||||
volumes:
|
||||
- ${OZONEDIR}:/opt/hadoop
|
||||
ports:
|
||||
- 9870
|
||||
environment:
|
||||
ENSURE_NAMENODE_DIR: /data/namenode
|
||||
env_file:
|
||||
- ./docker-config
|
||||
command: ["/opt/hadoop/bin/hdfs","namenode"]
|
||||
datanode:
|
||||
image: apache/hadoop-runner
|
||||
volumes:
|
||||
- ${OZONEDIR}:/opt/hadoop
|
||||
ports:
|
||||
- 9864
|
||||
command: ["/opt/hadoop/bin/ozone","datanode"]
|
||||
env_file:
|
||||
- ./docker-config
|
||||
ksm:
|
||||
image: apache/hadoop-runner
|
||||
hostname: ksm
|
||||
volumes:
|
||||
- ${OZONEDIR}:/opt/hadoop
|
||||
ports:
|
||||
- 9874
|
||||
environment:
|
||||
ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
|
||||
env_file:
|
||||
- ./docker-config
|
||||
command: ["/opt/hadoop/bin/ozone","ksm"]
|
||||
scm:
|
||||
image: apache/hadoop-runner
|
||||
volumes:
|
||||
- ${OZONEDIR}:/opt/hadoop
|
||||
ports:
|
||||
- 9876
|
||||
env_file:
|
||||
- ./docker-config
|
||||
environment:
|
||||
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
|
||||
command: ["/opt/hadoop/bin/ozone","scm"]
|
||||
hadooplast:
|
||||
image: flokkr/hadoop:3.1.0
|
||||
volumes:
|
||||
- ${OZONEDIR}:/opt/ozone
|
||||
env_file:
|
||||
- ./docker-config
|
||||
environment:
|
||||
HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar
|
||||
command: ["watch","-n","100000","ls"]
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
|
||||
CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
|
||||
OZONE-SITE.XML_ozone.ksm.address=ksm
|
||||
OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
|
||||
OZONE-SITE.XML_ozone.scm.names=scm
|
||||
OZONE-SITE.XML_ozone.enabled=True
|
||||
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
|
||||
OZONE-SITE.XML_ozone.scm.block.client.address=scm
|
||||
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
|
||||
OZONE-SITE.XML_ozone.handler.type=distributed
|
||||
OZONE-SITE.XML_ozone.scm.client.address=scm
|
||||
OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
|
||||
OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
|
||||
HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
|
||||
HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
|
||||
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
|
||||
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
|
||||
HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
|
||||
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
|
||||
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
|
||||
LOG4J.PROPERTIES_log4j.category.org.apache.hadoop.util.NativeCodeLoader=ERROR
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
*** Settings ***
|
||||
Documentation Ozonefs test
|
||||
Library OperatingSystem
|
||||
Suite Setup Startup Ozone cluster with size 5
|
||||
Suite Teardown Teardown Ozone cluster
|
||||
Resource ../commonlib.robot
|
||||
|
||||
*** Variables ***
|
||||
${COMPOSEFILE} ${CURDIR}/docker-compose.yaml
|
||||
${PROJECTDIR} ${CURDIR}/../../../../../..
|
||||
|
||||
|
||||
*** Test Cases ***
|
||||
Create volume and bucket
|
||||
Execute on datanode ozone oz -createVolume http://ksm/fstest -user bilbo -quota 100TB -root
|
||||
Execute on datanode ozone oz -createBucket http://ksm/fstest/bucket1
|
||||
|
||||
Check volume from ozonefs
|
||||
${result} = Execute on hadooplast hdfs dfs -ls o3://bucket1.fstest/
|
||||
|
||||
Create directory from ozonefs
|
||||
Execute on hadooplast hdfs dfs -mkdir -p o3://bucket1.fstest/testdir/deep
|
||||
${result} = Execute on ksm ozone oz -listKey o3://ksm/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
|
||||
Should contain ${result} testdir/deep
|
||||
|
|
@ -18,14 +18,13 @@
|
|||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-project</artifactId>
|
||||
<version>3.2.0-SNAPSHOT</version>
|
||||
<relativePath>../../hadoop-project</relativePath>
|
||||
<artifactId>hadoop-ozone</artifactId>
|
||||
<version>0.2.1-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>hadoop-ozone-filesystem</artifactId>
|
||||
<name>Apache Hadoop Ozone FileSystem</name>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<version>0.2.1-SNAPSHOT</version>
|
||||
<properties>
|
||||
<file.encoding>UTF-8</file.encoding>
|
||||
<downloadSources>true</downloadSources>
|
||||
|
|
@ -44,6 +43,46 @@
|
|||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>3.1.1</version>
|
||||
<configuration>
|
||||
<artifactSet>
|
||||
<includes>
|
||||
<include>com.google.guava:guava:jar</include>
|
||||
<include>org.slf4j:slf4j-api:jar</include>
|
||||
<include>com.google.protobuf:protobuf-java</include>
|
||||
<include>com.nimbusds:nimbus-jose-jwt:jar</include>
|
||||
<include>com.github.stephenc.jcip:jcip-annotations</include>
|
||||
<include>com.google.code.findbugs:jsr305:jar</include>
|
||||
<include>org.apache.hadoop:hadoop-ozone-client</include>
|
||||
<include>org.apache.hadoop:hadoop-hdds-client</include>
|
||||
<include>org.apache.hadoop:hadoop-hdds-common</include>
|
||||
<include>org.fusesource.leveldbjni:leveldbjni-all</include>
|
||||
<include>org.apache.ratis:ratis-server</include>
|
||||
<include>org.apache.ratis:ratis-proto-shaded:jar</include>
|
||||
<include>com.google.auto.value:auto-value-annotations</include>
|
||||
<include>com.squareup:javapoet:jar</include>
|
||||
<include>org.jctools:jctools-core</include>
|
||||
<include>org.apache.ratis:ratis-common</include>
|
||||
<include>org.apache.ratis:ratis-client</include>
|
||||
<include>org.apache.ratis:ratis-netty</include>
|
||||
<include>org.apache.ratis:ratis-grpc</include>
|
||||
<include>org.rocksdb:rocksdbjni</include>
|
||||
<include>org.apache.hadoop:hadoop-ozone-common</include>
|
||||
</includes>
|
||||
</artifactSet>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
|
|
@ -56,7 +95,9 @@
|
|||
</goals>
|
||||
<configuration>
|
||||
<!-- build a shellprofile -->
|
||||
<outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
|
||||
<outputFile>
|
||||
${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt
|
||||
</outputFile>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
|
|
@ -83,7 +124,6 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdds-common</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
|
|
@ -108,12 +148,10 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdds-client</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-ozone-common</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
|
|
@ -129,7 +167,6 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-ozone-client</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
|
|
@ -34,6 +34,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|||
<module>client</module>
|
||||
<module>ozone-manager</module>
|
||||
<module>tools</module>
|
||||
<module>ozonefs</module>
|
||||
<module>integration-test</module>
|
||||
<module>objectstore-service</module>
|
||||
<module>docs</module>
|
||||
|
|
|
|||
|
|
@ -586,7 +586,11 @@
|
|||
<artifactId>hadoop-ozone-docs</artifactId>
|
||||
<version>${hdds.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-ozone-filesystem</artifactId>
|
||||
<version>${hdds.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-hdds-common</artifactId>
|
||||
|
|
|
|||
|
|
@ -192,20 +192,5 @@
|
|||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>hdds</id>
|
||||
<activation>
|
||||
<activeByDefault>false</activeByDefault>
|
||||
</activation>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-ozone-filesystem</artifactId>
|
||||
<scope>compile</scope>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -67,15 +67,4 @@
|
|||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>hdds</id>
|
||||
<activation>
|
||||
<activeByDefault>false</activeByDefault>
|
||||
</activation>
|
||||
<modules>
|
||||
<module>hadoop-ozone</module>
|
||||
</modules>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
|
|
|||
Loading…
Reference in New Issue