YARN-8873. [YARN-8811] Add CSI java-based client library. Contributed by Weiwei Yang.
This commit is contained in:
Sunil G
parent
e3cca12048
commit
0efddd85f0
|
|
@ -0,0 +1,19 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<FindBugsFilter>
|
||||||
|
|
||||||
|
</FindBugsFilter>
|
||||||
|
|
@ -0,0 +1,209 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. See accompanying LICENSE file.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<parent>
|
||||||
|
<artifactId>hadoop-yarn</artifactId>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<version>3.3.0-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<artifactId>hadoop-yarn-csi</artifactId>
|
||||||
|
<name>Apache Hadoop YARN CSI</name>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<protobuf.version>3.6.1</protobuf.version>
|
||||||
|
<guava.version>20.0</guava.version>
|
||||||
|
<grpc.version>1.15.1</grpc.version>
|
||||||
|
<netty-all.version>4.1.27.Final</netty-all.version>
|
||||||
|
<os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.google.guava</groupId>
|
||||||
|
<artifactId>guava</artifactId>
|
||||||
|
<version>${guava.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.google.protobuf</groupId>
|
||||||
|
<artifactId>protobuf-java</artifactId>
|
||||||
|
<version>${protobuf.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.netty</groupId>
|
||||||
|
<artifactId>netty-all</artifactId>
|
||||||
|
<version>${netty-all.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.grpc</groupId>
|
||||||
|
<artifactId>grpc-core</artifactId>
|
||||||
|
<version>${grpc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.grpc</groupId>
|
||||||
|
<artifactId>grpc-protobuf</artifactId>
|
||||||
|
<version>${grpc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.grpc</groupId>
|
||||||
|
<artifactId>grpc-stub</artifactId>
|
||||||
|
<version>${grpc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.grpc</groupId>
|
||||||
|
<artifactId>grpc-netty</artifactId>
|
||||||
|
<version>${grpc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>commons-logging</groupId>
|
||||||
|
<artifactId>commons-logging</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-common</artifactId>
|
||||||
|
<type>test-jar</type>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<extensions>
|
||||||
|
<extension>
|
||||||
|
<groupId>kr.motd.maven</groupId>
|
||||||
|
<artifactId>os-maven-plugin</artifactId>
|
||||||
|
<version>${os-maven-plugin.version}</version>
|
||||||
|
</extension>
|
||||||
|
</extensions>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-dependency-plugin</artifactId>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-clean-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<filesets>
|
||||||
|
<fileset>
|
||||||
|
<directory>target/</directory>
|
||||||
|
</fileset>
|
||||||
|
</filesets>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.xolstice.maven.plugins</groupId>
|
||||||
|
<artifactId>protobuf-maven-plugin</artifactId>
|
||||||
|
<version>${protobuf-maven-plugin.version}</version>
|
||||||
|
<configuration>
|
||||||
|
<protocArtifact>com.google.protobuf:protoc:3.6.1:exe:${os.detected.classifier}</protocArtifact>
|
||||||
|
<pluginId>grpc-java</pluginId>
|
||||||
|
<pluginArtifact>io.grpc:protoc-gen-grpc-java:1.15.1:exe:${os.detected.classifier}</pluginArtifact>
|
||||||
|
</configuration>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<goals>
|
||||||
|
<goal>compile</goal>
|
||||||
|
<goal>compile-custom</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-shade-plugin</artifactId>
|
||||||
|
<version>${maven-shade-plugin.version}</version>
|
||||||
|
<configuration>
|
||||||
|
<createDependencyReducedPom>false</createDependencyReducedPom>
|
||||||
|
</configuration>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>shade</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<relocations>
|
||||||
|
<relocation>
|
||||||
|
<pattern>io.grpc</pattern>
|
||||||
|
<shadedPattern>csi.io.grpc</shadedPattern>
|
||||||
|
</relocation>
|
||||||
|
<relocation>
|
||||||
|
<!-- this includes protobuf-java and guava libraries -->
|
||||||
|
<pattern>com.google</pattern>
|
||||||
|
<shadedPattern>csi.com.google</shadedPattern>
|
||||||
|
</relocation>
|
||||||
|
<relocation>
|
||||||
|
<pattern>io.netty</pattern>
|
||||||
|
<shadedPattern>csi.io.netty</shadedPattern>
|
||||||
|
</relocation>
|
||||||
|
</relocations>
|
||||||
|
<transformers>
|
||||||
|
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
|
||||||
|
</transformers>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<!-- We need to rename the native library file in order to let shaded classes to load them -->
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-antrun-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>unpack</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<configuration>
|
||||||
|
<target>
|
||||||
|
<echo message="Unpack hadoop-yarn-csi jar file"/>
|
||||||
|
<unzip src="${project.build.directory}/${project.artifactId}-${project.version}.jar"
|
||||||
|
dest="${project.build.directory}/unpacked/"/>
|
||||||
|
<echo message="Append the shaded prefix to netty's native file in META-INF"/>
|
||||||
|
<move file="${project.build.directory}/unpacked/META-INF/native/libnetty_transport_native_epoll_x86_64.so"
|
||||||
|
tofile="${project.build.directory}/unpacked/META-INF/native/libcsi_netty_transport_native_epoll_x86_64.so" />
|
||||||
|
<echo message="Re-pack the jar"/>
|
||||||
|
<jar destfile="${project.build.directory}/${project.artifactId}-${project.version}.jar"
|
||||||
|
basedir="${project.build.directory}/unpacked"/>
|
||||||
|
</target>
|
||||||
|
</configuration>
|
||||||
|
<goals>
|
||||||
|
<goal>run</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.rat</groupId>
|
||||||
|
<artifactId>apache-rat-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<excludes>
|
||||||
|
<exclude>target/generated-sources/**</exclude>
|
||||||
|
<exclude>target/surefire-reports/**</exclude>
|
||||||
|
<exclude>target/protoc-dependencies/**</exclude>
|
||||||
|
</excludes>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.csi.client;
|
||||||
|
|
||||||
|
import csi.v0.Csi.GetPluginInfoResponse;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* General interface for a CSI client. This interface defines all APIs
|
||||||
|
* that CSI spec supports, including both identity/controller/node service
|
||||||
|
* APIs.
|
||||||
|
*/
|
||||||
|
public interface CsiClient {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets some basic info about the CSI plugin, including the driver name,
|
||||||
|
* version and optionally some manifest info.
|
||||||
|
* @return {@link GetPluginInfoResponse}
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
GetPluginInfoResponse getPluginInfo() throws IOException;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.csi.client;
|
||||||
|
|
||||||
|
import csi.v0.Csi.GetPluginInfoRequest;
|
||||||
|
import csi.v0.Csi.GetPluginInfoResponse;
|
||||||
|
import org.apache.hadoop.yarn.csi.utils.GrpcHelper;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A CSI client implementation that communicates with a CSI driver via
|
||||||
|
* unix domain socket. It leverages gRPC blocking stubs to synchronize
|
||||||
|
* the call with CSI driver. CSI spec is designed as a set of synchronized
|
||||||
|
* APIs, in order to make the call idempotent for failure recovery,
|
||||||
|
* so the client does the same.
|
||||||
|
*/
|
||||||
|
public class CsiClientImpl implements CsiClient {
|
||||||
|
|
||||||
|
private final SocketAddress address;
|
||||||
|
|
||||||
|
public CsiClientImpl(String address) {
|
||||||
|
this.address = GrpcHelper.getSocketAddress(address);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetPluginInfoResponse getPluginInfo() throws IOException {
|
||||||
|
try (CsiGrpcClient client = CsiGrpcClient.newBuilder()
|
||||||
|
.setDomainSocketAddress(address).build()) {
|
||||||
|
GetPluginInfoRequest request = GetPluginInfoRequest.getDefaultInstance();
|
||||||
|
return client.createIdentityBlockingStub().getPluginInfo(request);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,127 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.csi.client;
|
||||||
|
|
||||||
|
import csi.v0.ControllerGrpc;
|
||||||
|
import csi.v0.IdentityGrpc;
|
||||||
|
import csi.v0.NodeGrpc;
|
||||||
|
import io.grpc.ManagedChannel;
|
||||||
|
import io.grpc.netty.NettyChannelBuilder;
|
||||||
|
import io.netty.channel.epoll.EpollDomainSocketChannel;
|
||||||
|
import io.netty.channel.epoll.EpollEventLoopGroup;
|
||||||
|
import io.netty.channel.unix.DomainSocketAddress;
|
||||||
|
import io.netty.util.concurrent.DefaultThreadFactory;
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A CSI gRPC client, it connects a CSI driver via a given unix domain socket.
|
||||||
|
*/
|
||||||
|
public final class CsiGrpcClient implements AutoCloseable {
|
||||||
|
|
||||||
|
private static final Log LOG = LogFactory.getLog(CsiGrpcClient.class);
|
||||||
|
|
||||||
|
private final ManagedChannel channel;
|
||||||
|
|
||||||
|
private CsiGrpcClient(ManagedChannel channel) {
|
||||||
|
this.channel = channel;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static GrpcClientBuilder newBuilder() {
|
||||||
|
return new GrpcClientBuilder();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Grpc Client builder.
|
||||||
|
*/
|
||||||
|
public static class GrpcClientBuilder {
|
||||||
|
|
||||||
|
private SocketAddress socket;
|
||||||
|
|
||||||
|
public GrpcClientBuilder setDomainSocketAddress(SocketAddress address) {
|
||||||
|
this.socket = address;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
private ManagedChannel getChannel(SocketAddress socketAddress)
|
||||||
|
throws IOException {
|
||||||
|
DefaultThreadFactory tf = new DefaultThreadFactory(
|
||||||
|
"yarn-csi-client-", true);
|
||||||
|
EpollEventLoopGroup loopGroup = new EpollEventLoopGroup(0, tf);
|
||||||
|
if (socketAddress instanceof DomainSocketAddress) {
|
||||||
|
ManagedChannel channel = NettyChannelBuilder.forAddress(socketAddress)
|
||||||
|
.channelType(EpollDomainSocketChannel.class)
|
||||||
|
.eventLoopGroup(loopGroup)
|
||||||
|
.usePlaintext()
|
||||||
|
.build();
|
||||||
|
return channel;
|
||||||
|
} else {
|
||||||
|
throw new IOException("Currently only unix domain socket is supported");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public CsiGrpcClient build() throws IOException {
|
||||||
|
ManagedChannel socketChannel = getChannel(socket);
|
||||||
|
return new CsiGrpcClient(socketChannel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shutdown the communication channel gracefully,
|
||||||
|
* wait for 5 seconds before it is enforced.
|
||||||
|
* @throws InterruptedException
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
try {
|
||||||
|
this.channel.shutdown().awaitTermination(5, TimeUnit.SECONDS);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
LOG.error("Failed to gracefully shutdown"
|
||||||
|
+ " gRPC communication channel in 5 seconds", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a blocking stub for CSI identity plugin on the given channel.
|
||||||
|
* @return the blocking stub
|
||||||
|
*/
|
||||||
|
public IdentityGrpc.IdentityBlockingStub createIdentityBlockingStub() {
|
||||||
|
return IdentityGrpc.newBlockingStub(channel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a blocking stub for CSI controller plugin on the given channel.
|
||||||
|
* @return the blocking stub
|
||||||
|
*/
|
||||||
|
public ControllerGrpc.ControllerBlockingStub createControllerBlockingStub(){
|
||||||
|
return ControllerGrpc.newBlockingStub(channel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a blocking stub for CSI node plugin on the given channel.
|
||||||
|
* @return the blocking stub
|
||||||
|
*/
|
||||||
|
public NodeGrpc.NodeBlockingStub createNodeBlockingStub() {
|
||||||
|
return NodeGrpc.newBlockingStub(channel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* This package contains CSI client classes.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.csi.client;
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* This package contains classes for CSI.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.csi;
|
||||||
|
|
@ -0,0 +1,52 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.csi.utils;
|
||||||
|
|
||||||
|
import io.netty.channel.unix.DomainSocketAddress;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper classes for gRPC utility functions.
|
||||||
|
*/
|
||||||
|
public final class GrpcHelper {
|
||||||
|
|
||||||
|
protected static final String UNIX_DOMAIN_SOCKET_PREFIX = "unix://";
|
||||||
|
|
||||||
|
private GrpcHelper() {
|
||||||
|
// hide constructor for utility class
|
||||||
|
}
|
||||||
|
|
||||||
|
public static SocketAddress getSocketAddress(String value) {
|
||||||
|
if (value.startsWith(UNIX_DOMAIN_SOCKET_PREFIX)) {
|
||||||
|
String filePath = value.substring(UNIX_DOMAIN_SOCKET_PREFIX.length());
|
||||||
|
File file = new File(filePath);
|
||||||
|
if (!file.isAbsolute()) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Unix domain socket file path must be absolute, file: " + value);
|
||||||
|
}
|
||||||
|
// Create the SocketAddress referencing the file.
|
||||||
|
return new DomainSocketAddress(file);
|
||||||
|
} else {
|
||||||
|
throw new IllegalArgumentException("Given address " + value
|
||||||
|
+ " is not a valid unix domain socket path");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* This package contains utility classes for CSI.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.csi.utils;
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,65 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.csi.client;
|
||||||
|
|
||||||
|
import io.grpc.Server;
|
||||||
|
import io.grpc.netty.NettyServerBuilder;
|
||||||
|
import io.netty.channel.epoll.EpollEventLoopGroup;
|
||||||
|
import io.netty.channel.epoll.EpollServerDomainSocketChannel;
|
||||||
|
import org.apache.hadoop.yarn.csi.utils.GrpcHelper;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.logging.Logger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A fake implementation of CSI driver.
|
||||||
|
* This is for testing purpose only.
|
||||||
|
*/
|
||||||
|
public class FakeCsiDriver {
|
||||||
|
|
||||||
|
private static final Logger LOG = Logger
|
||||||
|
.getLogger(FakeCsiDriver.class.getName());
|
||||||
|
|
||||||
|
private Server server;
|
||||||
|
private String socketAddress;
|
||||||
|
|
||||||
|
public FakeCsiDriver(String socketAddress) {
|
||||||
|
this.socketAddress = socketAddress;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void start() throws IOException {
|
||||||
|
EpollEventLoopGroup group = new EpollEventLoopGroup();
|
||||||
|
server = NettyServerBuilder
|
||||||
|
.forAddress(GrpcHelper.getSocketAddress(socketAddress))
|
||||||
|
.channelType(EpollServerDomainSocketChannel.class)
|
||||||
|
.workerEventLoopGroup(group)
|
||||||
|
.bossEventLoopGroup(group)
|
||||||
|
.addService(new FakeCsiIdentityService())
|
||||||
|
.build();
|
||||||
|
server.start();
|
||||||
|
LOG.info("Server started, listening on " + socketAddress);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void stop() {
|
||||||
|
if (server != null) {
|
||||||
|
server.shutdown();
|
||||||
|
LOG.info("Server has been shutdown");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.csi.client;
|
||||||
|
|
||||||
|
import csi.v0.Csi.GetPluginInfoRequest;
|
||||||
|
import csi.v0.Csi.GetPluginInfoResponse;
|
||||||
|
import csi.v0.IdentityGrpc;
|
||||||
|
import io.grpc.stub.StreamObserver;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A fake implementation of CSI identity plugin gRPC service.
|
||||||
|
* This is for testing purpose only.
|
||||||
|
*/
|
||||||
|
public class FakeCsiIdentityService extends IdentityGrpc.IdentityImplBase {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void getPluginInfo(GetPluginInfoRequest request,
|
||||||
|
StreamObserver<GetPluginInfoResponse> responseObserver) {
|
||||||
|
GetPluginInfoResponse response = GetPluginInfoResponse.newBuilder()
|
||||||
|
.setName("fake-csi-identity-service")
|
||||||
|
.setVendorVersion("0.1")
|
||||||
|
.build();
|
||||||
|
responseObserver.onNext(response);
|
||||||
|
responseObserver.onCompleted();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,77 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.csi.client;
|
||||||
|
|
||||||
|
import csi.v0.Csi;
|
||||||
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Assume;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test class for CSI client.
|
||||||
|
*/
|
||||||
|
public class TestCsiClient {
|
||||||
|
|
||||||
|
private static File testRoot = null;
|
||||||
|
private static String domainSocket = null;
|
||||||
|
private static FakeCsiDriver driver = null;
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void setUp() throws IOException {
|
||||||
|
testRoot = GenericTestUtils.getTestDir("csi-test");
|
||||||
|
File socketPath = new File(testRoot, "csi.sock");
|
||||||
|
FileUtils.forceMkdirParent(socketPath);
|
||||||
|
domainSocket = "unix://" + socketPath.getAbsolutePath();
|
||||||
|
driver = new FakeCsiDriver(domainSocket);
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void tearDown() throws IOException {
|
||||||
|
if (testRoot != null) {
|
||||||
|
FileUtils.deleteDirectory(testRoot);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void beforeMethod() {
|
||||||
|
// Skip tests on non-linux systems
|
||||||
|
String osName = System.getProperty("os.name").toLowerCase();
|
||||||
|
Assume.assumeTrue(osName.contains("nix") || osName.contains("nux"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testIdentityService() throws IOException {
|
||||||
|
try {
|
||||||
|
driver.start();
|
||||||
|
CsiClient client = new CsiClientImpl(domainSocket);
|
||||||
|
Csi.GetPluginInfoResponse response = client.getPluginInfo();
|
||||||
|
Assert.assertEquals("fake-csi-identity-service", response.getName());
|
||||||
|
} finally {
|
||||||
|
driver.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* This package contains classes for CSI client library testing.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.csi.client;
|
||||||
|
|
@ -244,5 +244,6 @@
|
||||||
<module>hadoop-yarn-client</module>
|
<module>hadoop-yarn-client</module>
|
||||||
<module>hadoop-yarn-registry</module>
|
<module>hadoop-yarn-registry</module>
|
||||||
<module>hadoop-yarn-ui</module>
|
<module>hadoop-yarn-ui</module>
|
||||||
|
<module>hadoop-yarn-csi</module>
|
||||||
</modules>
|
</modules>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue