From c1d64d60f6ef3cb9ed89669501ca5b1efbab3c28 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Fri, 11 May 2018 11:08:45 -0700 Subject: [PATCH] HDDS-39. Ozone: Compile Ozone/HDFS/Cblock protobuf files with proto3 compiler using maven protoc plugin. Contributed by Mukul Kumar Singh. --- .../apache/hadoop/hdds/scm/XceiverClient.java | 16 +++--- .../hadoop/hdds/scm/XceiverClientHandler.java | 10 ++-- .../hdds/scm/XceiverClientInitializer.java | 18 ++++--- .../hadoop/hdds/scm/XceiverClientMetrics.java | 2 +- .../hadoop/hdds/scm/XceiverClientRatis.java | 18 +++---- .../scm/client/ContainerOperationClient.java | 5 +- .../hdds/scm/storage/ChunkInputStream.java | 6 +-- .../hdds/scm/storage/ChunkOutputStream.java | 8 +-- hadoop-hdds/common/pom.xml | 52 ++++++++++++++++++- .../apache/hadoop/hdds/client/BlockID.java | 2 +- .../hadoop/hdds/scm/XceiverClientSpi.java | 4 +- .../hadoop/hdds/scm/client/ScmClient.java | 3 +- .../helpers/StorageContainerException.java | 2 +- .../scm/storage/ContainerProtocolCalls.java | 40 +++++++------- .../container/common/helpers/ChunkInfo.java | 2 +- .../container/common/helpers/KeyData.java | 2 +- .../com/google/protobuf/ShadedProtoUtil.java | 38 -------------- .../com/google/protobuf/package-info.java | 22 -------- .../proto/DatanodeContainerProtocol.proto | 2 +- .../container/common/helpers/ChunkUtils.java | 36 ++++++------- .../common/helpers/ContainerData.java | 4 +- .../common/helpers/ContainerMetrics.java | 2 +- .../common/helpers/ContainerUtils.java | 6 +-- .../container/common/helpers/FileUtils.java | 4 +- .../container/common/helpers/KeyUtils.java | 10 ++-- .../common/impl/ChunkManagerImpl.java | 10 ++-- .../common/impl/ContainerManagerImpl.java | 42 +++++++-------- .../container/common/impl/Dispatcher.java | 26 +++++----- .../container/common/impl/KeyManagerImpl.java | 6 +-- .../common/interfaces/ChunkManager.java | 2 +- .../interfaces/ContainerDispatcher.java | 4 +- .../background/BlockDeletingService.java | 5 +- .../transport/server/XceiverServer.java | 15 +++--- .../server/XceiverServerHandler.java | 8 +-- .../server/XceiverServerInitializer.java | 18 ++++--- .../server/ratis/ContainerStateMachine.java | 25 ++++----- .../cli/container/InfoContainerHandler.java | 5 +- .../client/io/ChunkGroupInputStream.java | 2 +- .../client/io/ChunkGroupOutputStream.java | 2 +- .../client/io/OzoneContainerTranslation.java | 2 +- .../ozone/container/ContainerTestHelper.java | 10 ++-- .../common/TestBlockDeletingService.java | 2 +- .../common/impl/TestContainerPersistence.java | 6 +-- .../metrics/TestContainerMetrics.java | 8 +-- .../ozoneimpl/TestOzoneContainer.java | 3 +- .../container/server/TestContainerServer.java | 10 ++-- .../ozone/scm/TestContainerSmallFile.java | 2 +- .../ozone/scm/TestXceiverClientMetrics.java | 12 ++--- .../hadoop/ozone/web/client/TestKeys.java | 2 +- .../genesis/BenchMarkDatanodeDispatcher.java | 25 +++++---- hadoop-project/pom.xml | 11 ++-- 51 files changed, 291 insertions(+), 286 deletions(-) delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java index 5c702c63b61..6d33cd4e361 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java @@ -20,18 +20,18 @@ package org.apache.hadoop.hdds.scm; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import io.netty.bootstrap.Bootstrap; -import io.netty.channel.Channel; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioSocketChannel; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; +import org.apache.ratis.shaded.io.netty.bootstrap.Bootstrap; +import org.apache.ratis.shaded.io.netty.channel.Channel; +import org.apache.ratis.shaded.io.netty.channel.EventLoopGroup; +import org.apache.ratis.shaded.io.netty.channel.nio.NioEventLoopGroup; +import org.apache.ratis.shaded.io.netty.channel.socket.nio.NioSocketChannel; +import org.apache.ratis.shaded.io.netty.handler.logging.LogLevel; +import org.apache.ratis.shaded.io.netty.handler.logging.LoggingHandler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.slf4j.Logger; diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java index e2b55ac7e8e..6a2286c523f 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java @@ -18,13 +18,13 @@ package org.apache.hadoop.hdds.scm; import com.google.common.base.Preconditions; -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; +import org.apache.ratis.shaded.io.netty.channel.Channel; +import org.apache.ratis.shaded.io.netty.channel.ChannelHandlerContext; +import org.apache.ratis.shaded.io.netty.channel.SimpleChannelInboundHandler; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; import org.apache.hadoop.util.Time; import org.slf4j.Logger; diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientInitializer.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientInitializer.java index e10a9f63965..90e2f5ad91a 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientInitializer.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientInitializer.java @@ -17,15 +17,17 @@ */ package org.apache.hadoop.hdds.scm; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.codec.protobuf.ProtobufDecoder; -import io.netty.handler.codec.protobuf.ProtobufEncoder; -import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; -import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender; +import org.apache.ratis.shaded.io.netty.channel.ChannelInitializer; +import org.apache.ratis.shaded.io.netty.channel.ChannelPipeline; +import org.apache.ratis.shaded.io.netty.channel.socket.SocketChannel; +import org.apache.ratis.shaded.io.netty.handler.codec.protobuf.ProtobufDecoder; +import org.apache.ratis.shaded.io.netty.handler.codec.protobuf.ProtobufEncoder; +import org.apache.ratis.shaded.io.netty.handler.codec.protobuf + .ProtobufVarint32FrameDecoder; +import org.apache.ratis.shaded.io.netty.handler.codec.protobuf + .ProtobufVarint32LengthFieldPrepender; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import java.util.concurrent.Semaphore; diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java index a61eba142ab..fbc348cd6a1 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdds.scm; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java index d010c6913f8..0effa8fff33 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java @@ -19,14 +19,15 @@ package org.apache.hadoop.hdds.scm; import com.google.common.base.Preconditions; -import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.shaded.com.google.protobuf + .InvalidProtocolBufferException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.ratis.RatisHelper; @@ -37,7 +38,6 @@ import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.rpc.RpcType; import org.apache.ratis.rpc.SupportedRpcType; import org.apache.ratis.shaded.com.google.protobuf.ByteString; -import org.apache.ratis.shaded.com.google.protobuf.ShadedProtoUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -211,8 +211,7 @@ public final class XceiverClientRatis extends XceiverClientSpi { private RaftClientReply sendRequest(ContainerCommandRequestProto request) throws IOException { boolean isReadOnlyRequest = isReadOnly(request); - ByteString byteString = - ShadedProtoUtil.asShadedByteString(request.toByteArray()); + ByteString byteString = request.toByteString(); LOG.debug("sendCommand {} {}", isReadOnlyRequest, request); final RaftClientReply reply = isReadOnlyRequest ? getClient().sendReadOnly(() -> byteString) : @@ -224,8 +223,7 @@ public final class XceiverClientRatis extends XceiverClientSpi { private CompletableFuture sendRequestAsync( ContainerCommandRequestProto request) throws IOException { boolean isReadOnlyRequest = isReadOnly(request); - ByteString byteString = - ShadedProtoUtil.asShadedByteString(request.toByteArray()); + ByteString byteString = request.toByteString(); LOG.debug("sendCommandAsync {} {}", isReadOnlyRequest, request); return isReadOnlyRequest ? getClient().sendReadOnlyAsync(() -> byteString) : getClient().sendAsync(() -> byteString); @@ -237,7 +235,7 @@ public final class XceiverClientRatis extends XceiverClientSpi { final RaftClientReply reply = sendRequest(request); Preconditions.checkState(reply.isSuccess()); return ContainerCommandResponseProto.parseFrom( - ShadedProtoUtil.asByteString(reply.getMessage().getContent())); + reply.getMessage().getContent()); } /** @@ -257,7 +255,7 @@ public final class XceiverClientRatis extends XceiverClientSpi { .thenApply(reply -> { try { return ContainerCommandResponseProto.parseFrom( - ShadedProtoUtil.asByteString(reply.getMessage().getContent())); + reply.getMessage().getContent()); } catch (InvalidProtocolBufferException e) { throw new CompletionException(e); } diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java index 15d197c405d..07f6cec2f81 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java @@ -25,8 +25,9 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.hdds.scm.protocolPB .StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerData; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerData; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ReadContainerResponseProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java index c4c336221c5..020c6847bbd 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java @@ -18,12 +18,12 @@ package org.apache.hadoop.hdds.scm.storage; -import com.google.protobuf.ByteString; +import org.apache.ratis.shaded.com.google.protobuf.ByteString; import org.apache.hadoop.fs.Seekable; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ChunkInfo; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ReadChunkResponseProto; import org.apache.hadoop.hdds.client.BlockID; diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java index 8fce00de1e4..779e63696a6 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java @@ -18,13 +18,13 @@ package org.apache.hadoop.hdds.scm.storage; -import com.google.protobuf.ByteString; +import org.apache.ratis.shaded.com.google.protobuf.ByteString; import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ChunkInfo; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.KeyData; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.KeyValue; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyData; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue; import org.apache.hadoop.hdds.client.BlockID; import java.io.IOException; diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index bf53042a853..6310df1a68a 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -78,7 +78,58 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + + kr.motd.maven + os-maven-plugin + ${os-maven-plugin.version} + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + true + + + com.google.protobuf:protoc:${protobuf-compile.version}:exe:${os.detected.classifier} + + ${basedir}/src/main/proto/ + + DatanodeContainerProtocol.proto + + target/generated-sources/java + false + + + + compile-protoc + + compile + test-compile + + + + + + maven-antrun-plugin + + + generate-sources + + + + + + + + run + + + + org.apache.hadoop hadoop-maven-plugins @@ -107,7 +158,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> ${basedir}/src/main/proto StorageContainerLocationProtocol.proto - DatanodeContainerProtocol.proto hdds.proto ScmBlockLocationProtocol.proto diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java index 355a36de859..7bf8f01ee34 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdds.client; import org.apache.commons.lang.builder.ToStringBuilder; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; /** diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java index c96f79b2d5f..56cc74125b1 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java @@ -21,9 +21,9 @@ package org.apache.hadoop.hdds.scm; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index dcf9fed800c..b52819a3001 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -20,7 +20,8 @@ package org.apache.hadoop.hdds.scm.client; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerData; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerData; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import java.io.IOException; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/StorageContainerException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/StorageContainerException.java index 35d8444b9dc..f1405fff946 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/StorageContainerException.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/StorageContainerException.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdds.scm.container.common.helpers; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import java.io.IOException; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java index ca388d9750d..5fbf373d25b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java @@ -18,39 +18,41 @@ package org.apache.hadoop.hdds.scm.storage; -import com.google.protobuf.ByteString; +import org.apache.ratis.shaded.com.google.protobuf.ByteString; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ChunkInfo; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.GetKeyRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .GetKeyRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .GetKeyResponseProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .GetSmallFileRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .GetSmallFileResponseProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.KeyData; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.PutKeyRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyData; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .PutKeyRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .PutSmallFileRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ReadChunkRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ReadChunkResponseProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ReadContainerRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ReadContainerResponseProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Type; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .WriteChunkRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.KeyValue; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue; import org.apache.hadoop.hdds.client.BlockID; import java.io.IOException; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java index 7cf95a9470d..21916b585e6 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java @@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.container.common.helpers; import com.google.common.base.Preconditions; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import java.io.IOException; import java.util.Map; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java index c485c7fa917..129e4a8fedb 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.ozone.container.common.helpers; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.client.BlockID; import java.io.IOException; diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java deleted file mode 100644 index 29242ad99f8..00000000000 --- a/hadoop-hdds/common/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ratis.shaded.com.google.protobuf; - -/** Utilities for the shaded protobuf in Ratis. */ -public interface ShadedProtoUtil { - /** - * @param bytes - * @return the wrapped shaded {@link ByteString} (no coping). - */ - static ByteString asShadedByteString(byte[] bytes) { - return ByteString.wrap(bytes); - } - - /** - * @param shaded - * @return a {@link com.google.protobuf.ByteString} (require coping). - */ - static com.google.protobuf.ByteString asByteString(ByteString shaded) { - return com.google.protobuf.ByteString.copyFrom( - shaded.asReadOnlyByteBuffer()); - } -} diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java deleted file mode 100644 index 032dd9634b2..00000000000 --- a/hadoop-hdds/common/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ratis.shaded.com.google.protobuf; - -/** - * This package contains classes related to the shaded protobuf in Apache Ratis. - */ diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto index 80bc22d8f04..3479866d250 100644 --- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto +++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto @@ -24,7 +24,7 @@ // This file contains protocol buffers that are used to transfer data // to and from the datanode. -option java_package = "org.apache.hadoop.hdds.protocol.proto"; +option java_package = "org.apache.hadoop.hdds.protocol.datanode.proto"; option java_outer_classname = "ContainerProtos"; option java_generate_equals_and_hash = true; package hadoop.hdds.datanode; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java index 8c5609d63c0..eba85947cfe 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java @@ -18,12 +18,12 @@ package org.apache.hadoop.ozone.container.common.helpers; import com.google.common.base.Preconditions; -import com.google.protobuf.ByteString; +import org.apache.ratis.shaded.com.google.protobuf.ByteString; import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.impl.ChunkManagerImpl; @@ -40,22 +40,22 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.concurrent.ExecutionException; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .CHECKSUM_MISMATCH; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .CONTAINER_INTERNAL_ERROR; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .CONTAINER_NOT_FOUND; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .INVALID_WRITE_SIZE; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .IO_EXCEPTION; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .OVERWRITE_FLAG_REQUIRED; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .UNABLE_TO_FIND_CHUNK; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .UNABLE_TO_FIND_DATA_DIR; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.CHECKSUM_MISMATCH; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.CONTAINER_INTERNAL_ERROR; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.CONTAINER_NOT_FOUND; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.INVALID_WRITE_SIZE; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.IO_EXCEPTION; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.OVERWRITE_FLAG_REQUIRED; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.UNABLE_TO_FIND_CHUNK; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.UNABLE_TO_FIND_DATA_DIR; /** * Set of utility functions used by the chunk Manager. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java index 947dc7d463d..63111c814fe 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java @@ -20,8 +20,8 @@ package org.apache.hadoop.ozone.container.common.helpers; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerLifeCycleState; import org.apache.hadoop.ozone.OzoneConsts; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java index d4d732b8b69..4300b2da42e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java @@ -21,7 +21,7 @@ package org.apache.hadoop.ozone.container.common.helpers; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java index 959d88c0274..9b5231664fb 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java @@ -25,7 +25,7 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl; @@ -42,9 +42,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import static org.apache.commons.io.FilenameUtils.removeExtension; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result .INVALID_ARGUMENT; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result .UNABLE_TO_FIND_DATA_DIR; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java index ec274525e4f..a2875befc63 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java @@ -18,8 +18,8 @@ package org.apache.hadoop.ozone.container.common.helpers; import com.google.common.base.Preconditions; -import com.google.protobuf.ByteString; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.ratis.shaded.com.google.protobuf.ByteString; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; /** * File Utils are helper routines used by putSmallFile and getSmallFile diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java index dbd5772d321..f831d455b6a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java @@ -21,17 +21,17 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.ozone.container.common.utils.ContainerCache; import org.apache.hadoop.utils.MetadataStore; import java.io.IOException; import java.nio.charset.Charset; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .NO_SUCH_KEY; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .UNABLE_TO_READ_METADATA_DB; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.NO_SUCH_KEY; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.UNABLE_TO_READ_METADATA_DB; /** * Utils functions to help key functions. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java index 350519692c1..fa820266026 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; @@ -40,10 +40,10 @@ import java.nio.file.StandardCopyOption; import java.security.NoSuchAlgorithmException; import java.util.concurrent.ExecutionException; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .CONTAINER_INTERNAL_ERROR; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .UNSUPPORTED_REQUEST; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.CONTAINER_INTERNAL_ERROR; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.UNSUPPORTED_REQUEST; /** * An implementation of ChunkManager that is used by default in ozone. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java index cb603347ac4..240beba6f3e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.protocol.proto @@ -79,26 +79,26 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .CONTAINER_EXISTS; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .CONTAINER_INTERNAL_ERROR; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .CONTAINER_NOT_FOUND; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .ERROR_IN_COMPACT_DB; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .INVALID_CONFIG; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .IO_EXCEPTION; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .NO_SUCH_ALGORITHM; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .UNABLE_TO_READ_METADATA_DB; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .UNCLOSED_CONTAINER_IO; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .UNSUPPORTED_REQUEST; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.CONTAINER_EXISTS; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.CONTAINER_INTERNAL_ERROR; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.CONTAINER_NOT_FOUND; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.ERROR_IN_COMPACT_DB; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.INVALID_CONFIG; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.IO_EXCEPTION; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.NO_SUCH_ALGORITHM; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.UNABLE_TO_READ_METADATA_DB; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.UNCLOSED_CONTAINER_IO; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.UNSUPPORTED_REQUEST; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION; /** diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java index 8d1b17ca09f..3b478cdb05a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java @@ -19,16 +19,16 @@ package org.apache.hadoop.ozone.container.common.impl; import com.google.common.base.Preconditions; -import com.google.protobuf.ByteString; +import org.apache.ratis.shaded.com.google.protobuf.ByteString; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Type; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; import org.apache.hadoop.ozone.container.common.helpers.ChunkUtils; @@ -48,14 +48,14 @@ import java.security.NoSuchAlgorithmException; import java.util.LinkedList; import java.util.List; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .CLOSED_CONTAINER_IO; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .GET_SMALL_FILE_ERROR; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .NO_SUCH_ALGORITHM; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .PUT_SMALL_FILE_ERROR; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.CLOSED_CONTAINER_IO; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.GET_SMALL_FILE_ERROR; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.NO_SUCH_ALGORITHM; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.PUT_SMALL_FILE_ERROR; /** * Ozone Container dispatcher takes a call from the netty server and routes it diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java index f920ded2935..0ca73545f6f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java @@ -23,7 +23,7 @@ import com.google.common.primitives.Longs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.ozone.container.common.helpers.ContainerData; import org.apache.hadoop.ozone.container.common.helpers.KeyData; @@ -40,8 +40,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result - .NO_SUCH_KEY; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .Result.NO_SUCH_KEY; /** * Key Manager impl. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java index 26dcf217cdf..c58fb9dbbcf 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java @@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.container.common.interfaces; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java index 984fe41b91e..7e12614c6e3 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java @@ -18,9 +18,9 @@ package org.apache.hadoop.ozone.container.common.interfaces; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; /** diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java index 7c3fa30bd1d..99845fa1e40 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java @@ -19,13 +19,14 @@ package org.apache.hadoop.ozone.container.common.statemachine.background; import com.google.common.collect.Lists; -import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.shaded.com.google.protobuf + .InvalidProtocolBufferException; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.helpers.ContainerData; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java index 50e45b45bf6..7105fd7881a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java @@ -19,13 +19,14 @@ package org.apache.hadoop.ozone.container.common.transport.server; import com.google.common.base.Preconditions; -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; +import org.apache.ratis.shaded.io.netty.bootstrap.ServerBootstrap; +import org.apache.ratis.shaded.io.netty.channel.Channel; +import org.apache.ratis.shaded.io.netty.channel.EventLoopGroup; +import org.apache.ratis.shaded.io.netty.channel.nio.NioEventLoopGroup; +import org.apache.ratis.shaded.io.netty.channel.socket.nio + .NioServerSocketChannel; +import org.apache.ratis.shaded.io.netty.handler.logging.LogLevel; +import org.apache.ratis.shaded.io.netty.handler.logging.LoggingHandler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java index 5947dde75db..37652991c8e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java @@ -18,11 +18,11 @@ package org.apache.hadoop.ozone.container.common.transport.server; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.ratis.shaded.io.netty.channel.ChannelHandlerContext; +import org.apache.ratis.shaded.io.netty.channel.SimpleChannelInboundHandler; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; import org.slf4j.Logger; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java index 78ba26b4de1..e405cf99ccc 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java @@ -19,14 +19,16 @@ package org.apache.hadoop.ozone.container.common.transport.server; import com.google.common.base.Preconditions; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.codec.protobuf.ProtobufDecoder; -import io.netty.handler.codec.protobuf.ProtobufEncoder; -import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; -import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.ratis.shaded.io.netty.channel.ChannelInitializer; +import org.apache.ratis.shaded.io.netty.channel.ChannelPipeline; +import org.apache.ratis.shaded.io.netty.channel.socket.SocketChannel; +import org.apache.ratis.shaded.io.netty.handler.codec.protobuf.ProtobufDecoder; +import org.apache.ratis.shaded.io.netty.handler.codec.protobuf.ProtobufEncoder; +import org.apache.ratis.shaded.io.netty.handler.codec.protobuf + .ProtobufVarint32FrameDecoder; +import org.apache.ratis.shaded.io.netty.handler.codec.protobuf + .ProtobufVarint32LengthFieldPrepender; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 89eaaced032..56c52bb80e1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -19,13 +19,14 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis; import com.google.common.base.Preconditions; -import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.ratis.shaded.com.google.protobuf + .InvalidProtocolBufferException; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .WriteChunkRequestProto; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; import org.apache.ratis.conf.RaftProperties; @@ -34,7 +35,6 @@ import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.server.storage.RaftStorage; import org.apache.ratis.shaded.com.google.protobuf.ByteString; -import org.apache.ratis.shaded.com.google.protobuf.ShadedProtoUtil; import org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto; import org.apache.ratis.shaded.proto.RaftProtos.SMLogEntryProto; import org.apache.ratis.statemachine.StateMachineStorage; @@ -159,8 +159,8 @@ public class ContainerStateMachine extends BaseStateMachine { .build(); log = SMLogEntryProto.newBuilder() - .setData(getShadedByteString(commitContainerCommandProto)) - .setStateMachineData(getShadedByteString(dataContainerCommandProto)) + .setData(commitContainerCommandProto.toByteString()) + .setStateMachineData(dataContainerCommandProto.toByteString()) .build(); } else if (proto.getCmdType() == ContainerProtos.Type.CreateContainer) { log = SMLogEntryProto.newBuilder() @@ -175,21 +175,16 @@ public class ContainerStateMachine extends BaseStateMachine { return new TransactionContextImpl(this, request, log); } - private ByteString getShadedByteString(ContainerCommandRequestProto proto) { - return ShadedProtoUtil.asShadedByteString(proto.toByteArray()); - } - private ContainerCommandRequestProto getRequestProto(ByteString request) throws InvalidProtocolBufferException { - return ContainerCommandRequestProto.parseFrom( - ShadedProtoUtil.asByteString(request)); + return ContainerCommandRequestProto.parseFrom(request); } private Message runCommand(ContainerCommandRequestProto requestProto) { LOG.trace("dispatch {}", requestProto); ContainerCommandResponseProto response = dispatcher.dispatch(requestProto); LOG.trace("response {}", response); - return () -> ShadedProtoUtil.asShadedByteString(response.toByteArray()); + return () -> response.toByteString(); } private CompletableFuture handleWriteChunk( diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoContainerHandler.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoContainerHandler.java index cefa28cc40b..6027becd26b 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoContainerHandler.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoContainerHandler.java @@ -26,8 +26,9 @@ import org.apache.hadoop.hdds.scm.cli.OzoneCommandHandler; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerData; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerData; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerLifeCycleState; import java.io.IOException; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java index ccc59116be0..e1a29182b0d 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java @@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.client.io; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.Seekable; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java index 41ceee41f09..c6e56b3b9b9 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java @@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.client.io; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.hadoop.fs.FSExceptionMessages; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.Result; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java index e7215ef80a0..e74fffd7db2 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java @@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.client.io; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.KeyData; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyData; import org.apache.hadoop.hdds.client.BlockID; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java index fed725c198d..d2a64340dcc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java @@ -19,19 +19,19 @@ package org.apache.hadoop.ozone.container; import com.google.common.base.Preconditions; -import com.google.protobuf.ByteString; +import org.apache.ratis.shaded.com.google.protobuf.ByteString; import org.apache.commons.codec.binary.Hex; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.KeyValue; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index a60da21808d..56fd0b179ac 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -23,7 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java index e51c3f75af0..a7cab4e783b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java @@ -22,7 +22,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -36,9 +36,7 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerData; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.helpers.KeyData; import org.apache.hadoop.ozone.container.common.helpers.KeyUtils; -import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.util.Time; import org.apache.hadoop.utils.MetadataStore; import org.junit.After; import org.junit.AfterClass; @@ -77,7 +75,7 @@ import static org.apache.hadoop.ozone.container.ContainerTestHelper.getChunk; import static org.apache.hadoop.ozone.container.ContainerTestHelper.getData; import static org.apache.hadoop.ozone.container.ContainerTestHelper .setDataChecksum; -import static org.apache.hadoop.hdds.protocol.proto.ContainerProtos +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .Stage.COMBINED; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java index 0bba5c1c638..2921be237f0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java @@ -26,9 +26,11 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerCommandResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerCommandRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerCommandResponseProto; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.container.ContainerTestHelper; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index 4e1d14be6d1..513974af4ff 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.container.ozoneimpl; import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -29,7 +29,6 @@ import org.apache.hadoop.hdds.scm.XceiverClient; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.util.Time; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java index b207914475b..eb170eaf04d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java @@ -18,11 +18,13 @@ package org.apache.hadoop.ozone.container.server; -import io.netty.channel.embedded.EmbeddedChannel; +import org.apache.ratis.shaded.io.netty.channel.embedded.EmbeddedChannel; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerCommandResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerCommandRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerCommandResponseProto; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.ozone.OzoneConfigKeys; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java index f56d78c8d6e..ce1fe4615ca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.scm; import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientMetrics.java index 99742c2024f..d6f5d32f3a6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientMetrics.java @@ -26,24 +26,22 @@ import java.util.ArrayList; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; -import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerCommandResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerCommandRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerCommandResponseProto; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.container.ContainerTestHelper; -import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientMetrics; import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.util.Time; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java index b621a08d2bd..57d4287b630 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.ozone.container.common.helpers.ContainerData; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.helpers.KeyData; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java index 2da68746fa6..a5d268d5089 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.ozone.genesis; -import com.google.protobuf.ByteString; +import org.apache.ratis.shaded.com.google.protobuf.ByteString; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.RandomStringUtils; @@ -53,15 +53,22 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ROOT_PREFIX; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.CreateContainerRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ReadChunkRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.WriteChunkRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.PutKeyRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.GetKeyRequestProto; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerData; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerCommandRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .CreateContainerRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ReadChunkRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .WriteChunkRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .PutKeyRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .GetKeyRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .ContainerData; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 793ffb456b5..862a69348f4 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -104,6 +104,11 @@ 6.2.1.jre7 2.7.5 + + 0.5.1 + 3.1.0 + 1.5.0.Final + 1.8 @@ -413,7 +418,7 @@ ${hadoop.version} - + org.apache.hadoop hadoop-yarn-applications-distributedshell ${hadoop.version} @@ -1737,8 +1742,8 @@ - - org.apache.maven.plugins + + org.apache.maven.plugins maven-antrun-plugin