diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index f908d987115..067cd7bb556 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -190,6 +190,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> ReconfigurationProtocol.proto StorageContainerLocationProtocol.proto DatanodeContainerProtocol.proto + Ozone.proto + KeySpaceManagerProtocol.proto diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java new file mode 100644 index 00000000000..472af65165e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ksm.helpers; + +import com.google.common.base.Preconditions; + +import java.util.HashMap; +import java.util.Map; + +/** + * A class that encapsulates the createVolume Args. + */ +public final class VolumeArgs { + private final String adminName; + private final String ownerName; + private final String volume; + private final long quotaInBytes; + private final Map extendedAttributes; + + /** + * Private constructor, constructed via builder. + * + * @param adminName - Administrator name. + * @param ownerName - Volume owner's name + * @param volume - volume name + * @param quotaInBytes - Volume Quota in bytes. + * @param keyValueMap - keyValue map. + */ + private VolumeArgs(String adminName, String ownerName, String volume, + long quotaInBytes, Map keyValueMap) { + this.adminName = adminName; + this.ownerName = ownerName; + this.volume = volume; + this.quotaInBytes = quotaInBytes; + this.extendedAttributes = keyValueMap; + } + + /** + * Returns the Admin Name. + * + * @return String. + */ + public String getAdminName() { + return adminName; + } + + /** + * Returns the owner Name. + * + * @return String + */ + public String getOwnerName() { + return ownerName; + } + + /** + * Returns the volume Name. + * + * @return String + */ + public String getVolume() { + return volume; + } + + /** + * Returns Quota in Bytes. + * + * @return long, Quota in bytes. + */ + public long getQuotaInBytes() { + return quotaInBytes; + } + + public Map getExtendedAttributes() { + return extendedAttributes; + } + + static class Builder { + private String adminName; + private String ownerName; + private String volume; + private long quotaInBytes; + private Map extendedAttributes; + + /** + * Constructs a builder. + */ + Builder() { + extendedAttributes = new HashMap<>(); + } + + public void setAdminName(String adminName) { + this.adminName = adminName; + } + + public void setOwnerName(String ownerName) { + this.ownerName = ownerName; + } + + public void setVolume(String volume) { + this.volume = volume; + } + + public void setQuotaInBytes(long quotaInBytes) { + this.quotaInBytes = quotaInBytes; + } + + public void addMetadata(String key, String value) { + extendedAttributes.put(key, value); // overwrite if present. + } + + /** + * Constructs a CreateVolumeArgument. + * + * @return CreateVolumeArgs. + */ + public VolumeArgs build() { + Preconditions.checkNotNull(adminName); + Preconditions.checkNotNull(ownerName); + Preconditions.checkNotNull(volume); + return new VolumeArgs(adminName, ownerName, volume, quotaInBytes, + extendedAttributes); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java new file mode 100644 index 00000000000..8e487731054 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java @@ -0,0 +1,18 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ksm.helpers; \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java new file mode 100644 index 00000000000..5b208fa8759 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ksm; +/** + This package contains client side protocol library to communicate with KSM. + */ \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeyspaceManagerProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeyspaceManagerProtocol.java new file mode 100644 index 00000000000..3a07d97de09 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeyspaceManagerProtocol.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ksm.protocol; + +import org.apache.hadoop.ksm.helpers.VolumeArgs; + +import java.io.IOException; +import java.util.List; + +/** + * Protocol to talk to KSM. + */ +public interface KeyspaceManagerProtocol { + + /** + * Creates a volume. + * @param args - Arguments to create Volume. + * @throws IOException + */ + void createVolume(VolumeArgs args) throws IOException; + + /** + * Changes the owner of a volume. + * @param volume - Name of the volume. + * @param owner - Name of the owner. + * @throws IOException + */ + void setOwner(String volume, String owner) throws IOException; + + /** + * Changes the Quota on a volume. + * @param volume - Name of the volume. + * @param quota - Quota in bytes. + * @throws IOException + */ + void setQuota(String volume, long quota) throws IOException; + + /** + * Checks if the specified user can access this volume. + * @param volume - volume + * @param userName - user name + * @throws IOException + */ + void checkVolumeAccess(String volume, String userName) throws IOException; + + /** + * Gets the volume information. + * @param volume - Volume name.s + * @return VolumeArgs or exception is thrown. + * @throws IOException + */ + VolumeArgs getVolumeinfo(String volume) throws IOException; + + /** + * Deletes the an exisiting empty volume. + * @param volume - Name of the volume. + * @throws IOException + */ + void deleteVolume(String volume) throws IOException; + + /** + * Lists volume owned by a specific user. + * @param userName - user name + * @param prefix - Filter prefix -- Return only entries that match this. + * @param prevKey - Previous key -- List starts from the next from the prevkey + * @param maxKeys - Max number of keys to return. + * @return List of Volumes. + * @throws IOException + */ + List listVolumeByUser(String userName, String prefix, String + prevKey, long maxKeys) throws IOException; + + /** + * Lists volume all volumes in the cluster. + * @param prefix - Filter prefix -- Return only entries that match this. + * @param prevKey - Previous key -- List starts from the next from the prevkey + * @param maxKeys - Max number of keys to return. + * @return List of Volumes. + * @throws IOException + */ + List listAllVolumes(String prefix, String + prevKey, long maxKeys) throws IOException; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java new file mode 100644 index 00000000000..d1d12dc34ed --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ksm.protocol; \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolPB.java new file mode 100644 index 00000000000..1490008d674 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolPB.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ksm.protocolPB; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyspaceManagerService; + +/** + * Protocol used to communicate with KSM. + */ +@ProtocolInfo(protocolName = + "org.apache.hadoop.ozone.protocol.KeyspaceManagerProtocol", + protocolVersion = 1) +@InterfaceAudience.Private +public interface KeySpaceManagerProtocolPB + extends KeyspaceManagerService.BlockingInterface { +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/package-info.java new file mode 100644 index 00000000000..5dfa5ea0839 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ksm.protocolPB; \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java index e779247652c..03f9c82461e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java @@ -152,6 +152,7 @@ public final class ScmConfigKeys { "ozone.scm.container.provision_batch_size"; public static final int OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE_DEFAULT = 10; + /** * Never constructed. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java index 4456138b2ae..2ebc1e62c98 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java @@ -19,7 +19,7 @@ package org.apache.hadoop.scm.container.common.helpers; import com.google.common.base.Preconditions; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; @@ -60,7 +60,7 @@ public class Pipeline { * @param pipeline - ProtoBuf definition for the pipeline. * @return Pipeline Object */ - public static Pipeline getFromProtoBuf(ContainerProtos.Pipeline pipeline) { + public static Pipeline getFromProtoBuf(OzoneProtos.Pipeline pipeline) { Preconditions.checkNotNull(pipeline); Pipeline newPipeline = new Pipeline(pipeline.getLeaderID()); for (HdfsProtos.DatanodeIDProto dataID : pipeline.getMembersList()) { @@ -104,9 +104,9 @@ public class Pipeline { * * @return Protobuf message */ - public ContainerProtos.Pipeline getProtobufMessage() { - ContainerProtos.Pipeline.Builder builder = - ContainerProtos.Pipeline.newBuilder(); + public OzoneProtos.Pipeline getProtobufMessage() { + OzoneProtos.Pipeline.Builder builder = + OzoneProtos.Pipeline.newBuilder(); for (DatanodeID datanode : datanodes.values()) { builder.addMembers(datanode.getProtoBufMessage()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java index 962ca6b3c46..d93d2cf6207 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java @@ -30,7 +30,7 @@ import com.google.protobuf.ByteString; import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo; import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyValue; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue; import org.apache.hadoop.scm.XceiverClientManager; import org.apache.hadoop.scm.XceiverClientSpi; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto index 2d018ee5296..012ff1dedb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto @@ -29,6 +29,7 @@ option java_outer_classname = "ContainerProtos"; option java_generate_equals_and_hash = true; package hadoop.hdfs.ozone; import "hdfs.proto"; +import "Ozone.proto"; /** * Commands that are used to manipulate the state of containers on a datanode. @@ -190,17 +191,7 @@ message ContainerCommandResponseProto { } -// A pipeline is composed of one or more datanodes that back a container. -message Pipeline { - required string leaderID = 1; - repeated DatanodeIDProto members = 2; - required string containerName = 3; -} -message KeyValue { - required string key = 1; - optional string value = 2; -} message ContainerData { required string name = 1; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto new file mode 100644 index 00000000000..3ac25b88747 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * These .proto interfaces are private and unstable. + * Please see http://wiki.apache.org/hadoop/Compatibility + * for what changes are allowed for a *unstable* .proto interface. + */ + +option java_package = "org.apache.hadoop.ozone.protocol.proto"; +option java_outer_classname = "KeySpaceManagerProtocolProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.hdfs.ozone; + +/** +This is file contains the protocol to communicate with +Ozone key space manager. Ozone KSM manages the namespace for ozone. +This is similar to Namenode for Ozone. +*/ + +import "Ozone.proto"; + +enum Status { + OK = 1; + VOLUME_NOT_UNIQUE = 2; + VOLUME_NOT_FOUND = 3; + VOLUME_NOT_EMPTY = 4; + USER_NOT_FOUND = 5; + ACCESS_DENIED = 6; +} + + +message VolumeInfo { + required string adminName = 1; + required string ownerName = 2; + required string volume = 3; + optional uint64 quotaInBytes = 4; + repeated KeyValue metadata = 5; +} + +/** + Creates a volume +*/ +message CreateVolumeRequest { + required VolumeInfo volumeInfo = 1; +} + +message CreateVolumeResponse { + + required Status status = 1; +} + +/** + Changes the Volume Properties -- like ownership and quota for a volume. +*/ +message SetVolumePropertyRequest { + required string volumeName = 1; + optional string ownerName = 2; + optional uint64 quotaInBytes = 3; +} + +message SetVolumePropertyResponse { + required Status status = 1; +} + +/** + Checks if a specified user has access to the volume. +*/ +message CheckVolumeAccessRequest { + required string volumeName = 1; + required string userName = 2; +} + +message CheckVolumeAccessResponse { + + required Status status = 1; +} + + +/** + Returns information about a volume. +*/ + +message InfoVolumeRequest { + required string volumeName = 1; +} + +message InfoVolumeResponse { + required Status status = 1; + optional VolumeInfo volumeInfo = 2; + +} + +/** + Deletes an existing volume. +*/ +message DeleteVolumeRequest { + required string volumeName = 1; +} + +message DeleteVolumeResponse { + required Status status = 1; +} + + +/** + List Volumes -- List all volumes in the cluster or by user. +*/ + +message ListVolumeRequest { + enum Scope { + USER_VOLUMES = 1; // User volumes -- called by user + VOLUMES_BY_USER = 2; // User volumes - called by Admin + VOLUMES_BY_CLUSTER = 3; // All volumes in the cluster + } + required Scope scope = 1; + required string volumeName = 2; + optional string userName = 3; + optional string prefix = 4; + optional string prevKey = 5; + optional uint64 maxKeys = 6; +} + +message ListVolumeResponse { + enum Status { + OK = 1; + ACCESS_DENIED = 2; + REQUIRED_ARG_MISSING = 3; + } + required Status status = 1; + repeated VolumeInfo volumeInfo = 2; +} + +/** + The KSM service that takes care of Ozone namespace. +*/ +service KeyspaceManagerService { + + /** + Creates a Volume. + */ + rpc createVolume(CreateVolumeRequest) + returns(CreateVolumeResponse); + + /** + Allows modificiation of volume properties. + */ + rpc setVolumeProperty(SetVolumePropertyRequest) + returns (SetVolumePropertyResponse); + + /** + Checks if the specified volume is accesible by the specified user. + */ + rpc checkVolumeAccess(CheckVolumeAccessRequest) + returns (CheckVolumeAccessResponse); + + /** + Gets Volume information. + */ + rpc infoVolume(InfoVolumeRequest) + returns(InfoVolumeResponse); + /** + Deletes a volume if it is empty. + */ + rpc deleteVolume(DeleteVolumeRequest) + returns (DeleteVolumeResponse); + + /** + Lists Volumes + */ + rpc listVolumes(ListVolumeRequest) + returns (ListVolumeResponse); +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto new file mode 100644 index 00000000000..740038ae9fa --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * These .proto interfaces are private and unstable. + * Please see http://wiki.apache.org/hadoop/Compatibility + * for what changes are allowed for a *unstable* .proto interface. + */ + +option java_package = "org.apache.hadoop.ozone.protocol.proto"; +option java_outer_classname = "OzoneProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.hdfs.ozone; +import "hdfs.proto"; + + +// A pipeline is composed of one or more datanodes that back a container. +message Pipeline { + required string leaderID = 1; + repeated DatanodeIDProto members = 2; + required string containerName = 3; +} + +message KeyValue { + required string key = 1; + optional string value = 2; +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto index 8dfeb8b4ba3..772db2b7b30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto @@ -29,7 +29,7 @@ option java_generate_equals_and_hash = true; package hadoop.hdfs; import "hdfs.proto"; -import "DatanodeContainerProtocol.proto"; +import "Ozone.proto"; /** * keys - batch of object keys to find diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 9d21f60475c..8ce29de0fab 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -52,6 +52,7 @@ function hadoop_usage hadoop_add_subcommand "jmxget" "get JMX exported values from NameNode or DataNode." hadoop_add_subcommand "journalnode" "run the DFS journalnode" hadoop_add_subcommand "jscsi" "run cblock jscsi server" + hadoop_add_subcommand "ksm" "Ozone keyspace manager" hadoop_add_subcommand "lsSnapshottableDir" "list all snapshottable dirs owned by the current user" hadoop_add_subcommand "mover" "run a utility to move block replicas across storage types" hadoop_add_subcommand "namenode" "run the DFS namenode" @@ -164,6 +165,12 @@ function hdfscmd_case jmxget) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet ;; + ksm) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME=org.apache.hadoop.ozone.ksm.KeySpaceManager + hadoop_debug "Appending HADOOP_KSM_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_KSM_OPTS}" + ;; lsSnapshottableDir) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir ;; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java index 908cc44eb84..79275ebd0eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java @@ -35,6 +35,10 @@ import java.util.HashSet; import java.util.Map; import java.util.concurrent.TimeUnit; +import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY; +import static org.apache.hadoop.ozone.ksm.KSMConfigKeys + .OZONE_KSM_BIND_HOST_DEFAULT; +import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_PORT_DEFAULT; import static org.apache.hadoop.scm.ScmConfigKeys .OZONE_SCM_DEADNODE_INTERVAL_DEFAULT; import static org.apache.hadoop.scm.ScmConfigKeys @@ -224,6 +228,26 @@ public final class OzoneClientUtils { port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); } + + /** + * Retrieve the socket address that is used by KSM. + * @param conf + * @return Target InetSocketAddress for the SCM service endpoint. + */ + public static InetSocketAddress getKsmAddress( + Configuration conf) { + final Optional host = getHostNameFromConfigKeys(conf, + OZONE_KSM_ADDRESS_KEY); + + // If no port number is specified then we'll just try the defaultBindPort. + final Optional port = getPortNumberFromConfigKeys(conf, + OZONE_KSM_ADDRESS_KEY); + + return NetUtils.createSocketAddr( + host.or(OZONE_KSM_BIND_HOST_DEFAULT) + ":" + + port.or(OZONE_KSM_PORT_DEFAULT)); + } + /** * Retrieve the hostname, trying the supplied config keys in order. * Each config value may be absent, or if present in the format diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java index 7a3d4494d7d..690e3b59c1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java @@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.common.helpers; import com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; import java.io.IOException; import java.util.Map; @@ -110,8 +111,8 @@ public class ChunkInfo { } for (Map.Entry entry : metadata.entrySet()) { - ContainerProtos.KeyValue.Builder keyValBuilder = - ContainerProtos.KeyValue.newBuilder(); + OzoneProtos.KeyValue.Builder keyValBuilder = + OzoneProtos.KeyValue.newBuilder(); builder.addMetadata(keyValBuilder.setKey(entry.getKey()) .setValue(entry.getValue()).build()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java index 2f26649acbc..1b70b1b3238 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java @@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.common.helpers; import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; import org.apache.hadoop.util.Time; import java.io.IOException; @@ -113,8 +114,8 @@ public class ContainerData { builder.setOpen(this.isOpen()); for (Map.Entry entry : metadata.entrySet()) { - ContainerProtos.KeyValue.Builder keyValBuilder = - ContainerProtos.KeyValue.newBuilder(); + OzoneProtos.KeyValue.Builder keyValBuilder = + OzoneProtos.KeyValue.newBuilder(); builder.addMetadata(keyValBuilder.setKey(entry.getKey()) .setValue(entry.getValue()).build()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java index 289dbe48fed..590513eeb8f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.container.common.helpers; import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; import java.io.IOException; import java.util.Collections; @@ -81,8 +82,8 @@ public class KeyData { builder.setName(this.getKeyName()); builder.addAllChunks(this.chunks); for (Map.Entry entry : metadata.entrySet()) { - ContainerProtos.KeyValue.Builder keyValBuilder = - ContainerProtos.KeyValue.newBuilder(); + OzoneProtos.KeyValue.Builder keyValBuilder = + OzoneProtos.KeyValue.newBuilder(); builder.addMetadata(keyValBuilder.setKey(entry.getKey()) .setValue(entry.getValue()).build()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java new file mode 100644 index 00000000000..a23d47ba121 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.ksm; + +/** + * KSM Constants. + */ +public final class KSMConfigKeys { + /** + * Never constructed. + */ + private KSMConfigKeys() { + } + + + public static final String OZONE_KSM_HANDLER_COUNT_KEY = + "ozone.scm.handler.count.key"; + public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 200; + + public static final String OZONE_KSM_ADDRESS_KEY = + "ozone.ksm.address"; + public static final String OZONE_KSM_BIND_HOST_DEFAULT = + "0.0.0.0"; + public static final int OZONE_KSM_PORT_DEFAULT = 9862; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java new file mode 100644 index 00000000000..725088534df --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java @@ -0,0 +1,289 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.ksm; + +import com.google.protobuf.BlockingService; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ksm.helpers.VolumeArgs; +import org.apache.hadoop.ksm.protocol.KeyspaceManagerProtocol; +import org.apache.hadoop.ksm.protocolPB.KeySpaceManagerProtocolPB; +import org.apache.hadoop.ozone.OzoneClientUtils; +import org.apache.hadoop.ozone.OzoneConfiguration; +import org.apache.hadoop.ozone.protocolPB + .KeyspaceManagerProtocolServerSideTranslatorPB; +import org.apache.hadoop.ozone.scm.StorageContainerManager; +import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.List; + +import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY; +import static org.apache.hadoop.ozone.ksm.KSMConfigKeys + .OZONE_KSM_HANDLER_COUNT_DEFAULT; +import static org.apache.hadoop.ozone.ksm.KSMConfigKeys + .OZONE_KSM_HANDLER_COUNT_KEY; +import static org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos + .KeyspaceManagerService.newReflectiveBlockingService; +import static org.apache.hadoop.util.ExitUtil.terminate; + +/** + * Ozone Keyspace manager is the metadata manager of ozone. + */ +@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"}) +public class KeySpaceManager implements KeyspaceManagerProtocol { + // TODO: Support JMX + private static final Logger LOG = + LoggerFactory.getLogger(KeySpaceManager.class); + + private final RPC.Server ksmRpcServer; + private final InetSocketAddress ksmRpcAddress; + + public KeySpaceManager(OzoneConfiguration conf) throws IOException { + final int handlerCount = conf.getInt(OZONE_KSM_HANDLER_COUNT_KEY, + OZONE_KSM_HANDLER_COUNT_DEFAULT); + + RPC.setProtocolEngine(conf, KeySpaceManagerProtocolPB.class, + ProtobufRpcEngine.class); + + BlockingService ksmService = newReflectiveBlockingService( + new KeyspaceManagerProtocolServerSideTranslatorPB(this)); + final InetSocketAddress ksmNodeRpcAddr = OzoneClientUtils. + getKsmAddress(conf); + ksmRpcServer = startRpcServer(conf, ksmNodeRpcAddr, + KeySpaceManagerProtocolPB.class, ksmService, + handlerCount); + ksmRpcAddress = updateListenAddress(conf, + OZONE_KSM_ADDRESS_KEY, ksmNodeRpcAddr, ksmRpcServer); + + //TODO : Add call to register MXBean for JMX. + } + + /** + * Starts an RPC server, if configured. + * + * @param conf configuration + * @param addr configured address of RPC server + * @param protocol RPC protocol provided by RPC server + * @param instance RPC protocol implementation instance + * @param handlerCount RPC server handler count + * + * @return RPC server + * @throws IOException if there is an I/O error while creating RPC server + */ + private static RPC.Server startRpcServer(OzoneConfiguration conf, + InetSocketAddress addr, Class protocol, BlockingService instance, + int handlerCount) throws IOException { + RPC.Server rpcServer = new RPC.Builder(conf) + .setProtocol(protocol) + .setInstance(instance) + .setBindAddress(addr.getHostString()) + .setPort(addr.getPort()) + .setNumHandlers(handlerCount) + .setVerbose(false) + .setSecretManager(null) + .build(); + + DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer); + return rpcServer; + } + + /** + * Main entry point for starting KeySpaceManager. + * + * @param argv arguments + * @throws IOException if startup fails due to I/O error + */ + public static void main(String[] argv) throws IOException { + StringUtils.startupShutdownMessage(StorageContainerManager.class, + argv, LOG); + try { + KeySpaceManager ksm = new KeySpaceManager(new OzoneConfiguration()); + ksm.start(); + ksm.join(); + } catch (Throwable t) { + LOG.error("Failed to start the KeyspaceManager.", t); + terminate(1, t); + } + } + + /** + * Builds a message for logging startup information about an RPC server. + * + * @param description RPC server description + * @param addr RPC server listening address + * @return server startup message + */ + private static String buildRpcServerStartMessage(String description, + InetSocketAddress addr) { + return addr != null ? String.format("%s is listening at %s", + description, addr.toString()) : + String.format("%s not started", description); + } + + /** + * After starting an RPC server, updates configuration with the actual + * listening address of that server. The listening address may be different + * from the configured address if, for example, the configured address uses + * port 0 to request use of an ephemeral port. + * + * @param conf configuration to update + * @param rpcAddressKey configuration key for RPC server address + * @param addr configured address + * @param rpcServer started RPC server. + */ + private static InetSocketAddress updateListenAddress(OzoneConfiguration conf, + String rpcAddressKey, InetSocketAddress addr, RPC.Server rpcServer) { + InetSocketAddress listenAddr = rpcServer.getListenerAddress(); + InetSocketAddress updatedAddr = new InetSocketAddress( + addr.getHostString(), listenAddr.getPort()); + conf.set(rpcAddressKey, + listenAddr.getHostString() + ":" + listenAddr.getPort()); + return updatedAddr; + } + + /** + * Start service. + */ + public void start() { + LOG.info(buildRpcServerStartMessage("KeyspaceManager RPC server", + ksmRpcAddress)); + ksmRpcServer.start(); + } + + /** + * Wait until service has completed shutdown. + */ + public void join() { + try { + ksmRpcServer.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.info("Interrupted during KeyspaceManager join."); + } + } + + /** + * Creates a volume. + * + * @param args - Arguments to create Volume. + * @throws IOException + */ + @Override + public void createVolume(VolumeArgs args) throws IOException { + + } + + /** + * Changes the owner of a volume. + * + * @param volume - Name of the volume. + * @param owner - Name of the owner. + * @throws IOException + */ + @Override + public void setOwner(String volume, String owner) throws IOException { + + } + + /** + * Changes the Quota on a volume. + * + * @param volume - Name of the volume. + * @param quota - Quota in bytes. + * @throws IOException + */ + @Override + public void setQuota(String volume, long quota) throws IOException { + + } + + /** + * Checks if the specified user can access this volume. + * + * @param volume - volume + * @param userName - user name + * @throws IOException + */ + @Override + public void checkVolumeAccess(String volume, String userName) throws + IOException { + + } + + /** + * Gets the volume information. + * + * @param volume - Volume name.s + * @return VolumeArgs or exception is thrown. + * @throws IOException + */ + @Override + public VolumeArgs getVolumeinfo(String volume) throws IOException { + return null; + } + + /** + * Deletes the an exisiting empty volume. + * + * @param volume - Name of the volume. + * @throws IOException + */ + @Override + public void deleteVolume(String volume) throws IOException { + + } + + /** + * Lists volume owned by a specific user. + * + * @param userName - user name + * @param prefix - Filter prefix -- Return only entries that match this. + * @param prevKey - Previous key -- List starts from the next from the + * prevkey + * @param maxKeys - Max number of keys to return. + * @return List of Volumes. + * @throws IOException + */ + @Override + public List listVolumeByUser(String userName, String prefix, + String prevKey, long maxKeys) throws IOException { + return null; + } + + /** + * Lists volume all volumes in the cluster. + * + * @param prefix - Filter prefix -- Return only entries that match this. + * @param prevKey - Previous key -- List starts from the next from the + * prevkey + * @param maxKeys - Max number of keys to return. + * @return List of Volumes. + * @throws IOException + */ + @Override + public List listAllVolumes(String prefix, String prevKey, long + maxKeys) throws IOException { + return null; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java new file mode 100644 index 00000000000..09d9f326357 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.ksm; +/* + This package contains the keyspace manager classes. + */ \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java index cf742ae52ed..8d8f4f1e976 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -18,7 +19,7 @@ package org.apache.hadoop.ozone.protocol; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyValue; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue; import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeyspaceManagerProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeyspaceManagerProtocolServerSideTranslatorPB.java new file mode 100644 index 00000000000..0725d257077 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeyspaceManagerProtocolServerSideTranslatorPB.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.protocolPB; + +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; +import org.apache.hadoop.ksm.protocol.KeyspaceManagerProtocol; +import org.apache.hadoop.ksm.protocolPB.KeySpaceManagerProtocolPB; +import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos; + +/** + * This class is the server-side translator that forwards requests received on + * {@link org.apache.hadoop.ksm.protocolPB.KeySpaceManagerProtocolPB} to the + * KeyspaceManagerService server implementation. + */ +public class KeyspaceManagerProtocolServerSideTranslatorPB implements + KeySpaceManagerProtocolPB { + private final KeyspaceManagerProtocol impl; + + /** + * Constructs an instance of the server handler. + * + * @param impl KeySpaceManagerProtocolPB + */ + public KeyspaceManagerProtocolServerSideTranslatorPB( + KeyspaceManagerProtocol impl) { + this.impl = impl; + } + + @Override + public KeySpaceManagerProtocolProtos.CreateVolumeResponse createVolume( + RpcController controller, KeySpaceManagerProtocolProtos + .CreateVolumeRequest + request) throws ServiceException { + return null; + } + + @Override + public KeySpaceManagerProtocolProtos.SetVolumePropertyResponse + setVolumeProperty(RpcController controller, KeySpaceManagerProtocolProtos + .SetVolumePropertyRequest request) throws ServiceException { + return null; + } + + @Override + public KeySpaceManagerProtocolProtos.CheckVolumeAccessResponse + checkVolumeAccess(RpcController controller, KeySpaceManagerProtocolProtos + .CheckVolumeAccessRequest request) throws ServiceException { + return null; + } + + @Override + public KeySpaceManagerProtocolProtos.InfoVolumeResponse infoVolume( + RpcController controller, + KeySpaceManagerProtocolProtos.InfoVolumeRequest request) + throws ServiceException { + return null; + } + + @Override + public KeySpaceManagerProtocolProtos.DeleteVolumeResponse deleteVolume( + RpcController controller, KeySpaceManagerProtocolProtos + .DeleteVolumeRequest + request) throws ServiceException { + return null; + } + + @Override + public KeySpaceManagerProtocolProtos.ListVolumeResponse listVolumes( + RpcController controller, + KeySpaceManagerProtocolProtos.ListVolumeRequest request) + throws ServiceException { + return null; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java index 8b94e9672b2..d18aca80ebe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java @@ -26,8 +26,8 @@ import org.apache.commons.cli.ParseException; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Pipeline; import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.Pipeline; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java index cf0076c97f1..c2b74c6ba00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this @@ -19,7 +20,7 @@ package org.apache.hadoop.ozone.scm.container; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -172,7 +173,7 @@ public class ContainerMapping implements Mapping { containerName); } pipeline = Pipeline.getFromProtoBuf( - ContainerProtos.Pipeline.PARSER.parseFrom(pipelineBytes)); + OzoneProtos.Pipeline.PARSER.parseFrom(pipelineBytes)); return pipeline; } finally { lock.unlock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java index 9333fe6d594..18ade6e1035 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyValue; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue; import org.apache.hadoop.ozone.OzoneConsts.Versioning; import org.apache.hadoop.ozone.web.request.OzoneQuota; import org.apache.hadoop.ozone.web.response.BucketInfo; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto index 1ec0d7fceb0..0183baef422 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto @@ -27,7 +27,7 @@ option java_generic_services = true; option java_generate_equals_and_hash = true; package hadoop.cblock; -import "DatanodeContainerProtocol.proto"; +import "Ozone.proto"; /** * This message is sent from CBlock client side to CBlock server to diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto index db814c021f2..38c6dd16f41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto @@ -38,7 +38,7 @@ import "HdfsServer.proto"; import "DatanodeProtocol.proto"; -import "DatanodeContainerProtocol.proto"; +import "Ozone.proto"; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java index aef53e90a5a..a5dbea72df5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ozone.ksm.KSMConfigKeys; import org.apache.hadoop.scm.ScmConfigKeys; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -321,4 +322,29 @@ public class TestOzoneClientUtils { assertTrue(e instanceof IllegalArgumentException); } } + + @Test + public void testGetKSMAddress() { + final Configuration conf = new OzoneConfiguration(); + + // First try a client address with just a host name. Verify it falls + // back to the default port. + conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4"); + InetSocketAddress addr = OzoneClientUtils.getKsmAddress(conf); + assertThat(addr.getHostString(), is("1.2.3.4")); + assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT)); + + // Next try a client address with just a host name and port. Verify the port + // is ignored and the default KSM port is used. + conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4:100"); + addr = OzoneClientUtils.getKsmAddress(conf); + assertThat(addr.getHostString(), is("1.2.3.4")); + assertThat(addr.getPort(), is(100)); + + // Assert the we are able to use default configs if no value is specified. + conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, ""); + addr = OzoneClientUtils.getKsmAddress(conf); + assertThat(addr.getHostString(), is("0.0.0.0")); + assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT)); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java index 89b358aa4f1..89ad9501f30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos .ContainerCommandRequestProto; import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos .ContainerCommandResponseProto; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -361,8 +362,8 @@ public final class ContainerTestHelper { containerData.setName(containerName); String[] keys = metaData.keySet().toArray(new String[]{}); for(int i=0; i