From ba14d7777498f576694174e18e27dae0c848d509 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 11 Jan 2012 07:28:23 +0000 Subject: [PATCH] svn merge -c 1204544 from trunk for HDFS-2517. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23-PB@1229912 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../src/main/proto/JournalProtocol.proto | 83 +++++++++ .../hadoop-hdfs/src/main/proto/hdfs.proto | 168 +++++++++++++++++- 3 files changed, 247 insertions(+), 6 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b21ae549cdc..448b11ec23f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -7,6 +7,8 @@ Release 0.23-PB - Unreleased HDFS-395. DFS Scalability: Incremental block reports. (Tomasz Nykiel via hairong) + HDFS-2517. Add protobuf service for JounralProtocol. (suresh) + IMPROVEMENTS HDFS-2018. Move all journal stream management code into one place. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto new file mode 100644 index 00000000000..fa7ed15c64a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers that are used throughout HDFS -- i.e. +// by the client, server, and data transfer protocols. + +option java_package = "org.apache.hadoop.hdfs.protocol.proto"; +option java_outer_classname = "JournalProtocolProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; + +import "hdfs.proto"; + +/** + * registration - the registration info of the active NameNode + * firstTxnId - the first txid in the rolled edit log + * numTxns - Number of transactions in editlog + * records - bytes containing serialized journal records + */ +message JournalRequestProto { + required NamenodeRegistrationProto registration = 1; // Registration info + required uint64 firstTxnId = 2; // Transaction ID + required uint32 numTxns = 3; // Transaction ID + required bytes records = 4; // Journal record +} + +/** + * void response + */ +message JournalResponseProto { +} + +/** + * registration - the registration info of the active NameNode + * txid - first txid in the new log + */ +message StartLogSegmentRequestProto { + required NamenodeRegistrationProto registration = 1; // Registration info + required uint64 txid = 2; // Transaction ID +} + +/** + * void response + */ +message StartLogSegmentResponseProto { +} + +/** + * Protocol used to journal edits to a remote node. Currently, + * this is used to publish edits from the NameNode to a BackupNode. + * + * See the request and response for details of rpc call. + */ +service JournalProtocolService { + /** + * Request sent by active namenode to backup node via + * EditLogBackupOutputStream to stream editlog records. + */ + rpc journal(JournalRequestProto) returns (JournalResponseProto); + + /** + * Request sent by active namenode to backup node to notify + * that the NameNode has rolled its edit logs and is now writing a + * new log segment. + */ + rpc startLogSegment(StartLogSegmentRequestProto) + returns (StartLogSegmentResponseProto); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index a77a7c312e8..8b86066980d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -53,6 +53,12 @@ message DatanodeIDProto { required uint32 ipcPort = 4; // the port where the ipc Server is running } +/** + * DatanodeID array + */ +message DatanodeIDsProto { + repeated DatanodeIDProto datanodes = 1; +} /** * The status of a Datanode @@ -76,7 +82,6 @@ message DatanodeInfoProto { optional AdminState adminState = 10; } - /** * Summary of a file or directory */ @@ -152,10 +157,10 @@ message HdfsFileStatusProto { required string group = 6; required uint64 modification_time = 7; required uint64 access_time = 8; - // + // Optional fields for symlink - optional bytes symlink = 9; // if symlink, target encoded java UTF8 - // + optional bytes symlink = 9; // if symlink, target encoded java UTF8 + // Optional fields for file optional uint32 block_replication = 10; // Actually a short - only 16bits used optional uint64 blocksize = 11; @@ -169,7 +174,7 @@ message FsServerDefaultsProto { required uint64 blockSize = 1; required uint32 bytesPerChecksum = 2; required uint32 writePacketSize = 3; - required uint32 replication = 4; // Actually a short - only 16bits used + required uint32 replication = 4; // Actually a short - only 16 bits used required uint32 fileBufferSize = 5; } @@ -187,5 +192,156 @@ message DirectoryListingProto { */ message UpgradeStatusReportProto { required uint32 version = 1;; - required uint32 upgradeStatus = 2; // Between 0 and 100 indicating the % complete + required uint32 upgradeStatus = 2; // % completed in range 0 & 100 } + +/** + * Common node information shared by all the nodes in the cluster + */ +message StorageInfoProto { + required uint32 layoutVersion = 1; // Layout version of the file system + required uint32 namespceID = 2; // File system namespace ID + required string clusterID = 3; // ID of the cluster + required uint64 cTime = 4; // File system creation time +} + +/** + * Information sent by a namenode to identify itself to the primary namenode. + */ +message NamenodeRegistrationProto { + required string rpcAddress = 1; // host:port of the namenode RPC address + required string httpAddress = 2; // host:port of the namenode http server + enum NamenodeRoleProto { + NAMENODE = 1; + BACKUP = 2; + CHECKPOINT = 3; + } + required StorageInfoProto storageInfo = 3; // Node information + optional NamenodeRoleProto role = 4; // Namenode role +} + +/** + * Unique signature to identify checkpoint transactions. + */ +message CheckpointSignatureProto { + required string blockPoolId = 1; + required uint64 mostRecentCheckpointTxId = 2; + required uint64 curSegmentTxId = 3; + required StorageInfoProto storageInfo = 4; +} + +/** + * Command sent from one namenode to another namenode. + */ +message NamenodeCommandProto { + enum Type { + NamenodeCommand = 0; // Base command + CheckPointCommand = 1; // Check point command + } + required uint32 action = 1; + required Type type = 2; + optional CheckpointCommandProto checkpointCmd = 3; +} + +/** + * Command returned from primary to checkpointing namenode. + * This command has checkpoint signature that identifies + * checkpoint transaction and is needed for further + * communication related to checkpointing. + */ +message CheckpointCommandProto { + // Unique signature to identify checkpoint transation + required CheckpointSignatureProto signature = 1; + + // If true, return transfer image to primary upon the completion of checkpoint + required bool needToReturnImage = 2; +} + +/** + * Block information + */ +message BlockProto { + required uint64 blockId = 1; + required uint64 genStamp = 2; + optional uint64 numBytes = 3; +} + +/** + * Block and datanodes where is it located + */ +message BlockWithLocationsProto { + required BlockProto block = 1; // Block + repeated DatanodeIDProto datanodeIDs = 2; // Datanodes with replicas of the block +} + +/** + * List of block with locations + */ +message BlocksWithLocationsProto { + repeated BlockWithLocationsProto blocks = 1; +} + +/** + * Editlog information with available transactions + */ +message RemoteEditLogProto { + required uint64 startTxId = 1; // Starting available edit log transaction + required uint64 endTxId = 2; // Ending available edit log transaction +} + +/** + * Enumeration of editlogs available on a remote namenode + */ +message RemoteEditLogManifestProto { + repeated RemoteEditLogProto logs = 1; +} + +/** + * Namespace information that describes namespace on a namenode + */ +message NamespaceInfoProto { + required string buildVersion = 1; // Software build version + required uint32 distUpgradeVersion = 2; // Distributed upgrade version + required string blockPoolID = 3; // block pool used by the namespace + required StorageInfoProto storageInfo = 4;// Noe information +} + +/** + * Block access token information + */ +message BlockKeyProto { + required uint32 keyId = 1; // Key identifier + required uint64 expiryDate = 2; // Expiry time in milliseconds + required bytes keyBytes = 3; // Key secret +} + +/** + * Current key and set of block keys at the namenode. + */ +message ExportedBlockKeysProto { + required bool isBlockTokenEnabled = 1; + required uint64 keyUpdateInterval = 2; + required uint64 tokenLifeTime = 3; + required BlockKeyProto currentKey = 4; + repeated BlockKeyProto allKeys = 5; +} + +/** + * State of a block replica at a datanode + */ +enum ReplicaState { + FINALIZED = 0; // State of a replica when it is not modified + RBW = 1; // State of replica that is being written to + RWR = 2; // State of replica that is waiting to be recovered + RUR = 3; // State of replica that is under recovery + TEMPORARY = 4; // State of replica that is created for replication +} + +/** + * Block that needs to be recovered with at a given location + */ +message RecoveringBlockProto { + required uint64 newGenStamp = 1; // New genstamp post recovery + required LocatedBlockProto block = 2; // Block to be recovered +} +