Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-936. Need a tool to map containers to ozone objects. Contributed by Sarun Singla

This commit is contained in:
Jitendra Pandey 2019-02-13 15:43:14 -08:00
parent 024c87291c
commit fa067aa157
6 changed files with 410 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-ozone/tools/pom.xml
View File

@ -78,6 +78,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>2.15.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-ozone-manager</artifactId>
<version>0.4.0-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<plugins>

66
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/BlockIdDetails.java Normal file
View File

@ -0,0 +1,66 @@
package org.apache.hadoop.ozone.fsck;
import java.util.Objects;
/**
* Getter and Setter for BlockDetails.
*/
public class BlockIdDetails {
private String bucketName;
private String blockVol;
private String keyName;
public String getBucketName() {
return bucketName;
}
public void setBucketName(String bucketName) {
this.bucketName = bucketName;
}
public String getBlockVol() {
return blockVol;
}
public void setBlockVol(String blockVol) {
this.blockVol = blockVol;
}
public String getKeyName() {
return keyName;
}
public void setKeyName(String keyName) {
this.keyName = keyName;
}
@Override
public String toString() {
return "BlockIdDetails{" +
"bucketName='" + bucketName + '\'' +
", blockVol='" + blockVol + '\'' +
", keyName='" + keyName + '\'' +
'}';
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
BlockIdDetails that = (BlockIdDetails) o;
return Objects.equals(bucketName, that.bucketName) &&
Objects.equals(blockVol, that.blockVol) &&
Objects.equals(keyName, that.keyName);
}
@Override
public int hashCode() {
return Objects.hash(bucketName, blockVol, keyName);
}
}

134
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java Normal file
View File

@ -0,0 +1,134 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.fsck;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.utils.db.Table;
import org.apache.hadoop.utils.db.TableIterator;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
/**
* Generates Container Id to Blocks and BlockDetails mapping.
*/
public class ContainerMapper {
private static Table getMetaTable(OzoneConfiguration configuration)
throws IOException {
OmMetadataManagerImpl metadataManager =
new OmMetadataManagerImpl(configuration);
return metadataManager.getKeyTable();
}
public static void main(String[] args) throws IOException {
String path = args[0];
if (path == null) {
throw new IOException("Path cannot be null");
}
OzoneConfiguration configuration = new OzoneConfiguration();
configuration.set(OZONE_OM_DB_DIRS, path);
ContainerMapper containerMapper = new ContainerMapper();
Map<Long, List<Map<Long, BlockIdDetails>>> dataMap =
containerMapper.parseOmDB(configuration);
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writeValueAsString(dataMap));
}
/**
* Generates Container Id to Blocks and BlockDetails mapping.
* @param configuration @{@link OzoneConfiguration}
* @return Map<Long, List<Map<Long, @BlockDetails>>>
* Map of ContainerId -> (Block, Block info)
* @throws IOException
*/
public Map<Long, List<Map<Long, BlockIdDetails>>>
parseOmDB(OzoneConfiguration configuration) throws IOException {
String path = configuration.get(OZONE_OM_DB_DIRS);
if (path == null || path.isEmpty()) {
throw new IOException(OZONE_OM_DB_DIRS + "should be set ");
} else {
Table keyTable = getMetaTable(configuration);
Map<Long, List<Map<Long, BlockIdDetails>>> dataMap = new HashMap<>();
if (keyTable != null) {
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
keyValueTableIterator = keyTable.iterator()) {
while (keyValueTableIterator.hasNext()) {
Table.KeyValue<String, OmKeyInfo> keyValue =
keyValueTableIterator.next();
OmKeyInfo omKeyInfo = keyValue.getValue();
byte[] value = omKeyInfo.getProtobuf().toByteArray();
OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(
OzoneManagerProtocolProtos.KeyInfo.parseFrom(value));
for (OmKeyLocationInfoGroup keyLocationInfoGroup : keyInfo
.getKeyLocationVersions()) {
List<OmKeyLocationInfo> keyLocationInfo = keyLocationInfoGroup
.getLocationList();
for (OmKeyLocationInfo keyLocation : keyLocationInfo) {
BlockIdDetails blockIdDetails = new BlockIdDetails();
Map<Long, BlockIdDetails> innerMap = new HashMap<>();
long containerID = keyLocation.getBlockID().getContainerID();
long blockID = keyLocation.getBlockID().getLocalID();
blockIdDetails.setBucketName(keyInfo.getBucketName());
blockIdDetails.setBlockVol(keyInfo.getVolumeName());
blockIdDetails.setKeyName(keyInfo.getKeyName());
List<Map<Long, BlockIdDetails>> innerList = new ArrayList<>();
innerMap.put(blockID, blockIdDetails);
if (dataMap.containsKey(containerID)) {
innerList = dataMap.get(containerID);
}
innerList.add(innerMap);
dataMap.put(containerID, innerList);
}
}
}
}
}
return dataMap;
}
}
}

44
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/package-info.java Normal file
View File

@ -0,0 +1,44 @@
/**
* Package info.
* <p>
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* <p>
* fsck tool.
*/
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* fsck tool.
*/
package org.apache.hadoop.ozone.fsck;

117
hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java Normal file
View File

@ -0,0 +1,117 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.fsck;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.client.*;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.ratis.util.FileUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.*;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
import static org.junit.Assert.*;
/**
* Test cases for ContainerMapper.
*/
public class TestContainerMapper {
private static MiniOzoneCluster cluster = null;
private static OzoneClient ozClient = null;
private static ObjectStore store = null;
private static OzoneManager ozoneManager;
private static StorageContainerLocationProtocolClientSideTranslatorPB
storageContainerLocationClient;
private static final String SCM_ID = UUID.randomUUID().toString();
private static String volName = UUID.randomUUID().toString();
private static String bucketName = UUID.randomUUID().toString();
private static OzoneConfiguration conf;
private static List<String> keyList = new ArrayList<>();
private static String dbPath;
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
dbPath = GenericTestUtils.getRandomizedTempPath();
conf.set(OZONE_OM_DB_DIRS, dbPath);
conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, "100MB");
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(1)
.setScmId(SCM_ID)
.build();
cluster.waitForClusterToBeReady();
ozClient = OzoneClientFactory.getRpcClient(conf);
store = ozClient.getObjectStore();
storageContainerLocationClient =
cluster.getStorageContainerLocationClient();
ozoneManager = cluster.getOzoneManager();
store.createVolume(volName);
OzoneVolume volume = store.getVolume(volName);
volume.createBucket(bucketName);
OzoneBucket bucket = volume.getBucket(bucketName);
byte[] data = generateData(10 * 1024 * 1024, (byte)98);
for (int i = 0; i < 20; i++) {
String key = UUID.randomUUID().toString();
keyList.add(key);
OzoneOutputStream out = bucket.createKey(key, data.length,
ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
new HashMap<String, String>());
out.write(data, 0, data.length);
out.close();
}
cluster.stop();
}
@Test
public void testContainerMapper() throws Exception {
ContainerMapper containerMapper = new ContainerMapper();
Map<Long, List<Map<Long, BlockIdDetails>>> dataMap =
containerMapper.parseOmDB(conf);
// As we have created 20 keys with 10 MB size, and each
// container max size is 100 MB, it should create 3 containers because
// containers are closing before reaching the threshold
assertEquals(3, dataMap.size());
}
private static byte[] generateData(int size, byte val) {
byte[] chars = new byte[size];
Arrays.fill(chars, val);
return chars;
}
@AfterClass
public static void shutdown() throws IOException {
cluster.shutdown();
FileUtils.deleteFully(new File(dbPath));
}
}

44
hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/fsck/package-info.java Normal file
View File

@ -0,0 +1,44 @@
/**
* Package info.
* <p>
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* <p>
* fsck tool.
*/
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* fsck tool.
*/
package org.apache.hadoop.ozone.fsck;