HDDS-936. Need a tool to map containers to ozone objects. Contributed by Sarun Singla
This commit is contained in:
Jitendra Pandey
parent
024c87291c
commit
fa067aa157
|
|
@ -78,6 +78,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
<version>2.15.0</version>
|
<version>2.15.0</version>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-ozone-ozone-manager</artifactId>
|
||||||
|
<version>0.4.0-SNAPSHOT</version>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
<build>
|
<build>
|
||||||
<plugins>
|
<plugins>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
package org.apache.hadoop.ozone.fsck;
|
||||||
|
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Getter and Setter for BlockDetails.
|
||||||
|
*/
|
||||||
|
|
||||||
|
public class BlockIdDetails {
|
||||||
|
|
||||||
|
private String bucketName;
|
||||||
|
private String blockVol;
|
||||||
|
private String keyName;
|
||||||
|
|
||||||
|
public String getBucketName() {
|
||||||
|
return bucketName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setBucketName(String bucketName) {
|
||||||
|
this.bucketName = bucketName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getBlockVol() {
|
||||||
|
return blockVol;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setBlockVol(String blockVol) {
|
||||||
|
this.blockVol = blockVol;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getKeyName() {
|
||||||
|
return keyName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setKeyName(String keyName) {
|
||||||
|
this.keyName = keyName;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "BlockIdDetails{" +
|
||||||
|
"bucketName='" + bucketName + '\'' +
|
||||||
|
", blockVol='" + blockVol + '\'' +
|
||||||
|
", keyName='" + keyName + '\'' +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (this == o) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (o == null || getClass() != o.getClass()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
BlockIdDetails that = (BlockIdDetails) o;
|
||||||
|
return Objects.equals(bucketName, that.bucketName) &&
|
||||||
|
Objects.equals(blockVol, that.blockVol) &&
|
||||||
|
Objects.equals(keyName, that.keyName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Objects.hash(bucketName, blockVol, keyName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,134 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* <p>
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.ozone.fsck;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
|
||||||
|
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
|
||||||
|
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
|
||||||
|
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
|
||||||
|
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
|
||||||
|
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
|
||||||
|
import org.apache.hadoop.utils.db.Table;
|
||||||
|
import org.apache.hadoop.utils.db.TableIterator;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates Container Id to Blocks and BlockDetails mapping.
|
||||||
|
*/
|
||||||
|
|
||||||
|
public class ContainerMapper {
|
||||||
|
|
||||||
|
|
||||||
|
private static Table getMetaTable(OzoneConfiguration configuration)
|
||||||
|
throws IOException {
|
||||||
|
OmMetadataManagerImpl metadataManager =
|
||||||
|
new OmMetadataManagerImpl(configuration);
|
||||||
|
return metadataManager.getKeyTable();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args) throws IOException {
|
||||||
|
String path = args[0];
|
||||||
|
if (path == null) {
|
||||||
|
throw new IOException("Path cannot be null");
|
||||||
|
}
|
||||||
|
|
||||||
|
OzoneConfiguration configuration = new OzoneConfiguration();
|
||||||
|
configuration.set(OZONE_OM_DB_DIRS, path);
|
||||||
|
|
||||||
|
ContainerMapper containerMapper = new ContainerMapper();
|
||||||
|
Map<Long, List<Map<Long, BlockIdDetails>>> dataMap =
|
||||||
|
containerMapper.parseOmDB(configuration);
|
||||||
|
|
||||||
|
|
||||||
|
ObjectMapper mapper = new ObjectMapper();
|
||||||
|
System.out.println(mapper.writeValueAsString(dataMap));
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generates Container Id to Blocks and BlockDetails mapping.
|
||||||
|
* @param configuration @{@link OzoneConfiguration}
|
||||||
|
* @return Map<Long, List<Map<Long, @BlockDetails>>>
|
||||||
|
* Map of ContainerId -> (Block, Block info)
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public Map<Long, List<Map<Long, BlockIdDetails>>>
|
||||||
|
parseOmDB(OzoneConfiguration configuration) throws IOException {
|
||||||
|
String path = configuration.get(OZONE_OM_DB_DIRS);
|
||||||
|
if (path == null || path.isEmpty()) {
|
||||||
|
throw new IOException(OZONE_OM_DB_DIRS + "should be set ");
|
||||||
|
} else {
|
||||||
|
Table keyTable = getMetaTable(configuration);
|
||||||
|
Map<Long, List<Map<Long, BlockIdDetails>>> dataMap = new HashMap<>();
|
||||||
|
|
||||||
|
if (keyTable != null) {
|
||||||
|
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
|
||||||
|
keyValueTableIterator = keyTable.iterator()) {
|
||||||
|
while (keyValueTableIterator.hasNext()) {
|
||||||
|
Table.KeyValue<String, OmKeyInfo> keyValue =
|
||||||
|
keyValueTableIterator.next();
|
||||||
|
OmKeyInfo omKeyInfo = keyValue.getValue();
|
||||||
|
byte[] value = omKeyInfo.getProtobuf().toByteArray();
|
||||||
|
OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(
|
||||||
|
OzoneManagerProtocolProtos.KeyInfo.parseFrom(value));
|
||||||
|
for (OmKeyLocationInfoGroup keyLocationInfoGroup : keyInfo
|
||||||
|
.getKeyLocationVersions()) {
|
||||||
|
List<OmKeyLocationInfo> keyLocationInfo = keyLocationInfoGroup
|
||||||
|
.getLocationList();
|
||||||
|
for (OmKeyLocationInfo keyLocation : keyLocationInfo) {
|
||||||
|
BlockIdDetails blockIdDetails = new BlockIdDetails();
|
||||||
|
Map<Long, BlockIdDetails> innerMap = new HashMap<>();
|
||||||
|
|
||||||
|
long containerID = keyLocation.getBlockID().getContainerID();
|
||||||
|
long blockID = keyLocation.getBlockID().getLocalID();
|
||||||
|
blockIdDetails.setBucketName(keyInfo.getBucketName());
|
||||||
|
blockIdDetails.setBlockVol(keyInfo.getVolumeName());
|
||||||
|
blockIdDetails.setKeyName(keyInfo.getKeyName());
|
||||||
|
|
||||||
|
List<Map<Long, BlockIdDetails>> innerList = new ArrayList<>();
|
||||||
|
innerMap.put(blockID, blockIdDetails);
|
||||||
|
|
||||||
|
if (dataMap.containsKey(containerID)) {
|
||||||
|
innerList = dataMap.get(containerID);
|
||||||
|
}
|
||||||
|
|
||||||
|
innerList.add(innerMap);
|
||||||
|
dataMap.put(containerID, innerList);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return dataMap;
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
/**
|
||||||
|
* Package info.
|
||||||
|
* <p>
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
* <p>
|
||||||
|
* fsck tool.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* fsck tool.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
package org.apache.hadoop.ozone.fsck;
|
||||||
|
|
@ -0,0 +1,117 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.ozone.fsck;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hdds.client.ReplicationFactor;
|
||||||
|
import org.apache.hadoop.hdds.client.ReplicationType;
|
||||||
|
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
|
||||||
|
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
|
||||||
|
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
|
||||||
|
import org.apache.hadoop.ozone.MiniOzoneCluster;
|
||||||
|
import org.apache.hadoop.ozone.client.*;
|
||||||
|
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
|
||||||
|
import org.apache.hadoop.ozone.om.OzoneManager;
|
||||||
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
|
import org.apache.ratis.util.FileUtils;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.*;
|
||||||
|
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test cases for ContainerMapper.
|
||||||
|
*/
|
||||||
|
|
||||||
|
public class TestContainerMapper {
|
||||||
|
|
||||||
|
private static MiniOzoneCluster cluster = null;
|
||||||
|
private static OzoneClient ozClient = null;
|
||||||
|
private static ObjectStore store = null;
|
||||||
|
private static OzoneManager ozoneManager;
|
||||||
|
private static StorageContainerLocationProtocolClientSideTranslatorPB
|
||||||
|
storageContainerLocationClient;
|
||||||
|
private static final String SCM_ID = UUID.randomUUID().toString();
|
||||||
|
private static String volName = UUID.randomUUID().toString();
|
||||||
|
private static String bucketName = UUID.randomUUID().toString();
|
||||||
|
private static OzoneConfiguration conf;
|
||||||
|
private static List<String> keyList = new ArrayList<>();
|
||||||
|
private static String dbPath;
|
||||||
|
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void init() throws Exception {
|
||||||
|
conf = new OzoneConfiguration();
|
||||||
|
dbPath = GenericTestUtils.getRandomizedTempPath();
|
||||||
|
conf.set(OZONE_OM_DB_DIRS, dbPath);
|
||||||
|
conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, "100MB");
|
||||||
|
cluster = MiniOzoneCluster.newBuilder(conf)
|
||||||
|
.setNumDatanodes(1)
|
||||||
|
.setScmId(SCM_ID)
|
||||||
|
.build();
|
||||||
|
cluster.waitForClusterToBeReady();
|
||||||
|
ozClient = OzoneClientFactory.getRpcClient(conf);
|
||||||
|
store = ozClient.getObjectStore();
|
||||||
|
storageContainerLocationClient =
|
||||||
|
cluster.getStorageContainerLocationClient();
|
||||||
|
ozoneManager = cluster.getOzoneManager();
|
||||||
|
store.createVolume(volName);
|
||||||
|
OzoneVolume volume = store.getVolume(volName);
|
||||||
|
volume.createBucket(bucketName);
|
||||||
|
OzoneBucket bucket = volume.getBucket(bucketName);
|
||||||
|
byte[] data = generateData(10 * 1024 * 1024, (byte)98);
|
||||||
|
|
||||||
|
for (int i = 0; i < 20; i++) {
|
||||||
|
String key = UUID.randomUUID().toString();
|
||||||
|
keyList.add(key);
|
||||||
|
OzoneOutputStream out = bucket.createKey(key, data.length,
|
||||||
|
ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
|
||||||
|
new HashMap<String, String>());
|
||||||
|
out.write(data, 0, data.length);
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
|
cluster.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testContainerMapper() throws Exception {
|
||||||
|
ContainerMapper containerMapper = new ContainerMapper();
|
||||||
|
Map<Long, List<Map<Long, BlockIdDetails>>> dataMap =
|
||||||
|
containerMapper.parseOmDB(conf);
|
||||||
|
// As we have created 20 keys with 10 MB size, and each
|
||||||
|
// container max size is 100 MB, it should create 3 containers because
|
||||||
|
// containers are closing before reaching the threshold
|
||||||
|
assertEquals(3, dataMap.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] generateData(int size, byte val) {
|
||||||
|
byte[] chars = new byte[size];
|
||||||
|
Arrays.fill(chars, val);
|
||||||
|
return chars;
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void shutdown() throws IOException {
|
||||||
|
cluster.shutdown();
|
||||||
|
FileUtils.deleteFully(new File(dbPath));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
/**
|
||||||
|
* Package info.
|
||||||
|
* <p>
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
* <p>
|
||||||
|
* fsck tool.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* fsck tool.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
package org.apache.hadoop.ozone.fsck;
|
||||||
Loading…
Reference in New Issue