HDDS-213. Single lock to synchronize KeyValueContainer#update.
This commit is contained in:
parent
cb9574a337
commit
44e19fc7f7
|
@ -182,12 +182,14 @@ public class ContainerData {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Adds metadata.
|
* Add/Update metadata.
|
||||||
|
* We should hold the container lock before updating the metadata as this
|
||||||
|
* will be persisted on disk. Unless, we are reconstructing ContainerData
|
||||||
|
* from protoBuf or from on disk .container file in which case lock is not
|
||||||
|
* required.
|
||||||
*/
|
*/
|
||||||
public void addMetadata(String key, String value) throws IOException {
|
public void addMetadata(String key, String value) {
|
||||||
synchronized (this.metadata) {
|
metadata.put(key, value);
|
||||||
metadata.put(key, value);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -195,9 +197,19 @@ public class ContainerData {
|
||||||
* @return metadata
|
* @return metadata
|
||||||
*/
|
*/
|
||||||
public Map<String, String> getMetadata() {
|
public Map<String, String> getMetadata() {
|
||||||
synchronized (this.metadata) {
|
return Collections.unmodifiableMap(this.metadata);
|
||||||
return Collections.unmodifiableMap(this.metadata);
|
}
|
||||||
}
|
|
||||||
|
/**
|
||||||
|
* Set metadata.
|
||||||
|
* We should hold the container lock before updating the metadata as this
|
||||||
|
* will be persisted on disk. Unless, we are reconstructing ContainerData
|
||||||
|
* from protoBuf or from on disk .container file in which case lock is not
|
||||||
|
* required.
|
||||||
|
*/
|
||||||
|
public void setMetadata(Map<String, String> metadataMap) {
|
||||||
|
metadata.clear();
|
||||||
|
metadata.putAll(metadataMap);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -200,15 +200,7 @@ public final class ContainerDataYaml {
|
||||||
OzoneConsts.METADATA_PATH));
|
OzoneConsts.METADATA_PATH));
|
||||||
kvData.setChunksPath((String) nodes.get(OzoneConsts.CHUNKS_PATH));
|
kvData.setChunksPath((String) nodes.get(OzoneConsts.CHUNKS_PATH));
|
||||||
Map<String, String> meta = (Map) nodes.get(OzoneConsts.METADATA);
|
Map<String, String> meta = (Map) nodes.get(OzoneConsts.METADATA);
|
||||||
meta.forEach((key, val) -> {
|
kvData.setMetadata(meta);
|
||||||
try {
|
|
||||||
kvData.addMetadata(key, val);
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw new IllegalStateException("Unexpected " +
|
|
||||||
"Key Value Pair " + "(" + key + "," + val +")in the metadata " +
|
|
||||||
"for containerId " + (long) nodes.get("containerId"));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
String state = (String) nodes.get(OzoneConsts.STATE);
|
String state = (String) nodes.get(OzoneConsts.STATE);
|
||||||
switch (state) {
|
switch (state) {
|
||||||
case "OPEN":
|
case "OPEN":
|
||||||
|
|
|
@ -19,6 +19,8 @@
|
||||||
package org.apache.hadoop.ozone.container.keyvalue;
|
package org.apache.hadoop.ozone.container.keyvalue;
|
||||||
|
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.StandardCopyOption;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
||||||
import org.apache.hadoop.fs.FileUtil;
|
import org.apache.hadoop.fs.FileUtil;
|
||||||
|
@ -32,7 +34,6 @@ import org.apache.hadoop.io.nativeio.NativeIO;
|
||||||
import org.apache.hadoop.ozone.OzoneConfigKeys;
|
import org.apache.hadoop.ozone.OzoneConfigKeys;
|
||||||
import org.apache.hadoop.ozone.OzoneConsts;
|
import org.apache.hadoop.ozone.OzoneConsts;
|
||||||
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
|
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
|
||||||
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
|
|
||||||
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
|
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
|
||||||
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
|
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
|
||||||
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
|
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
|
||||||
|
@ -58,8 +59,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||||
|
|
||||||
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
||||||
.Result.CONTAINER_ALREADY_EXISTS;
|
.Result.CONTAINER_ALREADY_EXISTS;
|
||||||
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
|
||||||
.Result.CONTAINER_METADATA_ERROR;
|
|
||||||
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
||||||
.Result.CONTAINER_INTERNAL_ERROR;
|
.Result.CONTAINER_INTERNAL_ERROR;
|
||||||
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
||||||
|
@ -146,7 +145,7 @@ public class KeyValueContainer implements Container {
|
||||||
containerData.setVolume(containerVolume);
|
containerData.setVolume(containerVolume);
|
||||||
|
|
||||||
// Create .container file and .chksm file
|
// Create .container file and .chksm file
|
||||||
createContainerFile(containerFile, containerCheckSumFile);
|
writeToContainerFile(containerFile, containerCheckSumFile, true);
|
||||||
|
|
||||||
|
|
||||||
} catch (StorageContainerException ex) {
|
} catch (StorageContainerException ex) {
|
||||||
|
@ -177,36 +176,50 @@ public class KeyValueContainer implements Container {
|
||||||
* Creates .container file and checksum file.
|
* Creates .container file and checksum file.
|
||||||
*
|
*
|
||||||
* @param containerFile
|
* @param containerFile
|
||||||
* @param containerCheckSumFile
|
* @param checksumFile
|
||||||
|
* @param isCreate true if we are creating a new container file and false if
|
||||||
|
* we are updating an existing container file.
|
||||||
* @throws StorageContainerException
|
* @throws StorageContainerException
|
||||||
*/
|
*/
|
||||||
private void createContainerFile(File containerFile, File
|
private void writeToContainerFile(File containerFile, File
|
||||||
containerCheckSumFile) throws StorageContainerException {
|
checksumFile, boolean isCreate)
|
||||||
|
throws StorageContainerException {
|
||||||
File tempContainerFile = null;
|
File tempContainerFile = null;
|
||||||
File tempCheckSumFile = null;
|
File tempChecksumFile = null;
|
||||||
FileOutputStream containerCheckSumStream = null;
|
FileOutputStream containerCheckSumStream = null;
|
||||||
Writer writer = null;
|
Writer writer = null;
|
||||||
long containerId = containerData.getContainerID();
|
long containerId = containerData.getContainerID();
|
||||||
try {
|
try {
|
||||||
tempContainerFile = createTempFile(containerFile);
|
tempContainerFile = createTempFile(containerFile);
|
||||||
tempCheckSumFile = createTempFile(containerCheckSumFile);
|
tempChecksumFile = createTempFile(checksumFile);
|
||||||
ContainerDataYaml.createContainerFile(ContainerProtos.ContainerType
|
ContainerDataYaml.createContainerFile(ContainerProtos.ContainerType
|
||||||
.KeyValueContainer, tempContainerFile, containerData);
|
.KeyValueContainer, tempContainerFile, containerData);
|
||||||
|
|
||||||
//Compute Checksum for container file
|
//Compute Checksum for container file
|
||||||
String checksum = KeyValueContainerUtil.computeCheckSum(containerId,
|
String checksum = KeyValueContainerUtil.computeCheckSum(containerId,
|
||||||
tempContainerFile);
|
tempContainerFile);
|
||||||
containerCheckSumStream = new FileOutputStream(tempCheckSumFile);
|
containerCheckSumStream = new FileOutputStream(tempChecksumFile);
|
||||||
writer = new OutputStreamWriter(containerCheckSumStream, "UTF-8");
|
writer = new OutputStreamWriter(containerCheckSumStream, "UTF-8");
|
||||||
writer.write(checksum);
|
writer.write(checksum);
|
||||||
writer.flush();
|
writer.flush();
|
||||||
|
|
||||||
NativeIO.renameTo(tempContainerFile, containerFile);
|
if (isCreate) {
|
||||||
NativeIO.renameTo(tempCheckSumFile, containerCheckSumFile);
|
// When creating a new container, .container file should not exist
|
||||||
|
// already.
|
||||||
|
NativeIO.renameTo(tempContainerFile, containerFile);
|
||||||
|
NativeIO.renameTo(tempChecksumFile, checksumFile);
|
||||||
|
} else {
|
||||||
|
// When updating a container, the .container file should exist. If
|
||||||
|
// not, the container is in an inconsistent state.
|
||||||
|
Files.move(tempContainerFile.toPath(), containerFile.toPath(),
|
||||||
|
StandardCopyOption.REPLACE_EXISTING);
|
||||||
|
Files.move(tempChecksumFile.toPath(), checksumFile.toPath(),
|
||||||
|
StandardCopyOption.REPLACE_EXISTING);
|
||||||
|
}
|
||||||
|
|
||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
throw new StorageContainerException("Error during creation of " +
|
throw new StorageContainerException("Error during creation of " +
|
||||||
"required files(.container, .chksm) for container. Container Name: "
|
"required files(.container, .chksm) for container. ContainerID: "
|
||||||
+ containerId, ex, CONTAINER_FILES_CREATE_ERROR);
|
+ containerId, ex, CONTAINER_FILES_CREATE_ERROR);
|
||||||
} finally {
|
} finally {
|
||||||
IOUtils.closeStream(containerCheckSumStream);
|
IOUtils.closeStream(containerCheckSumStream);
|
||||||
|
@ -216,8 +229,8 @@ public class KeyValueContainer implements Container {
|
||||||
tempContainerFile.getAbsolutePath());
|
tempContainerFile.getAbsolutePath());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (tempCheckSumFile != null && tempCheckSumFile.exists()) {
|
if (tempChecksumFile != null && tempChecksumFile.exists()) {
|
||||||
if (!tempCheckSumFile.delete()) {
|
if (!tempChecksumFile.delete()) {
|
||||||
LOG.warn("Unable to delete container temporary checksum file: {}.",
|
LOG.warn("Unable to delete container temporary checksum file: {}.",
|
||||||
tempContainerFile.getAbsolutePath());
|
tempContainerFile.getAbsolutePath());
|
||||||
}
|
}
|
||||||
|
@ -236,68 +249,24 @@ public class KeyValueContainer implements Container {
|
||||||
|
|
||||||
|
|
||||||
private void updateContainerFile(File containerFile, File
|
private void updateContainerFile(File containerFile, File
|
||||||
containerCheckSumFile) throws StorageContainerException {
|
checksumFile) throws StorageContainerException {
|
||||||
|
|
||||||
File containerBkpFile = null;
|
|
||||||
File checkSumBkpFile = null;
|
|
||||||
long containerId = containerData.getContainerID();
|
long containerId = containerData.getContainerID();
|
||||||
|
|
||||||
try {
|
if (containerFile.exists() && checksumFile.exists()) {
|
||||||
if (containerFile.exists() && containerCheckSumFile.exists()) {
|
|
||||||
//Take backup of original files (.container and .chksm files)
|
|
||||||
containerBkpFile = new File(containerFile + ".bkp");
|
|
||||||
checkSumBkpFile = new File(containerCheckSumFile + ".bkp");
|
|
||||||
NativeIO.renameTo(containerFile, containerBkpFile);
|
|
||||||
NativeIO.renameTo(containerCheckSumFile, checkSumBkpFile);
|
|
||||||
createContainerFile(containerFile, containerCheckSumFile);
|
|
||||||
} else {
|
|
||||||
containerData.setState(ContainerProtos.ContainerLifeCycleState.INVALID);
|
|
||||||
throw new StorageContainerException("Container is an Inconsistent " +
|
|
||||||
"state, missing required files(.container, .chksm). ContainerID: " +
|
|
||||||
containerId, INVALID_CONTAINER_STATE);
|
|
||||||
}
|
|
||||||
} catch (StorageContainerException ex) {
|
|
||||||
throw ex;
|
|
||||||
} catch (IOException ex) {
|
|
||||||
// Restore from back up files.
|
|
||||||
try {
|
try {
|
||||||
if (containerBkpFile != null && containerBkpFile
|
writeToContainerFile(containerFile, checksumFile, false);
|
||||||
.exists() && containerFile.delete()) {
|
|
||||||
LOG.info("update failed for container Name: {}, restoring container" +
|
|
||||||
" file", containerId);
|
|
||||||
NativeIO.renameTo(containerBkpFile, containerFile);
|
|
||||||
}
|
|
||||||
if (checkSumBkpFile != null && checkSumBkpFile.exists() &&
|
|
||||||
containerCheckSumFile.delete()) {
|
|
||||||
LOG.info("update failed for container Name: {}, restoring checksum" +
|
|
||||||
" file", containerId);
|
|
||||||
NativeIO.renameTo(checkSumBkpFile, containerCheckSumFile);
|
|
||||||
}
|
|
||||||
throw new StorageContainerException("Error during updating of " +
|
|
||||||
"required files(.container, .chksm) for container. Container Name: "
|
|
||||||
+ containerId, ex, CONTAINER_FILES_CREATE_ERROR);
|
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
containerData.setState(ContainerProtos.ContainerLifeCycleState.INVALID);
|
//TODO : Container update failure is not handled currently. Might
|
||||||
LOG.error("During restore failed for container Name: " +
|
// lead to loss of .container file. When Update container feature
|
||||||
containerId);
|
// support is added, this failure should also be handled.
|
||||||
throw new StorageContainerException(
|
throw new StorageContainerException("Container update failed. " +
|
||||||
"Failed to restore container data from the backup. ID: "
|
"ContainerID: " + containerId, CONTAINER_FILES_CREATE_ERROR);
|
||||||
+ containerId, CONTAINER_FILES_CREATE_ERROR);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
if (containerBkpFile != null && containerBkpFile
|
|
||||||
.exists()) {
|
|
||||||
if(!containerBkpFile.delete()) {
|
|
||||||
LOG.warn("Unable to delete container backup file: {}",
|
|
||||||
containerBkpFile);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (checkSumBkpFile != null && checkSumBkpFile.exists()) {
|
|
||||||
if(!checkSumBkpFile.delete()) {
|
|
||||||
LOG.warn("Unable to delete container checksum backup file: {}",
|
|
||||||
checkSumBkpFile);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
throw new StorageContainerException("Container is an Inconsistent " +
|
||||||
|
"state, missing required files(.container, .chksm). ContainerID: " +
|
||||||
|
containerId, INVALID_CONTAINER_STATE);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -393,22 +362,21 @@ public class KeyValueContainer implements Container {
|
||||||
"Updating a closed container without force option is not allowed. " +
|
"Updating a closed container without force option is not allowed. " +
|
||||||
"ContainerID: " + containerId, UNSUPPORTED_REQUEST);
|
"ContainerID: " + containerId, UNSUPPORTED_REQUEST);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Map<String, String> oldMetadata = containerData.getMetadata();
|
||||||
try {
|
try {
|
||||||
|
writeLock();
|
||||||
for (Map.Entry<String, String> entry : metadata.entrySet()) {
|
for (Map.Entry<String, String> entry : metadata.entrySet()) {
|
||||||
containerData.addMetadata(entry.getKey(), entry.getValue());
|
containerData.addMetadata(entry.getKey(), entry.getValue());
|
||||||
}
|
}
|
||||||
} catch (IOException ex) {
|
|
||||||
throw new StorageContainerException("Container Metadata update error" +
|
|
||||||
". Container Name:" + containerId, ex, CONTAINER_METADATA_ERROR);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
writeLock();
|
|
||||||
String containerName = String.valueOf(containerId);
|
|
||||||
File containerFile = getContainerFile();
|
File containerFile = getContainerFile();
|
||||||
File containerCheckSumFile = getContainerCheckSumFile();
|
File containerCheckSumFile = getContainerCheckSumFile();
|
||||||
// update the new container data to .container File
|
// update the new container data to .container File
|
||||||
updateContainerFile(containerFile, containerCheckSumFile);
|
updateContainerFile(containerFile, containerCheckSumFile);
|
||||||
} catch (StorageContainerException ex) {
|
} catch (StorageContainerException ex) {
|
||||||
|
// TODO:
|
||||||
|
// On error, reset the metadata.
|
||||||
|
containerData.setMetadata(oldMetadata);
|
||||||
throw ex;
|
throw ex;
|
||||||
} finally {
|
} finally {
|
||||||
writeUnlock();
|
writeUnlock();
|
||||||
|
|
|
@ -109,25 +109,28 @@ public class ContainerReader implements Runnable {
|
||||||
for (File containerTopDir : containerTopDirs) {
|
for (File containerTopDir : containerTopDirs) {
|
||||||
if (containerTopDir.isDirectory()) {
|
if (containerTopDir.isDirectory()) {
|
||||||
File[] containerDirs = containerTopDir.listFiles();
|
File[] containerDirs = containerTopDir.listFiles();
|
||||||
for (File containerDir : containerDirs) {
|
if (containerDirs != null) {
|
||||||
File metadataPath = new File(containerDir + File.separator +
|
for (File containerDir : containerDirs) {
|
||||||
OzoneConsts.CONTAINER_META_PATH);
|
File metadataPath = new File(containerDir + File.separator +
|
||||||
String containerName = containerDir.getName();
|
OzoneConsts.CONTAINER_META_PATH);
|
||||||
if (metadataPath.exists()) {
|
String containerName = containerDir.getName();
|
||||||
File containerFile = KeyValueContainerLocationUtil
|
if (metadataPath.exists()) {
|
||||||
.getContainerFile(metadataPath, containerName);
|
File containerFile = KeyValueContainerLocationUtil
|
||||||
File checksumFile = KeyValueContainerLocationUtil
|
.getContainerFile(metadataPath, containerName);
|
||||||
.getContainerCheckSumFile(metadataPath, containerName);
|
File checksumFile = KeyValueContainerLocationUtil
|
||||||
if (containerFile.exists() && checksumFile.exists()) {
|
.getContainerCheckSumFile(metadataPath, containerName);
|
||||||
verifyContainerFile(containerName, containerFile,
|
if (containerFile.exists() && checksumFile.exists()) {
|
||||||
checksumFile);
|
verifyContainerFile(containerName, containerFile,
|
||||||
|
checksumFile);
|
||||||
|
} else {
|
||||||
|
LOG.error(
|
||||||
|
"Missing container metadata files for Container: " +
|
||||||
|
"{}", containerName);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
LOG.error("Missing container metadata files for Container: " +
|
LOG.error("Missing container metadata directory for " +
|
||||||
"{}", containerName);
|
"Container: {}", containerName);
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
LOG.error("Missing container metadata directory for " +
|
|
||||||
"Container: {}", containerName);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue