honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

create md5 files on the local index storage as well, so they won't have to be recomputed when performing initial recovery

This commit is contained in:
kimchy 2010-08-20 02:28:14 +03:00
parent acae0650c8
commit 6f1a9fbfb9
2 changed files with 56 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.Lists;
import org.elasticsearch.common.collect.Maps;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@ -130,16 +131,36 @@ public abstract class FsIndexStore extends AbstractIndexStore {
}
Map<String, StoreFileMetaData> files = Maps.newHashMap();
for (File file : shardIndexLocation.listFiles()) {
// ignore md5 files
if (file.getName().endsWith(".md5")) {
continue;
}
// calculate md5
FileInputStream is = new FileInputStream(file);
String md5 = shardIdCachedMd5s.get(file.getName());
if (md5 == null) {
try {
md5 = Digest.md5Hex(is);
} finally {
is.close();
// first, see if there is an md5 file associated with it
File md5File = new File(shardIndexLocation, file.getName() + ".md5");
if (md5File.exists()) {
try {
byte[] md5Bytes = Streams.copyToByteArray(md5File);
md5 = Digest.md5HexFromByteArray(md5Bytes);
} catch (Exception e) {
// ignore, compute...
}
}
// not created as a file, compute it
if (md5 == null) {
FileInputStream is = new FileInputStream(file);
try {
md5 = Digest.md5Hex(is);
} finally {
is.close();
}
}
if (md5 != null) {
shardIdCachedMd5s.put(file.getName(), md5);
}
shardIdCachedMd5s.put(file.getName(), md5);
}
files.put(file.getName(), new StoreFileMetaData(file.getName(), file.length(), file.lastModified(), md5));
}

30
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java
View File

@ -98,6 +98,14 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen
return null;
}
if (md.md5() == null) {
byte[] md5Bytes = Digest.md5HexToByteArray(md5);
if (shouldWriteMd5(name)) {
IndexOutput output = directory().createOutput(name + ".md5");
output.writeBytes(md5Bytes, md5Bytes.length);
output.close();
}
md = new StoreFileMetaData(md.name(), md.sizeInBytes(), md.sizeInBytes(), md5);
filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, md).immutableMap();
}
@ -155,6 +163,10 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen
return null;
}
private boolean shouldWriteMd5(String name) {
return !name.startsWith("segments");
}
/**
* The idea of the store directory is to cache file level meta data, as well as md5 of it
*/
@ -168,7 +180,18 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen
MapBuilder<String, StoreFileMetaData> builder = MapBuilder.newMapBuilder();
for (String file : delegate.listAll()) {
try {
builder.put(file, new StoreFileMetaData(file, delegate.fileLength(file), delegate.fileModified(file), preComputedMd5(file)));
String md5 = preComputedMd5(file);
if (md5 != null) {
if (shouldWriteMd5(file)) {
byte[] md5Bytes = Digest.md5HexToByteArray(md5);
IndexOutput output = delegate.createOutput(file + ".md5");
output.writeBytes(md5Bytes, md5Bytes.length);
output.close();
}
}
builder.put(file, new StoreFileMetaData(file, delegate.fileLength(file), delegate.fileModified(file), md5));
} catch (FileNotFoundException e) {
// ignore
}
@ -211,6 +234,11 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen
@Override public void deleteFile(String name) throws IOException {
delegate.deleteFile(name);
try {
delegate.deleteFile(name + ".md5");
} catch (Exception e) {
// ignore
}
synchronized (mutex) {
filesMetadata = MapBuilder.newMapBuilder(filesMetadata).remove(name).immutableMap();
files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]);