honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Translog: stats fail to serialize size 

Closes #10105
This commit is contained in:
Boaz Leskes 2015-03-16 09:04:52 -07:00
parent 5e9de0f8bc
commit a1f5c342af
2 changed files with 98 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/main/java/org/elasticsearch/index/translog/TranslogStats.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.translog;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
@ -35,9 +36,12 @@ public class TranslogStats implements ToXContent, Streamable {
private long translogSizeInBytes = 0;
private int estimatedNumberOfOperations = 0;
public TranslogStats() {}
public TranslogStats() {
}
public TranslogStats(int estimatedNumberOfOperations, long translogSizeInBytes) {
assert estimatedNumberOfOperations >= 0 : "estimatedNumberOfOperations must be >=0, got [" + estimatedNumberOfOperations + "]";
assert translogSizeInBytes >= 0 : "translogSizeInBytes must be >=0, got [" + translogSizeInBytes + "]";
this.estimatedNumberOfOperations = estimatedNumberOfOperations;
this.translogSizeInBytes = translogSizeInBytes;
}
@ -48,7 +52,15 @@ public class TranslogStats implements ToXContent, Streamable {
}
this.estimatedNumberOfOperations += translogStats.estimatedNumberOfOperations;
this.translogSizeInBytes =+ translogStats.translogSizeInBytes;
this.translogSizeInBytes = +translogStats.translogSizeInBytes;
}
public ByteSizeValue translogSizeInBytes() {
return new ByteSizeValue(translogSizeInBytes);
}
public long estimatedNumberOfOperations() {
return estimatedNumberOfOperations;
}
@Override
@ -70,10 +82,12 @@ public class TranslogStats implements ToXContent, Streamable {
@Override
public void readFrom(StreamInput in) throws IOException {
estimatedNumberOfOperations = in.readVInt();
translogSizeInBytes = in.readVLong();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(estimatedNumberOfOperations);
out.writeVLong(translogSizeInBytes);
}
}

113
src/test/java/org/elasticsearch/index/translog/AbstractSimpleTranslogTests.java
View File

@ -25,11 +25,12 @@ import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.io.stream.BytesStreamInput;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.Before;
@ -50,8 +51,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import static com.google.common.collect.Lists.newArrayList;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.*;
/**
*
@ -111,32 +111,32 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
@Test
public void testTransientTranslog() throws IOException {
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
snapshot.close();
translog.add(new Translog.Create("test", "1", new byte[]{1}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
snapshot.close();
translog.newTransientTranslog(2);
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
snapshot.close();
translog.add(new Translog.Index("test", "2", new byte[]{2}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(2));
assertThat(snapshot, TranslogSizeMatcher.translogSize(2));
assertThat(snapshot.estimatedTotalOperations(), equalTo(2));
snapshot.close();
translog.makeTransientCurrent();
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1)); // now its one, since it only includes "2"
assertThat(snapshot, TranslogSizeMatcher.translogSize(1)); // now its one, since it only includes "2"
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
snapshot.close();
}
@ -144,30 +144,30 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
@Test
public void testSimpleOperations() throws IOException {
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
snapshot.close();
translog.add(new Translog.Create("test", "1", new byte[]{1}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
snapshot.close();
translog.add(new Translog.Index("test", "2", new byte[]{2}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(2));
assertThat(snapshot, TranslogSizeMatcher.translogSize(2));
assertThat(snapshot.estimatedTotalOperations(), equalTo(2));
snapshot.close();
translog.add(new Translog.Delete(newUid("3")));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(3));
assertThat(snapshot, TranslogSizeMatcher.translogSize(3));
assertThat(snapshot.estimatedTotalOperations(), equalTo(3));
snapshot.close();
translog.add(new Translog.DeleteByQuery(new BytesArray(new byte[]{4}), null));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(4));
assertThat(snapshot, TranslogSizeMatcher.translogSize(4));
assertThat(snapshot.estimatedTotalOperations(), equalTo(4));
snapshot.close();
@ -198,7 +198,7 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
assertThat(translog.currentId(), Matchers.not(equalTo(firstId)));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot.estimatedTotalOperations(), equalTo(0));
snapshot.close();
}
@ -212,15 +212,66 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
}
}
protected TranslogStats stats() throws IOException {
// force flushing and updating of stats
translog.sync();
TranslogStats stats = translog.stats();
if (randomBoolean()) {
BytesStreamOutput out = new BytesStreamOutput();
stats.writeTo(out);
BytesStreamInput in = new BytesStreamInput(out.bytes());
stats = new TranslogStats();
stats.readFrom(in);
}
return stats;
}
@Test
public void testStats() throws IOException {
TranslogStats stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(0l));
long lastSize = stats.translogSizeInBytes().bytes();
assertThat(lastSize, equalTo(17l));
translog.add(new Translog.Create("test", "1", new byte[]{1}));
stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(1l));
assertThat(stats.translogSizeInBytes().bytes(), greaterThan(lastSize));
lastSize = stats.translogSizeInBytes().bytes();
translog.add(new Translog.Index("test", "2", new byte[]{2}));
stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(2l));
assertThat(stats.translogSizeInBytes().bytes(), greaterThan(lastSize));
lastSize = stats.translogSizeInBytes().bytes();
translog.add(new Translog.Delete(newUid("3")));
stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(3l));
assertThat(stats.translogSizeInBytes().bytes(), greaterThan(lastSize));
lastSize = stats.translogSizeInBytes().bytes();
translog.add(new Translog.DeleteByQuery(new BytesArray(new byte[]{4}), null));
stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(4l));
assertThat(stats.translogSizeInBytes().bytes(), greaterThan(lastSize));
translog.newTranslog(2);
stats = stats();
assertThat(stats.estimatedNumberOfOperations(), equalTo(0l));
assertThat(stats.translogSizeInBytes().bytes(), equalTo(17l));
}
@Test
public void testSnapshot() {
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
snapshot.close();
translog.add(new Translog.Create("test", "1", new byte[]{1}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
snapshot.close();
@ -231,7 +282,7 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
snapshot.close();
Translog.Snapshot snapshot1 = translog.snapshot();
MatcherAssert.assertThat(snapshot1, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot1, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot1.estimatedTotalOperations(), equalTo(1));
// seek to the end of the translog snapshot
@ -241,7 +292,7 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
translog.add(new Translog.Index("test", "2", new byte[]{2}));
snapshot = translog.snapshot(snapshot1);
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(2));
snapshot.close();
@ -258,7 +309,7 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
@Test
public void testSnapshotWithNewTranslog() throws IOException {
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
snapshot.close();
translog.add(new Translog.Create("test", "1", new byte[]{1}));
@ -271,7 +322,7 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
translog.add(new Translog.Index("test", "3", new byte[]{3}));
snapshot = translog.snapshot(actualSnapshot);
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
snapshot.close();
snapshot = translog.snapshot(actualSnapshot);
@ -289,22 +340,22 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
translog.add(new Translog.Create("test", "1", new byte[]{1}));
Translog.Snapshot firstSnapshot = translog.snapshot();
MatcherAssert.assertThat(firstSnapshot, TranslogSizeMatcher.translogSize(1));
assertThat(firstSnapshot, TranslogSizeMatcher.translogSize(1));
assertThat(firstSnapshot.estimatedTotalOperations(), equalTo(1));
translog.newTransientTranslog(2);
assertFileIsPresent(translog, 1);
translog.add(new Translog.Index("test", "2", new byte[]{2}));
MatcherAssert.assertThat(firstSnapshot, TranslogSizeMatcher.translogSize(1));
assertThat(firstSnapshot, TranslogSizeMatcher.translogSize(1));
assertThat(firstSnapshot.estimatedTotalOperations(), equalTo(1));
if (randomBoolean()) {
translog.clearUnreferenced();
}
translog.makeTransientCurrent();
Translog.Snapshot secondSnapshot = translog.snapshot();
Translog.Snapshot secondSnapshot = translog.snapshot();
translog.add(new Translog.Index("test", "3", new byte[]{3}));
MatcherAssert.assertThat(secondSnapshot, TranslogSizeMatcher.translogSize(1));
assertThat(secondSnapshot, TranslogSizeMatcher.translogSize(1));
assertThat(secondSnapshot.estimatedTotalOperations(), equalTo(1));
assertFileIsPresent(translog, 1);
assertFileIsPresent(translog, 2);
@ -337,7 +388,7 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
public void assertFileIsPresent(Translog translog, long id) {
for (Path location : translog.locations()) {
if(Files.exists(location.resolve(translog.getPath(id)))) {
if (Files.exists(location.resolve(translog.getPath(id)))) {
return;
}
}
@ -353,12 +404,12 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
@Test
public void testSnapshotWithSeekTo() {
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
snapshot.close();
translog.add(new Translog.Create("test", "1", new byte[]{1}));
snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
// seek to the end of the translog snapshot
while (snapshot.next() != null) {
// spin
@ -369,7 +420,7 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
translog.add(new Translog.Create("test", "2", new byte[]{1}));
snapshot = translog.snapshot();
snapshot.seekTo(lastPosition);
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
snapshot.close();
snapshot = translog.snapshot();
@ -584,8 +635,8 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
int corruptions = scaledRandomIntBetween(10, 50);
for (int i = 0; i < corruptions; i++) {
// note: with the current logic, this will sometimes be a no-op
long pos = randomIntBetween(0, (int)f.size());
ByteBuffer junk = ByteBuffer.wrap(new byte[] { randomByte() });
long pos = randomIntBetween(0, (int) f.size());
ByteBuffer junk = ByteBuffer.wrap(new byte[]{randomByte()});
f.write(junk, pos);
}
f.close();
@ -603,7 +654,7 @@ public abstract class AbstractSimpleTranslogTests extends ElasticsearchTestCase
assertTrue(Files.exists(path.resolve("translog-1")));
translog.add(new Translog.Create("test", "1", new byte[]{1}));
Translog.Snapshot snapshot = translog.snapshot();
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot, TranslogSizeMatcher.translogSize(1));
assertThat(snapshot.estimatedTotalOperations(), equalTo(1));
if (randomBoolean()) {
translog.close();