diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index b4c3daee08f..2a9ee444941 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -50,14 +50,12 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.EngineClosedException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.seqno.SequenceNumbersService; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.IndexShardClosedException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesService; @@ -391,8 +389,6 @@ public class TransportShardBulkAction extends TransportWriteAction indexSettings.getFlushThresholdSize().getBytes(); - } catch (AlreadyClosedException | EngineClosedException ex) { + } catch (AlreadyClosedException ex) { // that's fine we are already close - no need to flush } } @@ -1304,7 +1303,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl public void activateThrottling() { try { getEngine().activateThrottling(); - } catch (EngineClosedException ex) { + } catch (AlreadyClosedException ex) { // ignore } } @@ -1312,13 +1311,13 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl public void deactivateThrottling() { try { getEngine().deactivateThrottling(); - } catch (EngineClosedException ex) { + } catch (AlreadyClosedException ex) { // ignore } } private void handleRefreshException(Exception e) { - if (e instanceof EngineClosedException) { + if (e instanceof AlreadyClosedException) { // ignore } else if (e instanceof RefreshFailedEngineException) { RefreshFailedEngineException rfee = (RefreshFailedEngineException) e; @@ -1530,7 +1529,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl Engine getEngine() { Engine engine = getEngineOrNull(); if (engine == null) { - throw new EngineClosedException(shardId); + throw new AlreadyClosedException("engine is closed"); } return engine; } @@ -1667,7 +1666,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl private Engine createNewEngine(EngineConfig config) { synchronized (mutex) { if (state == IndexShardState.CLOSED) { - throw new EngineClosedException(shardId); + throw new AlreadyClosedException(shardId + " can't create engine - shard is closed"); } assert this.currentEngineReference.get() == null; Engine engine = newEngine(config); @@ -1769,7 +1768,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl try { final Engine engine = getEngine(); engine.getTranslog().ensureSynced(candidates.stream().map(Tuple::v1)); - } catch (EngineClosedException ex) { + } catch (AlreadyClosedException ex) { // that's fine since we already synced everything on engine close - this also is conform with the methods // documentation } catch (IOException ex) { // if this fails we are in deep shit - fail the request @@ -1884,8 +1883,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * refresh listeners. * Otherwise false. * - * @throws EngineClosedException if the engine is already closed - * @throws AlreadyClosedException if the internal indexwriter in the engine is already closed + * @throws AlreadyClosedException if the engine or internal indexwriter in the engine is already closed */ public boolean isRefreshNeeded() { return getEngine().refreshNeeded() || (refreshListeners != null && refreshListeners.refreshNeeded()); diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java index d7c86a77a33..cbbd7b84213 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java +++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java @@ -21,6 +21,7 @@ package org.elasticsearch.indices; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; +import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -30,7 +31,6 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.EngineClosedException; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.IndexingOperationListener; @@ -384,7 +384,7 @@ public class IndexingMemoryController extends AbstractComponent implements Index protected void checkIdle(IndexShard shard, long inactiveTimeNS) { try { shard.checkIdle(inactiveTimeNS); - } catch (EngineClosedException e) { + } catch (AlreadyClosedException e) { logger.trace((Supplier) () -> new ParameterizedMessage("ignore exception while checking if shard {} is inactive", shard.shardId()), e); } } diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index b929681032e..8e5950fe9f9 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.support.replication; +import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.UnavailableShardsException; @@ -55,7 +56,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.engine.EngineClosedException; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardClosedException; import org.elasticsearch.index.shard.IndexShardState; @@ -431,12 +431,12 @@ public class TransportReplicationActionTests extends ESTestCase { } } - private ElasticsearchException randomRetryPrimaryException(ShardId shardId) { + private Exception randomRetryPrimaryException(ShardId shardId) { return randomFrom( new ShardNotFoundException(shardId), new IndexNotFoundException(shardId.getIndex()), new IndexShardClosedException(shardId), - new EngineClosedException(shardId), + new AlreadyClosedException(shardId + " primary is closed"), new ReplicationOperation.RetryOnPrimaryException(shardId, "hello") ); } diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 832352b0278..0958bbc6055 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -48,7 +48,9 @@ import org.elasticsearch.index.cache.query.IndexQueryCache; import org.elasticsearch.index.cache.query.QueryCache; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineException; +import org.elasticsearch.index.engine.InternalEngineTests; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexSearcherWrapper; import org.elasticsearch.index.shard.IndexingOperationListener; @@ -247,7 +249,8 @@ public class IndexModuleTests extends ESTestCase { assertEquals(IndexingSlowLog.class, indexService.getIndexOperationListeners().get(0).getClass()); assertSame(listener, indexService.getIndexOperationListeners().get(1)); - Engine.Index index = new Engine.Index(new Term("_uid", "1"), null); + ParsedDocument doc = InternalEngineTests.createParsedDoc("1", "test", null); + Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc); ShardId shardId = new ShardId(new Index("foo", "bar"), 0); for (IndexingOperationListener l : indexService.getIndexOperationListeners()) { l.preIndex(shardId, index); diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java index 2f74194e256..e713f14be14 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java @@ -19,7 +19,6 @@ package org.elasticsearch.index; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.DocWriteResponse; @@ -34,7 +33,6 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.common.Priority; @@ -58,9 +56,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.ConnectionProfile; -import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; @@ -91,7 +86,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; /** * Tests for indices that use shadow replicas and a shared filesystem */ -@LuceneTestCase.AwaitsFix(bugUrl = "fix this fails intermittently") @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) public class IndexWithShadowReplicasIT extends ESIntegTestCase { @@ -459,7 +453,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { assertHitCount(resp, numPhase1Docs + numPhase2Docs); } - @AwaitsFix(bugUrl = "uncaught exception") public void testPrimaryRelocationWhereRecoveryFails() throws Exception { Path dataPath = createTempDir(); Settings nodeSettings = Settings.builder() diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 6f85d65bc91..ef05d8f27ca 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -106,9 +106,9 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.RootObjectMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.UidFieldMapper; import org.elasticsearch.index.seqno.SequenceNumbersService; -import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.IndexSearcherWrapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; @@ -145,6 +145,7 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; @@ -157,7 +158,6 @@ import java.util.function.LongSupplier; import java.util.function.Supplier; import static java.util.Collections.emptyMap; -import static java.util.Collections.max; import static org.elasticsearch.index.engine.Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY; import static org.elasticsearch.index.engine.Engine.Operation.Origin.PEER_RECOVERY; import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY; @@ -261,19 +261,22 @@ public class InternalEngineTests extends ESTestCase { } - private Document testDocumentWithTextField() { + private static Document testDocumentWithTextField() { Document document = testDocument(); document.add(new TextField("value", "test", Field.Store.YES)); return document; } - private Document testDocument() { + private static Document testDocument() { return new Document(); } + public static ParsedDocument createParsedDoc(String id, String type, String routing) { + return testParsedDocument(id, type, routing, testDocumentWithTextField(), new BytesArray("{ \"value\" : \"test\" }"), null); + } - private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, Document document, BytesReference source, Mapping mappingUpdate) { - Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE); + private static ParsedDocument testParsedDocument(String id, String type, String routing, Document document, BytesReference source, Mapping mappingUpdate) { + Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", 0); SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID(); document.add(uidField); @@ -401,11 +404,11 @@ public class InternalEngineTests extends ESTestCase { assertThat(engine.segmentsStats(false).getMemoryInBytes(), equalTo(0L)); // create two docs and refresh - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - Engine.Index first = new Engine.Index(newUid("1"), doc); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + Engine.Index first = indexForDoc(doc); Engine.IndexResult firstResult = engine.index(first); - ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_2, null); - Engine.Index second = new Engine.Index(newUid("2"), doc2); + ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_2, null); + Engine.Index second = indexForDoc(doc2); Engine.IndexResult secondResult = engine.index(second); assertThat(secondResult.getTranslogLocation(), greaterThan(firstResult.getTranslogLocation())); engine.refresh("test"); @@ -437,8 +440,8 @@ public class InternalEngineTests extends ESTestCase { assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertThat(segments.get(0).isCompound(), equalTo(true)); - ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, testDocumentWithTextField(), B_3, null); - engine.index(new Engine.Index(newUid("3"), doc3)); + ParsedDocument doc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_3, null); + engine.index(indexForDoc(doc3)); engine.refresh("test"); segments = engine.segments(false); @@ -464,7 +467,7 @@ public class InternalEngineTests extends ESTestCase { assertThat(segments.get(1).isCompound(), equalTo(true)); - engine.delete(new Engine.Delete("test", "1", newUid("1"))); + engine.delete(new Engine.Delete("test", "1", newUid(doc))); engine.refresh("test"); segments = engine.segments(false); @@ -484,8 +487,8 @@ public class InternalEngineTests extends ESTestCase { assertThat(segments.get(1).isCompound(), equalTo(true)); engine.onSettingsChanged(); - ParsedDocument doc4 = testParsedDocument("4", "4", "test", null, testDocumentWithTextField(), B_3, null); - engine.index(new Engine.Index(newUid("4"), doc4)); + ParsedDocument doc4 = testParsedDocument("4", "test", null, testDocumentWithTextField(), B_3, null); + engine.index(indexForDoc(doc4)); engine.refresh("test"); segments = engine.segments(false); @@ -518,19 +521,19 @@ public class InternalEngineTests extends ESTestCase { List segments = engine.segments(true); assertThat(segments.isEmpty(), equalTo(true)); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); engine.refresh("test"); segments = engine.segments(true); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).ramTree, notNullValue()); - ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_2, null); - engine.index(new Engine.Index(newUid("2"), doc2)); + ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_2, null); + engine.index(indexForDoc(doc2)); engine.refresh("test"); - ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, testDocumentWithTextField(), B_3, null); - engine.index(new Engine.Index(newUid("3"), doc3)); + ParsedDocument doc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_3, null); + engine.index(indexForDoc(doc3)); engine.refresh("test"); segments = engine.segments(true); @@ -544,12 +547,12 @@ public class InternalEngineTests extends ESTestCase { public void testSegmentsWithMergeFlag() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); engine.index(index); engine.flush(); assertThat(engine.segments(false).size(), equalTo(1)); - index = new Engine.Index(newUid("2"), doc); + index = indexForDoc(testParsedDocument("2", "test", null, testDocument(), B_1, null)); engine.index(index); engine.flush(); List segments = engine.segments(false); @@ -557,7 +560,7 @@ public class InternalEngineTests extends ESTestCase { for (Segment segment : segments) { assertThat(segment.getMergeId(), nullValue()); } - index = new Engine.Index(newUid("3"), doc); + index = indexForDoc(testParsedDocument("3", "test", null, testDocument(), B_1, null)); engine.index(index); engine.flush(); segments = engine.segments(false); @@ -566,7 +569,7 @@ public class InternalEngineTests extends ESTestCase { assertThat(segment.getMergeId(), nullValue()); } - index = new Engine.Index(newUid("4"), doc); + index = indexForDoc(doc); engine.index(index); engine.flush(); final long gen1 = store.readLastCommittedSegmentsInfo().getGeneration(); @@ -598,8 +601,8 @@ public class InternalEngineTests extends ESTestCase { Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { assertThat(engine.segmentsStats(true).getFileSizes().size(), equalTo(0)); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); engine.refresh("test"); SegmentsStats stats = engine.segmentsStats(true); @@ -608,8 +611,8 @@ public class InternalEngineTests extends ESTestCase { ObjectObjectCursor firstEntry = stats.getFileSizes().iterator().next(); - ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_2, null); - engine.index(new Engine.Index(newUid("2"), doc2)); + ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_2, null); + engine.index(indexForDoc(doc2)); engine.refresh("test"); assertThat(engine.segmentsStats(true).getFileSizes().get(firstEntry.key), greaterThan(firstEntry.value)); @@ -709,8 +712,8 @@ public class InternalEngineTests extends ESTestCase { public void testFlushIsDisabledDuringTranslogRecovery() throws IOException { assertFalse(engine.isRecovering()); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); engine.close(); engine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG)); @@ -718,8 +721,8 @@ public class InternalEngineTests extends ESTestCase { assertTrue(engine.isRecovering()); engine.recoverFromTranslog(); assertFalse(engine.isRecovering()); - doc = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("2"), doc)); + doc = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); engine.flush(); } @@ -730,13 +733,13 @@ public class InternalEngineTests extends ESTestCase { try { initialEngine = engine; for (int i = 0; i < ops; i++) { - final ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), SOURCE, null); + final ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), SOURCE, null); if (randomBoolean()) { - final Engine.Index operation = new Engine.Index(newUid("test#1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false); + final Engine.Index operation = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false); operations.add(operation); initialEngine.index(operation); } else { - final Engine.Delete operation = new Engine.Delete("test", "1", newUid("test#1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime()); + final Engine.Delete operation = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime()); operations.add(operation); initialEngine.delete(operation); } @@ -766,8 +769,8 @@ public class InternalEngineTests extends ESTestCase { initialEngine = engine; for (int i = 0; i < docs; i++) { final String id = Integer.toString(i); - final ParsedDocument doc = testParsedDocument(id, id, "test", null, testDocumentWithTextField(), SOURCE, null); - initialEngine.index(new Engine.Index(newUid(id), doc)); + final ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null); + initialEngine.index(indexForDoc(doc)); } } finally { IOUtils.close(initialEngine); @@ -795,33 +798,30 @@ public class InternalEngineTests extends ESTestCase { } public void testConcurrentGetAndFlush() throws Exception { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); final AtomicReference latestGetResult = new AtomicReference<>(); - latestGetResult.set(engine.get(new Engine.Get(true, newUid("1")))); + latestGetResult.set(engine.get(new Engine.Get(true, newUid(doc)))); final AtomicBoolean flushFinished = new AtomicBoolean(false); final CyclicBarrier barrier = new CyclicBarrier(2); - Thread getThread = new Thread() { - @Override - public void run() { - try { - barrier.await(); - } catch (InterruptedException | BrokenBarrierException e) { - throw new RuntimeException(e); + Thread getThread = new Thread(() -> { + try { + barrier.await(); + } catch (InterruptedException | BrokenBarrierException e) { + throw new RuntimeException(e); + } + while (flushFinished.get() == false) { + Engine.GetResult previousGetResult = latestGetResult.get(); + if (previousGetResult != null) { + previousGetResult.release(); } - while (flushFinished.get() == false) { - Engine.GetResult previousGetResult = latestGetResult.get(); - if (previousGetResult != null) { - previousGetResult.release(); - } - latestGetResult.set(engine.get(new Engine.Get(true, newUid("1")))); - if (latestGetResult.get().exists() == false) { - break; - } + latestGetResult.set(engine.get(new Engine.Get(true, newUid(doc)))); + if (latestGetResult.get().exists() == false) { + break; } } - }; + }); getThread.start(); barrier.await(); engine.flush(); @@ -839,8 +839,8 @@ public class InternalEngineTests extends ESTestCase { // create a document Document document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null); + engine.index(indexForDoc(doc)); // its not there... searchResult = engine.acquireSearcher("test"); @@ -849,12 +849,12 @@ public class InternalEngineTests extends ESTestCase { searchResult.close(); // but, not there non realtime - Engine.GetResult getResult = engine.get(new Engine.Get(false, newUid("1"))); + Engine.GetResult getResult = engine.get(new Engine.Get(false, newUid(doc))); assertThat(getResult.exists(), equalTo(false)); getResult.release(); // but, we can still get it (in realtime) - getResult = engine.get(new Engine.Get(true, newUid("1"))); + getResult = engine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); @@ -869,7 +869,7 @@ public class InternalEngineTests extends ESTestCase { searchResult.close(); // also in non realtime - getResult = engine.get(new Engine.Get(false, newUid("1"))); + getResult = engine.get(new Engine.Get(false, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); @@ -878,8 +878,8 @@ public class InternalEngineTests extends ESTestCase { document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE)); - doc = testParsedDocument("1", "1", "test", null, document, B_2, null); - engine.index(new Engine.Index(newUid("1"), doc)); + doc = testParsedDocument("1", "test", null, document, B_2, null); + engine.index(indexForDoc(doc)); // its not updated yet... searchResult = engine.acquireSearcher("test"); @@ -889,7 +889,7 @@ public class InternalEngineTests extends ESTestCase { searchResult.close(); // but, we can still get it (in realtime) - getResult = engine.get(new Engine.Get(true, newUid("1"))); + getResult = engine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); @@ -904,7 +904,7 @@ public class InternalEngineTests extends ESTestCase { searchResult.close(); // now delete - engine.delete(new Engine.Delete("test", "1", newUid("1"))); + engine.delete(new Engine.Delete("test", "1", newUid(doc))); // its not deleted yet searchResult = engine.acquireSearcher("test"); @@ -914,7 +914,7 @@ public class InternalEngineTests extends ESTestCase { searchResult.close(); // but, get should not see it (in realtime) - getResult = engine.get(new Engine.Get(true, newUid("1"))); + getResult = engine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(false)); getResult.release(); @@ -930,8 +930,8 @@ public class InternalEngineTests extends ESTestCase { // add it back document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - engine.index(new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED)); + doc = testParsedDocument("1", "test", null, document, B_1, null); + engine.index(new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED)); // its not there... searchResult = engine.acquireSearcher("test"); @@ -954,7 +954,7 @@ public class InternalEngineTests extends ESTestCase { engine.flush(); // and, verify get (in real time) - getResult = engine.get(new Engine.Get(true, newUid("1"))); + getResult = engine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); @@ -963,8 +963,8 @@ public class InternalEngineTests extends ESTestCase { // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); - doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + doc = testParsedDocument("1", "test", null, document, B_1, null); + engine.index(indexForDoc(doc)); // its not updated yet... searchResult = engine.acquireSearcher("test"); @@ -989,8 +989,8 @@ public class InternalEngineTests extends ESTestCase { searchResult.close(); // create a document - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); // its not there... searchResult = engine.acquireSearcher("test"); @@ -1008,7 +1008,7 @@ public class InternalEngineTests extends ESTestCase { // don't release the search result yet... // delete, refresh and do a new search, it should not be there - engine.delete(new Engine.Delete("test", "1", newUid("1"))); + engine.delete(new Engine.Delete("test", "1", newUid(doc))); engine.refresh("test"); Engine.Searcher updateSearchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(updateSearchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); @@ -1025,8 +1025,8 @@ public class InternalEngineTests extends ESTestCase { Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new LogByteSizeMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) { final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); Engine.CommitId commitID = engine.flush(); assertThat(commitID, equalTo(new Engine.CommitId(store.readLastCommittedSegmentsInfo().getId()))); byte[] wrongBytes = Base64.getDecoder().decode(commitID.toString()); @@ -1034,7 +1034,7 @@ public class InternalEngineTests extends ESTestCase { Engine.CommitId wrongId = new Engine.CommitId(wrongBytes); assertEquals("should fail to sync flush with wrong id (but no docs)", engine.syncFlush(syncId + "1", wrongId), Engine.SyncedFlushResult.COMMIT_MISMATCH); - engine.index(new Engine.Index(newUid("2"), doc)); + engine.index(indexForDoc(doc)); assertEquals("should fail to sync flush with right id but pending doc", engine.syncFlush(syncId + "2", commitID), Engine.SyncedFlushResult.PENDING_OPERATIONS); commitID = engine.flush(); @@ -1052,20 +1052,20 @@ public class InternalEngineTests extends ESTestCase { InternalEngine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new LogDocMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) { final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - Engine.Index doc1 = new Engine.Index(newUid("1"), doc); + Engine.Index doc1 = indexForDoc(testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null)); engine.index(doc1); assertEquals(engine.getLastWriteNanos(), doc1.startTime()); engine.flush(); - Engine.Index doc2 = new Engine.Index(newUid("2"), doc); + Engine.Index doc2 = indexForDoc(testParsedDocument("2", "test", null, testDocumentWithTextField(), B_1, null)); engine.index(doc2); assertEquals(engine.getLastWriteNanos(), doc2.startTime()); engine.flush(); final boolean forceMergeFlushes = randomBoolean(); + final ParsedDocument parsedDoc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_1, null); if (forceMergeFlushes) { - engine.index(new Engine.Index(newUid("3"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime() - engine.engineConfig.getFlushMergesAfter().nanos(), -1, false)); + engine.index(new Engine.Index(newUid(parsedDoc3), parsedDoc3, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime() - engine.engineConfig.getFlushMergesAfter().nanos(), -1, false)); } else { - engine.index(new Engine.Index(newUid("3"), doc)); + engine.index(indexForDoc(parsedDoc3)); } Engine.CommitId commitID = engine.flush(); assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID), @@ -1087,7 +1087,7 @@ public class InternalEngineTests extends ESTestCase { assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); if (randomBoolean()) { - Engine.Index doc4 = new Engine.Index(newUid("4"), doc); + Engine.Index doc4 = indexForDoc(testParsedDocument("4", "test", null, testDocumentWithTextField(), B_1, null)); engine.index(doc4); assertEquals(engine.getLastWriteNanos(), doc4.startTime()); } else { @@ -1105,8 +1105,8 @@ public class InternalEngineTests extends ESTestCase { public void testSyncedFlushSurvivesEngineRestart() throws IOException { final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); final Engine.CommitId commitID = engine.flush(); assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID), Engine.SyncedFlushResult.SUCCESS); @@ -1128,15 +1128,15 @@ public class InternalEngineTests extends ESTestCase { public void testSyncedFlushVanishesOnReplay() throws IOException { final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); final Engine.CommitId commitID = engine.flush(); assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID), Engine.SyncedFlushResult.SUCCESS); assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); - doc = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), new BytesArray("{}"), null); - engine.index(new Engine.Index(newUid("2"), doc)); + doc = testParsedDocument("2", "test", null, testDocumentWithTextField(), new BytesArray("{}"), null); + engine.index(indexForDoc(doc)); EngineConfig config = engine.config(); engine.close(); engine = new InternalEngine(copy(config, EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG)); @@ -1145,79 +1145,79 @@ public class InternalEngineTests extends ESTestCase { } public void testVersioningNewCreate() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index create = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED); Engine.IndexResult indexResult = engine.index(create); assertThat(indexResult.getVersion(), equalTo(1L)); - create = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), create.primaryTerm(), indexResult.getVersion(), create.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false); + create = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), create.primaryTerm(), indexResult.getVersion(), create.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false); indexResult = replicaEngine.index(create); assertThat(indexResult.getVersion(), equalTo(1L)); } public void testVersioningNewIndex() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); Engine.IndexResult indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(1L)); - index = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false); indexResult = replicaEngine.index(index); assertThat(indexResult.getVersion(), equalTo(1L)); } public void testExternalVersioningNewIndex() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false); Engine.IndexResult indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(12L)); - index = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false); indexResult = replicaEngine.index(index); assertThat(indexResult.getVersion(), equalTo(12L)); } public void testVersioningIndexConflict() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); Engine.IndexResult indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(1L)); - index = new Engine.Index(newUid("1"), doc); + index = indexForDoc(doc); indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(2L)); - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, 0, -1, false); indexResult = engine.index(index); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); // future versions should not work as well - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(index); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); } public void testExternalVersioningIndexConflict() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false); Engine.IndexResult indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(12L)); - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 14, VersionType.EXTERNAL, PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 14, VersionType.EXTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(14L)); - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 13, VersionType.EXTERNAL, PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 13, VersionType.EXTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(index); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); } public void testForceVersioningNotAllowedExceptForOlderIndices() throws Exception { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 42, VersionType.FORCE, PRIMARY, 0, -1, false); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 42, VersionType.FORCE, PRIMARY, 0, -1, false); Engine.IndexResult indexResult = engine.index(index); assertTrue(indexResult.hasFailure()); @@ -1229,13 +1229,13 @@ public class InternalEngineTests extends ESTestCase { .build()); try (Store store = createStore(); Engine engine = createEngine(oldIndexSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 84, VersionType.FORCE, PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 84, VersionType.FORCE, PRIMARY, 0, -1, false); Engine.IndexResult result = engine.index(index); assertTrue(result.hasFailure()); assertThat(result.getFailure(), instanceOf(IllegalArgumentException.class)); assertThat(result.getFailure().getMessage(), containsString("version type [FORCE] may not be used for non-translog operations")); - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 84, VersionType.FORCE, + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 84, VersionType.FORCE, Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY, 0, -1, false); result = engine.index(index); assertThat(result.getVersion(), equalTo(84L)); @@ -1243,42 +1243,42 @@ public class InternalEngineTests extends ESTestCase { } public void testVersioningIndexConflictWithFlush() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); Engine.IndexResult indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(1L)); - index = new Engine.Index(newUid("1"), doc); + index = indexForDoc(doc); indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(2L)); engine.flush(); - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(index); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); // future versions should not work as well - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(index); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); } public void testExternalVersioningIndexConflictWithFlush() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false); Engine.IndexResult indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(12L)); - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 14, VersionType.EXTERNAL, PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 14, VersionType.EXTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(14L)); engine.flush(); - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 13, VersionType.EXTERNAL, PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 13, VersionType.EXTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(index); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); @@ -1290,8 +1290,8 @@ public class InternalEngineTests extends ESTestCase { new LogByteSizeMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) { // use log MP here we test some behavior in ESMP int numDocs = randomIntBetween(10, 100); for (int i = 0; i < numDocs; i++) { - ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid(Integer.toString(i)), doc); + ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); engine.index(index); engine.refresh("test"); } @@ -1301,8 +1301,8 @@ public class InternalEngineTests extends ESTestCase { engine.forceMerge(true, 1, false, false, false); assertEquals(engine.segments(true).size(), 1); - ParsedDocument doc = testParsedDocument(Integer.toString(0), Integer.toString(0), "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid(Integer.toString(0)), doc); + ParsedDocument doc = testParsedDocument(Integer.toString(0), "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); engine.delete(new Engine.Delete(index.type(), index.id(), index.uid())); engine.forceMerge(true, 10, true, false, false); //expunge deletes @@ -1312,8 +1312,8 @@ public class InternalEngineTests extends ESTestCase { assertEquals(engine.config().getMergePolicy().toString(), numDocs - 1, test.reader().maxDoc()); } - doc = testParsedDocument(Integer.toString(1), Integer.toString(1), "test", null, testDocument(), B_1, null); - index = new Engine.Index(newUid(Integer.toString(1)), doc); + doc = testParsedDocument(Integer.toString(1), "test", null, testDocument(), B_1, null); + index = indexForDoc(doc); engine.delete(new Engine.Delete(index.type(), index.id(), index.uid())); engine.forceMerge(true, 10, false, false, false); //expunge deletes @@ -1347,8 +1347,8 @@ public class InternalEngineTests extends ESTestCase { int numDocs = randomIntBetween(1, 20); for (int j = 0; j < numDocs; j++) { i++; - ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid(Integer.toString(i)), doc); + ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); engine.index(index); } engine.refresh("test"); @@ -1359,7 +1359,7 @@ public class InternalEngineTests extends ESTestCase { return; } } - } catch (AlreadyClosedException | EngineClosedException ex) { + } catch (AlreadyClosedException ex) { // fine } } @@ -1380,57 +1380,57 @@ public class InternalEngineTests extends ESTestCase { } public void testVersioningDeleteConflict() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); Engine.IndexResult indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(1L)); - index = new Engine.Index(newUid("1"), doc); + index = indexForDoc(doc); indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(2L)); - Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0); + Engine.Delete delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0); Engine.DeleteResult result = engine.delete(delete); assertTrue(result.hasFailure()); assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class)); // future versions should not work as well - delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0); + delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0); result = engine.delete(delete); assertTrue(result.hasFailure()); assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class)); // now actually delete - delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0); + delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0); result = engine.delete(delete); assertThat(result.getVersion(), equalTo(3L)); // now check if we can index to a delete doc with version - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(index); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); } public void testVersioningDeleteConflictWithFlush() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); Engine.IndexResult indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(1L)); - index = new Engine.Index(newUid("1"), doc); + index = indexForDoc(doc); indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(2L)); engine.flush(); - Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0); + Engine.Delete delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0); Engine.DeleteResult deleteResult = engine.delete(delete); assertTrue(deleteResult.hasFailure()); assertThat(deleteResult.getFailure(), instanceOf(VersionConflictEngineException.class)); // future versions should not work as well - delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0); + delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0); deleteResult = engine.delete(delete); assertTrue(deleteResult.hasFailure()); assertThat(deleteResult.getFailure(), instanceOf(VersionConflictEngineException.class)); @@ -1438,58 +1438,58 @@ public class InternalEngineTests extends ESTestCase { engine.flush(); // now actually delete - delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0); + delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0); deleteResult = engine.delete(delete); assertThat(deleteResult.getVersion(), equalTo(3L)); engine.flush(); // now check if we can index to a delete doc with version - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0, -1, false); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(index); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); } public void testVersioningCreateExistsException() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index create = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index create = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false); Engine.IndexResult indexResult = engine.index(create); assertThat(indexResult.getVersion(), equalTo(1L)); - create = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false); + create = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(create); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); } public void testVersioningCreateExistsExceptionWithFlush() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index create = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index create = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false); Engine.IndexResult indexResult = engine.index(create); assertThat(indexResult.getVersion(), equalTo(1L)); engine.flush(); - create = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false); + create = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false); indexResult = engine.index(create); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); } public void testVersioningReplicaConflict1() { - final ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - final Engine.Index v1Index = new Engine.Index(newUid("1"), doc); + final ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + final Engine.Index v1Index = indexForDoc(doc); final Engine.IndexResult v1Result = engine.index(v1Index); assertThat(v1Result.getVersion(), equalTo(1L)); - final Engine.Index v2Index = new Engine.Index(newUid("1"), doc); + final Engine.Index v2Index = indexForDoc(doc); final Engine.IndexResult v2Result = engine.index(v2Index); assertThat(v2Result.getVersion(), equalTo(2L)); // apply the second index to the replica, should work fine final Engine.Index replicaV2Index = new Engine.Index( - newUid("1"), + newUid(doc), doc, v2Result.getSeqNo(), v2Index.primaryTerm(), @@ -1504,7 +1504,7 @@ public class InternalEngineTests extends ESTestCase { // now, the old one should produce an indexing result final Engine.Index replicaV1Index = new Engine.Index( - newUid("1"), + newUid(doc), doc, v1Result.getSeqNo(), v1Index.primaryTerm(), @@ -1527,14 +1527,14 @@ public class InternalEngineTests extends ESTestCase { } public void testVersioningReplicaConflict2() { - final ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - final Engine.Index v1Index = new Engine.Index(newUid("1"), doc); + final ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + final Engine.Index v1Index = indexForDoc(doc); final Engine.IndexResult v1Result = engine.index(v1Index); assertThat(v1Result.getVersion(), equalTo(1L)); // apply the first index to the replica, should work fine final Engine.Index replicaV1Index = new Engine.Index( - newUid("1"), + newUid(doc), doc, v1Result.getSeqNo(), v1Index.primaryTerm(), @@ -1548,12 +1548,12 @@ public class InternalEngineTests extends ESTestCase { assertThat(replicaV1Result.getVersion(), equalTo(1L)); // index it again - final Engine.Index v2Index = new Engine.Index(newUid("1"), doc); + final Engine.Index v2Index = indexForDoc(doc); final Engine.IndexResult v2Result = engine.index(v2Index); assertThat(v2Result.getVersion(), equalTo(2L)); // now delete it - final Engine.Delete delete = new Engine.Delete("test", "1", newUid("1")); + final Engine.Delete delete = new Engine.Delete("test", "1", newUid(doc)); final Engine.DeleteResult deleteResult = engine.delete(delete); assertThat(deleteResult.getVersion(), equalTo(3L)); @@ -1561,7 +1561,7 @@ public class InternalEngineTests extends ESTestCase { final Engine.Delete replicaDelete = new Engine.Delete( "test", "1", - newUid("1"), + newUid(doc), deleteResult.getSeqNo(), delete.primaryTerm(), deleteResult.getVersion(), @@ -1579,7 +1579,7 @@ public class InternalEngineTests extends ESTestCase { // now do the second index on the replica, it should result in the current version final Engine.Index replicaV2Index = new Engine.Index( - newUid("1"), + newUid(doc), doc, v2Result.getSeqNo(), v2Index.primaryTerm(), @@ -1596,33 +1596,33 @@ public class InternalEngineTests extends ESTestCase { } public void testBasicCreatedFlag() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); Engine.IndexResult indexResult = engine.index(index); assertTrue(indexResult.isCreated()); - index = new Engine.Index(newUid("1"), doc); + index = indexForDoc(doc); indexResult = engine.index(index); assertFalse(indexResult.isCreated()); - engine.delete(new Engine.Delete(null, "1", newUid("1"))); + engine.delete(new Engine.Delete(null, "1", newUid(doc))); - index = new Engine.Index(newUid("1"), doc); + index = indexForDoc(doc); indexResult = engine.index(index); assertTrue(indexResult.isCreated()); } public void testCreatedFlagAfterFlush() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null); - Engine.Index index = new Engine.Index(newUid("1"), doc); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); Engine.IndexResult indexResult = engine.index(index); assertTrue(indexResult.isCreated()); - engine.delete(new Engine.Delete(null, "1", newUid("1"))); + engine.delete(new Engine.Delete(null, "1", newUid(doc))); engine.flush(); - index = new Engine.Index(newUid("1"), doc); + index = indexForDoc(doc); indexResult = engine.index(index); assertTrue(indexResult.isCreated()); } @@ -1667,14 +1667,14 @@ public class InternalEngineTests extends ESTestCase { try { // First, with DEBUG, which should NOT log IndexWriter output: - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); engine.flush(); assertFalse(mockAppender.sawIndexWriterMessage); // Again, with TRACE, which should log IndexWriter output: Loggers.setLevel(rootLogger, Level.TRACE); - engine.index(new Engine.Index(newUid("2"), doc)); + engine.index(indexForDoc(doc)); engine.flush(); assertTrue(mockAppender.sawIndexWriterMessage); @@ -1723,8 +1723,8 @@ public class InternalEngineTests extends ESTestCase { } else { // index a document id = randomFrom(ids); - ParsedDocument doc = testParsedDocument("test#" + id, id, "test", null, testDocumentWithTextField(), SOURCE, null); - final Engine.Index index = new Engine.Index(newUid("test#" + id), doc, + ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null); + final Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, rarely() ? 100 : Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, 0, -1, false); @@ -1820,22 +1820,19 @@ public class InternalEngineTests extends ESTestCase { // create N indexing threads to index documents simultaneously for (int threadNum = 0; threadNum < numIndexingThreads; threadNum++) { final int threadIdx = threadNum; - Thread indexingThread = new Thread() { - @Override - public void run() { - try { - barrier.await(); // wait for all threads to start at the same time - // index random number of docs - for (int i = 0; i < numDocsPerThread; i++) { - final String id = "thread" + threadIdx + "#" + i; - ParsedDocument doc = testParsedDocument(id, id, "test", null, testDocument(), B_1, null); - engine.index(new Engine.Index(newUid(id), doc)); - } - } catch (Exception e) { - throw new RuntimeException(e); + Thread indexingThread = new Thread(() -> { + try { + barrier.await(); // wait for all threads to start at the same time + // index random number of docs + for (int i = 0; i < numDocsPerThread; i++) { + final String id = "thread" + threadIdx + "#" + i; + ParsedDocument doc = testParsedDocument(id, "test", null, testDocument(), B_1, null); + engine.index(indexForDoc(doc)); } + } catch (Exception e) { + throw new RuntimeException(e); } - }; + }); indexingThreads.add(indexingThread); } @@ -1930,15 +1927,15 @@ public class InternalEngineTests extends ESTestCase { try { // First, with DEBUG, which should NOT log IndexWriter output: - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + engine.index(indexForDoc(doc)); engine.flush(); assertFalse(mockAppender.sawIndexWriterMessage); assertFalse(mockAppender.sawIndexWriterIFDMessage); // Again, with TRACE, which should only log IndexWriter IFD output: Loggers.setLevel(iwIFDLogger, Level.TRACE); - engine.index(new Engine.Index(newUid("2"), doc)); + engine.index(indexForDoc(doc)); engine.flush(); assertFalse(mockAppender.sawIndexWriterMessage); assertTrue(mockAppender.sawIndexWriterIFDMessage); @@ -1959,14 +1956,14 @@ public class InternalEngineTests extends ESTestCase { Document document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_2, null); - engine.index(new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false)); + ParsedDocument doc = testParsedDocument("1", "test", null, document, B_2, null); + engine.index(new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false)); // Delete document we just added: - engine.delete(new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime())); + engine.delete(new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime())); // Get should not find the document - Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1"))); + Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(false)); // Give the gc pruning logic a chance to kick in @@ -1984,23 +1981,23 @@ public class InternalEngineTests extends ESTestCase { assertThat(getResult.exists(), equalTo(false)); // Try to index uid=1 with a too-old version, should fail: - Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false); + Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false); Engine.IndexResult indexResult = engine.index(index); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); // Get should still not find the document - getResult = engine.get(new Engine.Get(true, newUid("1"))); + getResult = engine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(false)); // Try to index uid=2 with a too-old version, should fail: - Engine.Index index1 = new Engine.Index(newUid("2"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false); + Engine.Index index1 = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false); indexResult = engine.index(index1); assertTrue(indexResult.hasFailure()); assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class)); // Get should not find the document - getResult = engine.get(new Engine.Get(true, newUid("2"))); + getResult = engine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(false)); } } @@ -2009,6 +2006,14 @@ public class InternalEngineTests extends ESTestCase { return new Term("_uid", id); } + protected Term newUid(ParsedDocument doc) { + return new Term("_uid", doc.uid()); + } + + private Engine.Index indexForDoc(ParsedDocument doc) { + return new Engine.Index(newUid(doc), doc); + } + public void testExtractShardId() { try (Engine.Searcher test = this.engine.acquireSearcher("test")) { ShardId shardId = ShardUtils.extractShardId(test.getDirectoryReader()); @@ -2091,8 +2096,8 @@ public class InternalEngineTests extends ESTestCase { public void testTranslogReplayWithFailure() throws IOException { final int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { - ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null); - Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); + ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null); + Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); Engine.IndexResult indexResult = engine.index(firstIndexRequest); assertThat(indexResult.getVersion(), equalTo(1L)); } @@ -2141,8 +2146,8 @@ public class InternalEngineTests extends ESTestCase { public void testSkipTranslogReplay() throws IOException { final int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { - ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null); - Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); + ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null); + Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); Engine.IndexResult indexResult = engine.index(firstIndexRequest); assertThat(indexResult.getVersion(), equalTo(1L)); } @@ -2234,8 +2239,8 @@ public class InternalEngineTests extends ESTestCase { } final int numExtraDocs = randomIntBetween(1, 10); for (int i = 0; i < numExtraDocs; i++) { - ParsedDocument doc = testParsedDocument("extra" + Integer.toString(i), "extra" + Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null); - Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); + ParsedDocument doc = testParsedDocument("extra" + Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null); + Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); Engine.IndexResult indexResult = engine.index(firstIndexRequest); assertThat(indexResult.getVersion(), equalTo(1L)); } @@ -2263,8 +2268,8 @@ public class InternalEngineTests extends ESTestCase { public void testTranslogReplay() throws IOException { final int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { - ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null); - Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); + ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null); + Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); Engine.IndexResult indexResult = engine.index(firstIndexRequest); assertThat(indexResult.getVersion(), equalTo(1L)); } @@ -2305,17 +2310,16 @@ public class InternalEngineTests extends ESTestCase { final boolean flush = randomBoolean(); int randomId = randomIntBetween(numDocs + 1, numDocs + 10); - String uuidValue = "test#" + Integer.toString(randomId); - ParsedDocument doc = testParsedDocument(uuidValue, Integer.toString(randomId), "test", null, testDocument(), new BytesArray("{}"), null); - Engine.Index firstIndexRequest = new Engine.Index(newUid(uuidValue), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false); + ParsedDocument doc = testParsedDocument(Integer.toString(randomId), "test", null, testDocument(), new BytesArray("{}"), null); + Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false); Engine.IndexResult indexResult = engine.index(firstIndexRequest); assertThat(indexResult.getVersion(), equalTo(1L)); if (flush) { engine.flush(); } - doc = testParsedDocument(uuidValue, Integer.toString(randomId), "test", null, testDocument(), new BytesArray("{}"), null); - Engine.Index idxRequest = new Engine.Index(newUid(uuidValue), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false); + doc = testParsedDocument(Integer.toString(randomId), "test", null, testDocument(), new BytesArray("{}"), null); + Engine.Index idxRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false); Engine.IndexResult result = engine.index(idxRequest); engine.refresh("test"); assertThat(result.getVersion(), equalTo(2L)); @@ -2332,7 +2336,7 @@ public class InternalEngineTests extends ESTestCase { } parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer(); assertEquals(flush ? 1 : 2, parser.recoveredOps.get()); - engine.delete(new Engine.Delete("test", Integer.toString(randomId), newUid(uuidValue))); + engine.delete(new Engine.Delete("test", Integer.toString(randomId), newUid(doc))); if (randomBoolean()) { engine.refresh("test"); } else { @@ -2381,8 +2385,8 @@ public class InternalEngineTests extends ESTestCase { public void testRecoverFromForeignTranslog() throws IOException { final int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { - ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null); - Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); + ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null); + Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); Engine.IndexResult index = engine.index(firstIndexRequest); assertThat(index.getVersion(), equalTo(1L)); } @@ -2469,8 +2473,8 @@ public class InternalEngineTests extends ESTestCase { // create { - ParsedDocument doc = testParsedDocument(Integer.toString(0), Integer.toString(0), "test", null, testDocument(), new BytesArray("{}"), null); - Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(0)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); + ParsedDocument doc = testParsedDocument(Integer.toString(0), "test", null, testDocument(), new BytesArray("{}"), null); + Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); try (InternalEngine engine = new InternalEngine(copy(config, EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG))){ assertFalse(engine.isRecovering()); @@ -2529,11 +2533,11 @@ public class InternalEngineTests extends ESTestCase { } public void testCheckDocumentFailure() throws Exception { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - Exception documentFailure = engine.checkIfDocumentFailureOrThrow(new Engine.Index(newUid("1"), doc), new IOException("simulated document failure")); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + Exception documentFailure = engine.checkIfDocumentFailureOrThrow(indexForDoc(doc), new IOException("simulated document failure")); assertThat(documentFailure, instanceOf(IOException.class)); try { - engine.checkIfDocumentFailureOrThrow(new Engine.Index(newUid("1"), doc), new CorruptIndexException("simulated environment failure", "")); + engine.checkIfDocumentFailureOrThrow(indexForDoc(doc), new CorruptIndexException("simulated environment failure", "")); fail("expected exception to be thrown"); } catch (Exception envirnomentException) { assertThat(envirnomentException.getMessage(), containsString("simulated environment failure")); @@ -2541,7 +2545,7 @@ public class InternalEngineTests extends ESTestCase { } private static class ThrowingIndexWriter extends IndexWriter { - private boolean throwDocumentFailure; + private AtomicReference failureToThrow = new AtomicReference<>(); public ThrowingIndexWriter(Directory d, IndexWriterConfig conf) throws IOException { super(d, conf); @@ -2549,54 +2553,109 @@ public class InternalEngineTests extends ESTestCase { @Override public long addDocument(Iterable doc) throws IOException { - if (throwDocumentFailure) { - throw new IOException("simulated"); + maybeThrowFailure(); + return super.addDocument(doc); + } + + private void maybeThrowFailure() throws IOException { + Exception failure = failureToThrow.get(); + if (failure instanceof RuntimeException) { + throw (RuntimeException)failure; + } else if (failure instanceof IOException) { + throw (IOException)failure; } else { - return super.addDocument(doc); + assert failure == null : "unsupported failure class: " + failure.getClass().getCanonicalName(); } } @Override public long deleteDocuments(Term... terms) throws IOException { - if (throwDocumentFailure) { - throw new IOException("simulated"); - } else { - return super.deleteDocuments(terms); - } + maybeThrowFailure(); + return super.deleteDocuments(terms); } - public void setThrowDocumentFailure(boolean throwDocumentFailure) { - this.throwDocumentFailure = throwDocumentFailure; + public void setThrowFailure(IOException documentFailure) { + Objects.requireNonNull(documentFailure); + failureToThrow.set(documentFailure); + } + + public void setThrowFailure(RuntimeException runtimeFailure) { + Objects.requireNonNull(runtimeFailure); + failureToThrow.set(runtimeFailure); + } + + public void clearFailure() { + failureToThrow.set(null); } } public void testHandleDocumentFailure() throws Exception { try (Store store = createStore()) { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); + final ParsedDocument doc1 = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + final ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_1, null); + final ParsedDocument doc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_1, null); ThrowingIndexWriter throwingIndexWriter = new ThrowingIndexWriter(store.directory(), new IndexWriterConfig()); try (Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE, () -> throwingIndexWriter)) { // test document failure while indexing - throwingIndexWriter.setThrowDocumentFailure(true); - Engine.IndexResult indexResult = engine.index(randomAppendOnly(1, doc, false)); + if (randomBoolean()) { + throwingIndexWriter.setThrowFailure(new IOException("simulated")); + } else { + throwingIndexWriter.setThrowFailure(new IllegalArgumentException("simulated max token length")); + } + Engine.IndexResult indexResult = engine.index(indexForDoc(doc1)); assertNotNull(indexResult.getFailure()); - throwingIndexWriter.setThrowDocumentFailure(false); - indexResult = engine.index(randomAppendOnly(1, doc, false)); + throwingIndexWriter.clearFailure(); + indexResult = engine.index(indexForDoc(doc1)); assertNull(indexResult.getFailure()); + engine.index(indexForDoc(doc2)); // test document failure while deleting - throwingIndexWriter.setThrowDocumentFailure(true); - Engine.DeleteResult deleteResult = engine.delete(new Engine.Delete("test", "", newUid("1"))); + if (randomBoolean()) { + throwingIndexWriter.setThrowFailure(new IOException("simulated")); + } else { + throwingIndexWriter.setThrowFailure(new IllegalArgumentException("simulated max token length")); + } + Engine.DeleteResult deleteResult = engine.delete(new Engine.Delete("test", "1", newUid(doc1))); assertNotNull(deleteResult.getFailure()); + + // test non document level failure is thrown + if (randomBoolean()) { + // simulate close by corruption + throwingIndexWriter.setThrowFailure(new CorruptIndexException("simulated", "hello")); + try { + if (randomBoolean()) { + engine.index(indexForDoc(doc3)); + } else { + engine.delete(new Engine.Delete("test", "2", newUid(doc2))); + } + fail("corruption should throw exceptions"); + } catch (Exception e) { + assertThat(e, instanceOf(CorruptIndexException.class)); + } + } else { + // normal close + engine.close(); + } + // now the engine is closed check we respond correctly + try { + if (randomBoolean()) { + engine.index(indexForDoc(doc1)); + } else { + engine.delete(new Engine.Delete("test", "", newUid(doc1))); + } + fail("engine should be closed"); + } catch (Exception e) { + assertThat(e, instanceOf(AlreadyClosedException.class)); + } } } - } public void testDoubleDelivery() throws IOException { - final ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); - Engine.Index operation = randomAppendOnly(1, doc, false); - Engine.Index retry = randomAppendOnly(1, doc, true); + final ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); + Engine.Index operation = randomAppendOnly(doc, false, 1); + Engine.Index retry = randomAppendOnly(doc, true, 1); if (randomBoolean()) { Engine.IndexResult indexResult = engine.index(operation); assertFalse(engine.indexWriterHasDeletions()); @@ -2624,8 +2683,8 @@ public class InternalEngineTests extends ESTestCase { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10); assertEquals(1, topDocs.totalHits); } - operation = randomAppendOnly(1, doc, false); - retry = randomAppendOnly(1, doc, true); + operation = randomAppendOnly(doc, false, 1); + retry = randomAppendOnly(doc, true, 1); if (randomBoolean()) { Engine.IndexResult indexResult = engine.index(operation); assertNotNull(indexResult.getTranslogLocation()); @@ -2650,20 +2709,20 @@ public class InternalEngineTests extends ESTestCase { public void testRetryWithAutogeneratedIdWorksAndNoDuplicateDocs() throws IOException { - final ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); + final ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); boolean isRetry = false; long autoGeneratedIdTimestamp = 0; - Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); + Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); Engine.IndexResult indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(1L)); - index = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); + index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); indexResult = replicaEngine.index(index); assertThat(indexResult.getVersion(), equalTo(1L)); isRetry = true; - index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); + index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); indexResult = engine.index(index); assertThat(indexResult.getVersion(), equalTo(1L)); engine.refresh("test"); @@ -2672,7 +2731,7 @@ public class InternalEngineTests extends ESTestCase { assertEquals(1, topDocs.totalHits); } - index = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); + index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); indexResult = replicaEngine.index(index); assertThat(indexResult.hasFailure(), equalTo(false)); replicaEngine.refresh("test"); @@ -2684,20 +2743,20 @@ public class InternalEngineTests extends ESTestCase { public void testRetryWithAutogeneratedIdsAndWrongOrderWorksAndNoDuplicateDocs() throws IOException { - final ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); + final ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); boolean isRetry = true; long autoGeneratedIdTimestamp = 0; - Engine.Index firstIndexRequest = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); + Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); Engine.IndexResult result = engine.index(firstIndexRequest); assertThat(result.getVersion(), equalTo(1L)); - Engine.Index firstIndexRequestReplica = new Engine.Index(newUid("1"), doc, result.getSeqNo(), firstIndexRequest.primaryTerm(), result.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); + Engine.Index firstIndexRequestReplica = new Engine.Index(newUid(doc), doc, result.getSeqNo(), firstIndexRequest.primaryTerm(), result.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); Engine.IndexResult indexReplicaResult = replicaEngine.index(firstIndexRequestReplica); assertThat(indexReplicaResult.getVersion(), equalTo(1L)); isRetry = false; - Engine.Index secondIndexRequest = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); + Engine.Index secondIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); Engine.IndexResult indexResult = engine.index(secondIndexRequest); assertTrue(indexResult.isCreated()); engine.refresh("test"); @@ -2706,7 +2765,7 @@ public class InternalEngineTests extends ESTestCase { assertEquals(1, topDocs.totalHits); } - Engine.Index secondIndexRequestReplica = new Engine.Index(newUid("1"), doc, result.getSeqNo(), secondIndexRequest.primaryTerm(), result.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); + Engine.Index secondIndexRequestReplica = new Engine.Index(newUid(doc), doc, result.getSeqNo(), secondIndexRequest.primaryTerm(), result.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry); replicaEngine.index(secondIndexRequestReplica); replicaEngine.refresh("test"); try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) { @@ -2715,11 +2774,13 @@ public class InternalEngineTests extends ESTestCase { } } - public Engine.Index randomAppendOnly(int docId, ParsedDocument doc, boolean retry) { + public Engine.Index randomAppendOnly(ParsedDocument doc, boolean retry, final long autoGeneratedIdTimestamp) { if (randomBoolean()) { - return new Engine.Index(newUid(Integer.toString(docId)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), docId, retry); + return new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, + VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, retry); } - return new Engine.Index(newUid(Integer.toString(docId)), doc, 0, 0, 1, VersionType.EXTERNAL, Engine.Operation.Origin.REPLICA, System.nanoTime(), docId, retry); + return new Engine.Index(newUid(doc), doc, 0, 0, 1, VersionType.EXTERNAL, + Engine.Operation.Origin.REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, retry); } public void testRetryConcurrently() throws InterruptedException, IOException { @@ -2727,9 +2788,9 @@ public class InternalEngineTests extends ESTestCase { int numDocs = randomIntBetween(1000, 10000); List docs = new ArrayList<>(); for (int i = 0; i < numDocs; i++) { - final ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); - Engine.Index originalIndex = randomAppendOnly(i, doc, false); - Engine.Index retryIndex = randomAppendOnly(i, doc, true); + final ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); + Engine.Index originalIndex = randomAppendOnly(doc, false, i); + Engine.Index retryIndex = randomAppendOnly(doc, true, i); docs.add(originalIndex); docs.add(retryIndex); } @@ -2737,21 +2798,18 @@ public class InternalEngineTests extends ESTestCase { CountDownLatch startGun = new CountDownLatch(thread.length); AtomicInteger offset = new AtomicInteger(-1); for (int i = 0; i < thread.length; i++) { - thread[i] = new Thread() { - @Override - public void run() { - startGun.countDown(); - try { - startGun.await(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } - int docOffset; - while ((docOffset = offset.incrementAndGet()) < docs.size()) { - engine.index(docs.get(docOffset)); - } + thread[i] = new Thread(() -> { + startGun.countDown(); + try { + startGun.await(); + } catch (InterruptedException e) { + throw new AssertionError(e); } - }; + int docOffset; + while ((docOffset = offset.incrementAndGet()) < docs.size()) { + engine.index(docs.get(docOffset)); + } + }); thread[i].start(); } for (int i = 0; i < thread.length; i++) { @@ -2790,8 +2848,8 @@ public class InternalEngineTests extends ESTestCase { assertEquals(0, engine.getNumIndexVersionsLookups()); List docs = new ArrayList<>(); for (int i = 0; i < numDocs; i++) { - final ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); - Engine.Index index = randomAppendOnly(i, doc, false); + final ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); + Engine.Index index = randomAppendOnly(doc, false, i); docs.add(index); } Collections.shuffle(docs, random()); @@ -2862,16 +2920,16 @@ public class InternalEngineTests extends ESTestCase { } catch (InterruptedException e) { throw new AssertionError(e); } - throw new AlreadyClosedException("boom"); + throw new ElasticsearchException("something completely different"); } } }); InternalEngine internalEngine = new InternalEngine(config); int docId = 0; - final ParsedDocument doc = testParsedDocument(Integer.toString(docId), Integer.toString(docId), "test", null, + final ParsedDocument doc = testParsedDocument(Integer.toString(docId), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); - Engine.Index index = randomAppendOnly(docId, doc, false); + Engine.Index index = randomBoolean() ? indexForDoc(doc) : randomAppendOnly(doc, false, docId); internalEngine.index(index); Runnable r = () -> { try { @@ -2882,11 +2940,11 @@ public class InternalEngineTests extends ESTestCase { try { internalEngine.refresh("test"); fail(); - } catch (EngineClosedException ex) { - // we can't guarantee that we are entering the refresh call before it's fully - // closed so we also expecting ECE here - assertTrue(ex.toString(), ex.getCause() instanceof MockDirectoryWrapper.FakeIOException); - } catch (RefreshFailedEngineException | AlreadyClosedException ex) { + } catch (AlreadyClosedException ex) { + if (ex.getCause() != null) { + assertTrue(ex.toString(), ex.getCause() instanceof MockDirectoryWrapper.FakeIOException); + } + } catch (RefreshFailedEngineException ex) { // fine } finally { start.countDown(); @@ -2913,11 +2971,11 @@ public class InternalEngineTests extends ESTestCase { // create a document Document document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null); + engine.index(indexForDoc(doc)); engine.refresh("test"); - seqID = getSequenceID(engine, new Engine.Get(false, newUid("1"))); + seqID = getSequenceID(engine, new Engine.Get(false, newUid(doc))); logger.info("--> got seqID: {}", seqID); assertThat(seqID.v1(), equalTo(0L)); assertThat(seqID.v2(), equalTo(0L)); @@ -2925,11 +2983,11 @@ public class InternalEngineTests extends ESTestCase { // Index the same document again document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); + doc = testParsedDocument("1", "test", null, document, B_1, null); + engine.index(indexForDoc(doc)); engine.refresh("test"); - seqID = getSequenceID(engine, new Engine.Get(false, newUid("1"))); + seqID = getSequenceID(engine, new Engine.Get(false, newUid(doc))); logger.info("--> got seqID: {}", seqID); assertThat(seqID.v1(), equalTo(1L)); assertThat(seqID.v2(), equalTo(0L)); @@ -2937,13 +2995,13 @@ public class InternalEngineTests extends ESTestCase { // Index the same document for the third time, this time changing the primary term document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - engine.index(new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 1, + doc = testParsedDocument("1", "test", null, document, B_1, null); + engine.index(new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 1, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false)); engine.refresh("test"); - seqID = getSequenceID(engine, new Engine.Get(false, newUid("1"))); + seqID = getSequenceID(engine, new Engine.Get(false, newUid(doc))); logger.info("--> got seqID: {}", seqID); assertThat(seqID.v1(), equalTo(2L)); assertThat(seqID.v2(), equalTo(1L)); @@ -2994,11 +3052,10 @@ public class InternalEngineTests extends ESTestCase { final InternalEngine finalInitialEngine = initialEngine; for (int i = 0; i < docs; i++) { final String id = Integer.toString(i); - final Term uid = newUid(id); - final ParsedDocument doc = testParsedDocument(id, id, "test", null, testDocumentWithTextField(), SOURCE, null); + final ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null); skip.set(randomBoolean()); - final Thread thread = new Thread(() -> finalInitialEngine.index(new Engine.Index(uid, doc))); + final Thread thread = new Thread(() -> finalInitialEngine.index(indexForDoc(doc))); thread.start(); if (skip.get()) { threads.add(thread); @@ -3036,8 +3093,8 @@ public class InternalEngineTests extends ESTestCase { initialEngine = engine; for (int i = 0; i < docs; i++) { final String id = Integer.toString(i); - final Term uid = newUid(id); - final ParsedDocument doc = testParsedDocument(id, id, "test", null, testDocumentWithTextField(), SOURCE, null); + final ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null); + final Term uid = newUid(doc); // create a gap at sequence number 3 * i + 1 initialEngine.index(new Engine.Index(uid, doc, 3 * i, 1, v, t, REPLICA, System.nanoTime(), ts, false)); initialEngine.delete(new Engine.Delete("type", id, uid, 3 * i + 2, 1, v, t, REPLICA, System.nanoTime())); @@ -3050,8 +3107,8 @@ public class InternalEngineTests extends ESTestCase { for (int i = 0; i < docs; i++) { final String id = Integer.toString(i); - final Term uid = newUid(id); - final ParsedDocument doc = testParsedDocument(id, id, "test", null, testDocumentWithTextField(), SOURCE, null); + final ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null); + final Term uid = newUid(doc); initialEngine.index(new Engine.Index(uid, doc, 3 * i + 1, 1, v, t, REPLICA, System.nanoTime(), ts, false)); } } finally { @@ -3068,14 +3125,14 @@ public class InternalEngineTests extends ESTestCase { final List operations = new ArrayList<>(); final int numberOfOperations = randomIntBetween(16, 32); - final Term uid = newUid("1"); final Document document = testDocumentWithTextField(); final AtomicLong sequenceNumber = new AtomicLong(); final Engine.Operation.Origin origin = randomFrom(LOCAL_TRANSLOG_RECOVERY, PEER_RECOVERY, PRIMARY, REPLICA); final LongSupplier sequenceNumberSupplier = origin == PRIMARY ? () -> SequenceNumbersService.UNASSIGNED_SEQ_NO : sequenceNumber::getAndIncrement; document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - final ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null); + final ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null); + final Term uid = newUid(doc); for (int i = 0; i < numberOfOperations; i++) { if (randomBoolean()) { final Engine.Index index = new Engine.Index( diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java index a7470666d63..cc92d9bd9c2 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java @@ -33,6 +33,7 @@ import org.apache.lucene.index.SnapshotDeletionPolicy; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.IOUtils; @@ -44,21 +45,18 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.VersionType; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.UidFieldMapper; -import org.elasticsearch.index.seqno.SequenceNumbersService; -import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.RefreshListeners; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; @@ -83,7 +81,6 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; -import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasKey; @@ -172,8 +169,8 @@ public class ShadowEngineTests extends ESTestCase { } - private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, ParseContext.Document document, BytesReference source, Mapping mappingsUpdate) { - Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE); + private ParsedDocument testParsedDocument(String id, String type, String routing, ParseContext.Document document, BytesReference source, Mapping mappingsUpdate) { + Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", 0); SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID(); document.add(uidField); @@ -254,8 +251,16 @@ public class ShadowEngineTests extends ESTestCase { return config; } - protected Term newUid(String id) { - return new Term("_uid", id); +// protected Term newUid(String id) { +// return new Term("_uid", id); +// } + + protected Term newUid(ParsedDocument doc) { + return new Term("_uid", doc.uid()); + } + + private Engine.Index indexForDoc(ParsedDocument doc) { + return new Engine.Index(newUid(doc), doc); } protected static final BytesReference B_1 = new BytesArray(new byte[]{1}); @@ -264,8 +269,8 @@ public class ShadowEngineTests extends ESTestCase { public void testCommitStats() { // create a doc and refresh - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + primaryEngine.index(indexForDoc(doc)); CommitStats stats1 = replicaEngine.commitStats(); assertThat(stats1.getGeneration(), greaterThan(0L)); @@ -296,11 +301,11 @@ public class ShadowEngineTests extends ESTestCase { assertThat(primaryEngine.segmentsStats(false).getMemoryInBytes(), equalTo(0L)); // create a doc and refresh - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + primaryEngine.index(indexForDoc(doc)); - ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_2, null); - primaryEngine.index(new Engine.Index(newUid("2"), doc2)); + ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_2, null); + primaryEngine.index(indexForDoc(doc2)); primaryEngine.refresh("test"); segments = primaryEngine.segments(false); @@ -358,8 +363,8 @@ public class ShadowEngineTests extends ESTestCase { assertThat(segments.get(0).isCompound(), equalTo(true)); - ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, testDocumentWithTextField(), B_3, null); - primaryEngine.index(new Engine.Index(newUid("3"), doc3)); + ParsedDocument doc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_3, null); + primaryEngine.index(indexForDoc(doc3)); primaryEngine.refresh("test"); segments = primaryEngine.segments(false); @@ -408,7 +413,7 @@ public class ShadowEngineTests extends ESTestCase { assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertThat(segments.get(1).isCompound(), equalTo(true)); - primaryEngine.delete(new Engine.Delete("test", "1", newUid("1"))); + primaryEngine.delete(new Engine.Delete("test", "1", newUid(doc))); primaryEngine.refresh("test"); segments = primaryEngine.segments(false); @@ -430,8 +435,8 @@ public class ShadowEngineTests extends ESTestCase { primaryEngine.flush(); replicaEngine.refresh("test"); - ParsedDocument doc4 = testParsedDocument("4", "4", "test", null, testDocumentWithTextField(), B_3, null); - primaryEngine.index(new Engine.Index(newUid("4"), doc4)); + ParsedDocument doc4 = testParsedDocument("4", "test", null, testDocumentWithTextField(), B_3, null); + primaryEngine.index(indexForDoc(doc4)); primaryEngine.refresh("test"); segments = primaryEngine.segments(false); @@ -463,19 +468,19 @@ public class ShadowEngineTests extends ESTestCase { List segments = primaryEngine.segments(true); assertThat(segments.isEmpty(), equalTo(true)); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + primaryEngine.index(indexForDoc(doc)); primaryEngine.refresh("test"); segments = primaryEngine.segments(true); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).ramTree, notNullValue()); - ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_2, null); - primaryEngine.index(new Engine.Index(newUid("2"), doc2)); + ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_2, null); + primaryEngine.index(indexForDoc(doc2)); primaryEngine.refresh("test"); - ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, testDocumentWithTextField(), B_3, null); - primaryEngine.index(new Engine.Index(newUid("3"), doc3)); + ParsedDocument doc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_3, null); + primaryEngine.index(indexForDoc(doc3)); primaryEngine.refresh("test"); segments = primaryEngine.segments(true); @@ -500,9 +505,9 @@ public class ShadowEngineTests extends ESTestCase { // create a document ParseContext.Document document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null); + ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null); try { - replicaEngine.index(new Engine.Index(newUid("1"), doc)); + replicaEngine.index(indexForDoc(doc)); fail("should have thrown an exception"); } catch (UnsupportedOperationException e) {} replicaEngine.refresh("test"); @@ -512,16 +517,16 @@ public class ShadowEngineTests extends ESTestCase { MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0)); searchResult.close(); - Engine.GetResult getResult = replicaEngine.get(new Engine.Get(true, newUid("1"))); + Engine.GetResult getResult = replicaEngine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(false)); getResult.release(); // index a document document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); - doc = testParsedDocument("1", "1", "test", null, document, B_1, null); + doc = testParsedDocument("1", "test", null, document, B_1, null); try { - replicaEngine.index(new Engine.Index(newUid("1"), doc)); + replicaEngine.index(indexForDoc(doc)); fail("should have thrown an exception"); } catch (UnsupportedOperationException e) {} replicaEngine.refresh("test"); @@ -531,15 +536,15 @@ public class ShadowEngineTests extends ESTestCase { MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0)); searchResult.close(); - getResult = replicaEngine.get(new Engine.Get(true, newUid("1"))); + getResult = replicaEngine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(false)); getResult.release(); // Now, add a document to the primary so we can test shadow engine deletes document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + doc = testParsedDocument("1", "test", null, document, B_1, null); + primaryEngine.index(indexForDoc(doc)); primaryEngine.flush(); replicaEngine.refresh("test"); @@ -550,14 +555,14 @@ public class ShadowEngineTests extends ESTestCase { searchResult.close(); // And the replica can retrieve it - getResult = replicaEngine.get(new Engine.Get(false, newUid("1"))); + getResult = replicaEngine.get(new Engine.Get(false, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); // try to delete it on the replica try { - replicaEngine.delete(new Engine.Delete("test", "1", newUid("1"))); + replicaEngine.delete(new Engine.Delete("test", "1", newUid(doc))); fail("should have thrown an exception"); } catch (UnsupportedOperationException e) {} replicaEngine.flush(); @@ -569,7 +574,7 @@ public class ShadowEngineTests extends ESTestCase { MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1)); searchResult.close(); - getResult = replicaEngine.get(new Engine.Get(false, newUid("1"))); + getResult = replicaEngine.get(new Engine.Get(false, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); @@ -579,7 +584,7 @@ public class ShadowEngineTests extends ESTestCase { MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1)); searchResult.close(); - getResult = primaryEngine.get(new Engine.Get(false, newUid("1"))); + getResult = primaryEngine.get(new Engine.Get(false, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); @@ -593,8 +598,8 @@ public class ShadowEngineTests extends ESTestCase { // create a document ParseContext.Document document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null); + primaryEngine.index(indexForDoc(doc)); // its not there... searchResult = primaryEngine.acquireSearcher("test"); @@ -609,18 +614,18 @@ public class ShadowEngineTests extends ESTestCase { searchResult.close(); // but, we can still get it (in realtime) - Engine.GetResult getResult = primaryEngine.get(new Engine.Get(true, newUid("1"))); + Engine.GetResult getResult = primaryEngine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); // can't get it from the replica, because it's not in the translog for a shadow replica - getResult = replicaEngine.get(new Engine.Get(true, newUid("1"))); + getResult = replicaEngine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(false)); getResult.release(); // but, not there non realtime - getResult = primaryEngine.get(new Engine.Get(false, newUid("1"))); + getResult = primaryEngine.get(new Engine.Get(false, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); getResult.release(); @@ -631,7 +636,7 @@ public class ShadowEngineTests extends ESTestCase { searchResult.close(); // also in non realtime - getResult = primaryEngine.get(new Engine.Get(false, newUid("1"))); + getResult = primaryEngine.get(new Engine.Get(false, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); @@ -646,8 +651,8 @@ public class ShadowEngineTests extends ESTestCase { document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE)); - doc = testParsedDocument("1", "1", "test", null, document, B_2, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + doc = testParsedDocument("1", "test", null, document, B_2, null); + primaryEngine.index(indexForDoc(doc)); // its not updated yet... searchResult = primaryEngine.acquireSearcher("test"); @@ -657,7 +662,7 @@ public class ShadowEngineTests extends ESTestCase { searchResult.close(); // but, we can still get it (in realtime) - getResult = primaryEngine.get(new Engine.Get(true, newUid("1"))); + getResult = primaryEngine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); @@ -690,7 +695,7 @@ public class ShadowEngineTests extends ESTestCase { searchResult.close(); // now delete - primaryEngine.delete(new Engine.Delete("test", "1", newUid("1"))); + primaryEngine.delete(new Engine.Delete("test", "1", newUid(doc))); // its not deleted yet searchResult = primaryEngine.acquireSearcher("test"); @@ -700,7 +705,7 @@ public class ShadowEngineTests extends ESTestCase { searchResult.close(); // but, get should not see it (in realtime) - getResult = primaryEngine.get(new Engine.Get(true, newUid("1"))); + getResult = primaryEngine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(false)); getResult.release(); @@ -716,8 +721,8 @@ public class ShadowEngineTests extends ESTestCase { // add it back document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + doc = testParsedDocument("1", "test", null, document, B_1, null); + primaryEngine.index(indexForDoc(doc)); // its not there... searchResult = primaryEngine.acquireSearcher("test"); @@ -740,7 +745,7 @@ public class ShadowEngineTests extends ESTestCase { primaryEngine.flush(); // and, verify get (in real time) - getResult = primaryEngine.get(new Engine.Get(true, newUid("1"))); + getResult = primaryEngine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); @@ -752,7 +757,7 @@ public class ShadowEngineTests extends ESTestCase { MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0)); searchResult.close(); - getResult = replicaEngine.get(new Engine.Get(true, newUid("1"))); + getResult = replicaEngine.get(new Engine.Get(true, newUid(doc))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); @@ -761,8 +766,8 @@ public class ShadowEngineTests extends ESTestCase { // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); - doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + doc = testParsedDocument("1", "test", null, document, B_1, null); + primaryEngine.index(indexForDoc(doc)); // its not updated yet... searchResult = primaryEngine.acquireSearcher("test"); @@ -797,8 +802,8 @@ public class ShadowEngineTests extends ESTestCase { searchResult.close(); // create a document - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + primaryEngine.index(indexForDoc(doc)); // its not there... searchResult = primaryEngine.acquireSearcher("test"); @@ -827,7 +832,7 @@ public class ShadowEngineTests extends ESTestCase { // don't release the replica search result yet... // delete, refresh and do a new search, it should not be there - primaryEngine.delete(new Engine.Delete("test", "1", newUid("1"))); + primaryEngine.delete(new Engine.Delete(doc.type(), doc.id(), newUid(doc))); primaryEngine.flush(); primaryEngine.refresh("test"); replicaEngine.refresh("test"); @@ -842,8 +847,8 @@ public class ShadowEngineTests extends ESTestCase { } public void testFailEngineOnCorruption() { - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + primaryEngine.index(indexForDoc(doc)); primaryEngine.flush(); MockDirectoryWrapper leaf = DirectoryUtils.getLeaf(replicaEngine.config().getStore().directory(), MockDirectoryWrapper.class); leaf.setRandomIOExceptionRate(1.0); @@ -860,7 +865,7 @@ public class ShadowEngineTests extends ESTestCase { MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1)); searchResult.close(); fail("exception expected"); - } catch (EngineClosedException ex) { + } catch (AlreadyClosedException ex) { // all is well } } @@ -879,8 +884,8 @@ public class ShadowEngineTests extends ESTestCase { */ public void testFailStart() throws IOException { // Need a commit point for this - ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null); - primaryEngine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null); + primaryEngine.index(indexForDoc(doc)); primaryEngine.flush(); // this test fails if any reader, searcher or directory is not closed - MDW FTW @@ -965,8 +970,8 @@ public class ShadowEngineTests extends ESTestCase { // create a document ParseContext.Document document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null); - pEngine.index(new Engine.Index(newUid("1"), doc)); + ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null); + pEngine.index(indexForDoc(doc)); pEngine.flush(true, true); t.join(); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index e234beb7904..4df4361db6a 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -214,7 +214,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase { assertEquals("b", fields[1].stringValue()); IndexShard shard = indexService.getShard(0); - shard.index(new Engine.Index(new Term("_uid", "1"), doc)); + shard.index(new Engine.Index(new Term("_uid", doc.uid() ), doc)); shard.refresh("test"); try (Engine.Searcher searcher = shard.acquireSearcher("test")) { LeafReader leaf = searcher.getDirectoryReader().leaves().get(0).reader(); @@ -253,7 +253,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase { assertEquals("b", fields[1].stringValue()); IndexShard shard = indexService.getShard(0); - shard.index(new Engine.Index(new Term("_uid", "1"), doc)); + shard.index(new Engine.Index(new Term("_uid", doc.uid()), doc)); shard.refresh("test"); try (Engine.Searcher searcher = shard.acquireSearcher("test")) { LeafReader leaf = searcher.getDirectoryReader().leaves().get(0).reader(); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index c17935375e8..3e35ed357ff 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -56,6 +56,7 @@ import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SeqNoFieldMapper; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.UidFieldMapper; import org.elasticsearch.index.seqno.SequenceNumbersService; import org.elasticsearch.index.translog.Translog; @@ -100,9 +101,9 @@ public class IndexShardIT extends ESSingleNodeTestCase { return pluginList(InternalSettingsPlugin.class); } - private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, long seqNo, + private ParsedDocument testParsedDocument(String id, String type, String routing, long seqNo, ParseContext.Document document, BytesReference source, Mapping mappingUpdate) { - Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE); + Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", 0); SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID(); document.add(uidField); @@ -325,14 +326,13 @@ public class IndexShardIT extends ESSingleNodeTestCase { client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get(); assertFalse(shard.shouldFlush()); ParsedDocument doc = testParsedDocument( - "1", "1", "test", null, SequenceNumbersService.UNASSIGNED_SEQ_NO, new ParseContext.Document(), new BytesArray(new byte[]{1}), null); - Engine.Index index = new Engine.Index(new Term("_uid", "1"), doc); + Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc); shard.index(index); assertTrue(shard.shouldFlush()); assertEquals(2, shard.getEngine().getTranslog().totalOperations()); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index d4e27e857de..79e3868da46 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -29,6 +29,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.Constants; import org.elasticsearch.Version; @@ -547,9 +548,9 @@ public class IndexShardTests extends IndexShardTestCase { closeShards(shard); } - private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, + private ParsedDocument testParsedDocument(String id, String type, String routing, ParseContext.Document document, BytesReference source, Mapping mappingUpdate) { - Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE); + Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", 0); SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID(); document.add(uidField); @@ -619,9 +620,9 @@ public class IndexShardTests extends IndexShardTestCase { }); recoveryShardFromStore(shard); - ParsedDocument doc = testParsedDocument("1", "1", "test", null, new ParseContext.Document(), + ParsedDocument doc = testParsedDocument("1", "test", null, new ParseContext.Document(), new BytesArray(new byte[]{1}), null); - Engine.Index index = new Engine.Index(new Term("_uid", "1"), doc); + Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc); shard.index(index); assertEquals(1, preIndex.get()); assertEquals(1, postIndexCreate.get()); @@ -640,7 +641,7 @@ public class IndexShardTests extends IndexShardTestCase { assertEquals(0, postDelete.get()); assertEquals(0, postDeleteException.get()); - Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", "1")); + Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", doc.uid())); shard.delete(delete); assertEquals(2, preIndex.get()); @@ -657,7 +658,7 @@ public class IndexShardTests extends IndexShardTestCase { try { shard.index(index); fail(); - } catch (IllegalIndexShardStateException e) { + } catch (AlreadyClosedException e) { } @@ -671,7 +672,7 @@ public class IndexShardTests extends IndexShardTestCase { try { shard.delete(delete); fail(); - } catch (IllegalIndexShardStateException e) { + } catch (AlreadyClosedException e) { } @@ -1376,10 +1377,10 @@ public class IndexShardTests extends IndexShardTestCase { for (int i = 0; i < numDocs; i++) { final String id = Integer.toString(i); final ParsedDocument doc = - testParsedDocument(id, id, "test", null, new ParseContext.Document(), new BytesArray("{}"), null); + testParsedDocument(id, "test", null, new ParseContext.Document(), new BytesArray("{}"), null); final Engine.Index index = new Engine.Index( - new Term("_uid", id), + new Term("_uid", doc.uid()), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, @@ -1406,10 +1407,10 @@ public class IndexShardTests extends IndexShardTestCase { for (final Integer i : ids) { final String id = Integer.toString(i); final ParsedDocument doc = - testParsedDocument(id, id, "test", null, new ParseContext.Document(), new BytesArray("{}"), null); + testParsedDocument(id, "test", null, new ParseContext.Document(), new BytesArray("{}"), null); final Engine.Index index = new Engine.Index( - new Term("_uid", id), + new Term("_uid", doc.uid()), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java index 3d5a9fdf137..2eb91a16d80 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java @@ -21,6 +21,8 @@ package org.elasticsearch.index.shard; import org.apache.lucene.index.Term; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.InternalEngineTests; +import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.seqno.SequenceNumbersService; import org.elasticsearch.test.ESTestCase; @@ -131,9 +133,10 @@ public class IndexingOperationListenerTests extends ESTestCase{ } Collections.shuffle(indexingOperationListeners, random()); IndexingOperationListener.CompositeListener compositeListener = - new IndexingOperationListener.CompositeListener(indexingOperationListeners, logger); - Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", "1")); - Engine.Index index = new Engine.Index(new Term("_uid", "1"), null); + new IndexingOperationListener.CompositeListener(indexingOperationListeners, logger); + ParsedDocument doc = InternalEngineTests.createParsedDoc("1", "test", null); + Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", doc.uid())); + Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc); compositeListener.postDelete(randomShardId, delete, new Engine.DeleteResult(1, SequenceNumbersService.UNASSIGNED_SEQ_NO, true)); assertEquals(0, preIndex.get()); assertEquals(0, postIndex.get()); diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index c1e2605ec21..e95d7ace10b 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SeqNoFieldMapper; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.UidFieldMapper; import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.Store; @@ -297,7 +298,7 @@ public class RefreshListenersTests extends ESTestCase { } listener.assertNoError(); - Engine.Get get = new Engine.Get(false, new Term("_uid", "test:"+threadId)); + Engine.Get get = new Engine.Get(false, new Term("_uid", Uid.createUid("test", threadId))); try (Engine.GetResult getResult = engine.get(get)) { assertTrue("document not found", getResult.exists()); assertEquals(iteration, getResult.version()); @@ -328,7 +329,7 @@ public class RefreshListenersTests extends ESTestCase { String uid = type + ":" + id; Document document = new Document(); document.add(new TextField("test", testFieldValue, Field.Store.YES)); - Field uidField = new Field("_uid", type + ":" + id, UidFieldMapper.Defaults.FIELD_TYPE); + Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY); SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID(); document.add(uidField); @@ -338,7 +339,7 @@ public class RefreshListenersTests extends ESTestCase { document.add(seqID.primaryTerm); BytesReference source = new BytesArray(new byte[] { 1 }); ParsedDocument doc = new ParsedDocument(versionField, seqID, id, type, null, Arrays.asList(document), source, null); - Engine.Index index = new Engine.Index(new Term("_uid", uid), doc); + Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc); return engine.index(index); } diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index a3e3f611b21..1a9d7c97dc3 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -56,6 +56,7 @@ import org.elasticsearch.index.engine.Engine.Operation.Origin; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SeqNoFieldMapper; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.UidFieldMapper; import org.elasticsearch.index.seqno.SequenceNumbersService; import org.elasticsearch.index.shard.ShardId; @@ -625,8 +626,12 @@ public class TranslogTests extends ESTestCase { } } - private Term newUid(String id) { - return new Term("_uid", id); + private Term newUid(ParsedDocument doc) { + return new Term("_uid", doc.uid()); + } + + private Term newUid(String uid) { + return new Term("_uid", uid); } public void testVerifyTranslogIsNotDeleted() throws IOException { @@ -2014,7 +2019,7 @@ public class TranslogTests extends ESTestCase { seqID.seqNo.setLongValue(randomSeqNum); seqID.seqNoDocValue.setLongValue(randomSeqNum); seqID.primaryTerm.setLongValue(randomPrimaryTerm); - Field uidField = new Field("_uid", "1", UidFieldMapper.Defaults.FIELD_TYPE); + Field uidField = new Field("_uid", Uid.createUid("test", "1"), UidFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", 1); Document document = new Document(); document.add(new TextField("value", "test", Field.Store.YES)); @@ -2025,7 +2030,7 @@ public class TranslogTests extends ESTestCase { document.add(seqID.primaryTerm); ParsedDocument doc = new ParsedDocument(versionField, seqID, "1", "type", null, Arrays.asList(document), B_1, null); - Engine.Index eIndex = new Engine.Index(newUid("1"), doc, randomSeqNum, randomPrimaryTerm, + Engine.Index eIndex = new Engine.Index(newUid(doc), doc, randomSeqNum, randomPrimaryTerm, 1, VersionType.INTERNAL, Origin.PRIMARY, 0, 0, false); Engine.IndexResult eIndexResult = new Engine.IndexResult(1, randomSeqNum, true); Translog.Index index = new Translog.Index(eIndex, eIndexResult); @@ -2036,7 +2041,7 @@ public class TranslogTests extends ESTestCase { Translog.Index serializedIndex = new Translog.Index(in); assertEquals(index, serializedIndex); - Engine.Delete eDelete = new Engine.Delete("type", "1", newUid("1"), randomSeqNum, randomPrimaryTerm, + Engine.Delete eDelete = new Engine.Delete(doc.type(), doc.id(), newUid(doc), randomSeqNum, randomPrimaryTerm, 2, VersionType.INTERNAL, Origin.PRIMARY, 0); Engine.DeleteResult eDeleteResult = new Engine.DeleteResult(2, randomSeqNum, true); Translog.Delete delete = new Translog.Delete(eDelete, eDeleteResult);