diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java index aba7fda1021..97fbda80dc6 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java @@ -31,7 +31,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.gateway.GatewayAllocator; @@ -102,7 +102,7 @@ public final class Allocators { } public static DiscoveryNode newNode(String nodeId, Map attributes) { - return new DiscoveryNode("", nodeId, DummyTransportAddress.INSTANCE, attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER, + return new DiscoveryNode("", nodeId, LocalTransportAddress.buildUnique(), attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA), Version.CURRENT); } } diff --git a/buildSrc/src/main/resources/checkstyle.xml b/buildSrc/src/main/resources/checkstyle.xml index de47736913f..706ef46ffa1 100644 --- a/buildSrc/src/main/resources/checkstyle.xml +++ b/buildSrc/src/main/resources/checkstyle.xml @@ -39,6 +39,27 @@ + + + + + waiting for 3 nodes to be up"); - assertBusy(new Runnable() { - @Override - public void run() { - NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get(); - assertThat(resp.getNodes().size(), equalTo(3)); - } + assertBusy(() -> { + NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get(); + assertThat(resp.getNodes().size(), equalTo(3)); }); logger.info("--> creating 'test' index"); @@ -126,7 +123,6 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase { Map explanations = cae.getNodeExplanations(); - Float noAttrWeight = -1f; Float barAttrWeight = -1f; Float fooBarAttrWeight = -1f; for (Map.Entry entry : explanations.entrySet()) { @@ -134,7 +130,6 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase { String nodeName = node.getName(); NodeExplanation explanation = entry.getValue(); ClusterAllocationExplanation.FinalDecision finalDecision = explanation.getFinalDecision(); - String finalExplanation = explanation.getFinalExplanation(); ClusterAllocationExplanation.StoreCopy storeCopy = explanation.getStoreCopy(); Decision d = explanation.getDecision(); float weight = explanation.getWeight(); @@ -143,7 +138,6 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase { assertEquals(d.type(), Decision.Type.NO); if (noAttrNode.equals(nodeName)) { assertThat(d.toString(), containsString("node does not match index include filters [foo:\"bar\"]")); - noAttrWeight = weight; assertNull(storeStatus); assertEquals("the shard cannot be assigned because one or more allocation decider returns a 'NO' decision", explanation.getFinalExplanation()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java index d5cefc6d1f3..6c23d1604b8 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java @@ -47,7 +47,6 @@ public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase { NodeExplanation explanation = cae.getNodeExplanations().values().iterator().next(); ClusterAllocationExplanation.FinalDecision fd = explanation.getFinalDecision(); ClusterAllocationExplanation.StoreCopy storeCopy = explanation.getStoreCopy(); - String finalExplanation = explanation.getFinalExplanation(); Decision d = explanation.getDecision(); assertNotNull("should have a decision", d); assertEquals(Decision.Type.NO, d.type()); @@ -76,7 +75,6 @@ public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase { d = explanation.getDecision(); fd = explanation.getFinalDecision(); storeCopy = explanation.getStoreCopy(); - finalExplanation = explanation.getFinalExplanation(); assertNotNull("should have a decision", d); assertEquals(Decision.Type.NO, d.type()); assertEquals(ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED, fd); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java index d0e8ef14d01..5b39c5d34dc 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java @@ -31,7 +31,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.Decision; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -41,7 +41,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -66,7 +65,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase { .numberOfShards(1) .numberOfReplicas(1) .build(); - private DiscoveryNode node = new DiscoveryNode("node-0", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + private DiscoveryNode node = new DiscoveryNode("node-0", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); private static Decision.Multi yesDecision = new Decision.Multi(); private static Decision.Multi noDecision = new Decision.Multi(); @@ -205,7 +204,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase { "assignedNode", allocationDelay, remainingDelay, null, false, nodeExplanations); BytesStreamOutput out = new BytesStreamOutput(); cae.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); ClusterAllocationExplanation cae2 = new ClusterAllocationExplanation(in); assertEquals(shard, cae2.getShard()); assertTrue(cae2.isPrimary()); @@ -215,9 +214,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase { assertEquals(allocationDelay, cae2.getAllocationDelayMillis()); assertEquals(remainingDelay, cae2.getRemainingDelayMillis()); for (Map.Entry entry : cae2.getNodeExplanations().entrySet()) { - DiscoveryNode node = entry.getKey(); NodeExplanation explanation = entry.getValue(); - IndicesShardStoresResponse.StoreStatus status = explanation.getStoreStatus(); assertNotNull(explanation.getStoreStatus()); assertNotNull(explanation.getDecision()); assertEquals(nodeWeight, explanation.getWeight()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java index 704c1348b7e..d0d452df478 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java @@ -84,7 +84,7 @@ public class ClusterHealthResponsesTests extends ESTestCase { if (randomBoolean()) { BytesStreamOutput out = new BytesStreamOutput(); clusterHealth.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); clusterHealth = ClusterHealthResponse.readResponseFrom(in); } return clusterHealth; diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java index 594028f4e6f..9027b3d372e 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java @@ -230,7 +230,7 @@ public class CancellableTasksTests extends TaskManagerTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { throwableReference.set(e); responseLatch.countDown(); } @@ -308,7 +308,7 @@ public class CancellableTasksTests extends TaskManagerTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { throwableReference.set(e); responseLatch.countDown(); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 6f8e3fda156..f9045b58413 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -738,12 +738,12 @@ public class TasksIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { throw new RuntimeException(e); } }); b.await(); - + // Now we can find it! GetTaskResponse response = expectFinishedTask(new TaskId("fake:1")); assertEquals("test", response.getTask().getTask().getAction()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index c4d49d899b9..2c78786ab04 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -249,7 +249,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { /** * Test class for testing task operations */ - static abstract class TestTasksAction extends TransportTasksAction { + abstract static class TestTasksAction extends TransportTasksAction { protected TestTasksAction(Settings settings, String actionName, ThreadPool threadPool, ClusterService clusterService, TransportService transportService) { @@ -338,7 +338,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { logger.warn("Couldn't get list of tasks", e); responseLatch.countDown(); } @@ -526,7 +526,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { responseLatch.countDown(); } }); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java index b736751b781..4f553dfb88a 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java @@ -166,7 +166,7 @@ public class ClusterRerouteRequestTests extends ESTestCase { private ClusterRerouteRequest roundTripThroughBytes(ClusterRerouteRequest original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { ClusterRerouteRequest copy = new ClusterRerouteRequest(); copy.readFrom(in); return copy; diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java index 00fcbf60a5a..657fec558b8 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java @@ -66,7 +66,7 @@ public class ClusterRerouteTests extends ESAllocationTestCase { BytesReference bytes = out.bytes(); NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); new NetworkModule(null, Settings.EMPTY, true, namedWriteableRegistry); - StreamInput wrap = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes.toBytes()), + StreamInput wrap = new NamedWriteableAwareStreamInput(bytes.streamInput(), namedWriteableRegistry); ClusterRerouteRequest deserializedReq = new ClusterRerouteRequest(); deserializedReq.readFrom(wrap); @@ -94,7 +94,7 @@ public class ClusterRerouteTests extends ESAllocationTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { } }; diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java index fc04de81254..b515829b72a 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java @@ -45,7 +45,7 @@ public class ClusterStateRequestTests extends ESTestCase { output.setVersion(testVersion); clusterStateRequest.writeTo(output); - StreamInput streamInput = StreamInput.wrap(output.bytes()); + StreamInput streamInput = output.bytes().streamInput(); streamInput.setVersion(testVersion); ClusterStateRequest deserializedCSRequest = new ClusterStateRequest(); deserializedCSRequest.readFrom(streamInput); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java index e4c2849b907..2e9239a2c3b 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java @@ -37,7 +37,7 @@ public class GetStoredScriptRequestTests extends ESTestCase { out.setVersion(randomVersion(random())); request.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); in.setVersion(out.getVersion()); GetStoredScriptRequest request2 = new GetStoredScriptRequest(); request2.readFrom(in); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 3e7323dceeb..428e859e342 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -41,14 +41,11 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; -import org.elasticsearch.node.service.NodeService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.junit.Ignore; import java.util.HashMap; -import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; @@ -232,7 +229,7 @@ public class CreateIndexIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { throw new RuntimeException(e); } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java index 04f6037f64b..7040c92ec1d 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java @@ -84,7 +84,7 @@ public class SyncedFlushUnitTests extends ESTestCase { assertThat(testPlan.result.restStatus(), equalTo(testPlan.totalCounts.failed > 0 ? RestStatus.CONFLICT : RestStatus.OK)); BytesStreamOutput out = new BytesStreamOutput(); testPlan.result.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); SyncedFlushResponse readResponse = new SyncedFlushResponse(); readResponse.readFrom(in); assertThat(readResponse.totalShards(), equalTo(testPlan.totalCounts.total)); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java index 6e3e5d76224..1cd1704e164 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java @@ -215,7 +215,7 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase { client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet(); } - private final static class IndexNodePredicate implements Predicate { + private static final class IndexNodePredicate implements Predicate { private final Set nodesWithShard; public IndexNodePredicate(String index) { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java index 3c12d7d9b10..9705009a044 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -53,8 +53,8 @@ public class IndicesShardStoreResponseTests extends ESTestCase { List failures = new ArrayList<>(); ImmutableOpenIntMap.Builder> storeStatuses = ImmutableOpenIntMap.builder(); - DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); - DiscoveryNode node2 = new DiscoveryNode("node2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode node2 = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); List storeStatusList = new ArrayList<>(); storeStatusList.add(new IndicesShardStoresResponse.StoreStatus(node1, 3, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null)); storeStatusList.add(new IndicesShardStoresResponse.StoreStatus(node2, ShardStateMetaData.NO_VERSION, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null)); @@ -122,7 +122,7 @@ public class IndicesShardStoreResponseTests extends ESTestCase { } public void testStoreStatusOrdering() throws Exception { - DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); List orderedStoreStatuses = new ArrayList<>(); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, ShardStateMetaData.NO_VERSION, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, ShardStateMetaData.NO_VERSION, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null)); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java index d78374d446f..3236de4aaeb 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java @@ -38,7 +38,7 @@ import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllo import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.gateway.NoopGatewayAllocator; @@ -140,7 +140,7 @@ public class TransportShrinkActionTests extends ESTestCase { } private DiscoveryNode newNode(String nodeId) { - return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(), + return new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), emptyMap(), Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), Version.CURRENT); } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java index 726dccee597..dfc10169e70 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.stats; +import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.engine.CommitStats; @@ -26,6 +27,8 @@ import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.test.ESSingleNodeTestCase; +import java.util.List; + import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasKey; @@ -108,4 +111,12 @@ public class IndicesStatsTests extends ESSingleNodeTestCase { } } + /** + * Gives access to package private IndicesStatsResponse constructor for test purpose. + **/ + public static IndicesStatsResponse newIndicesStatsResponse(ShardStats[] shards, int totalShards, int successfulShards, + int failedShards, List shardFailures) { + return new IndicesStatsResponse(shards, totalShards, successfulShards, failedShards, shardFailures); + } + } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java index d62fe30f6fa..58784fdb7df 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.admin.indices.template.put; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.cluster.metadata.AliasValidator; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -172,8 +171,8 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase { } @Override - public void onFailure(Throwable t) { - throwables.add(t); + public void onFailure(Exception e) { + throwables.add(e); } }); return throwables; @@ -205,8 +204,8 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase { } @Override - public void onFailure(Throwable t) { - throwables.add(t); + public void onFailure(Exception e) { + throwables.add(e); latch.countDown(); } }); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index 337f881d41b..142fb282c20 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -55,9 +55,9 @@ public class BulkRequestTests extends ESTestCase { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); assertThat(bulkRequest.numberOfActions(), equalTo(3)); - assertThat(((IndexRequest) bulkRequest.requests().get(0)).source().toBytes(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }").toBytes())); + assertThat(((IndexRequest) bulkRequest.requests().get(0)).source(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }"))); assertThat(bulkRequest.requests().get(1), instanceOf(DeleteRequest.class)); - assertThat(((IndexRequest) bulkRequest.requests().get(2)).source().toBytes(), equalTo(new BytesArray("{ \"field1\" : \"value3\" }").toBytes())); + assertThat(((IndexRequest) bulkRequest.requests().get(2)).source(), equalTo(new BytesArray("{ \"field1\" : \"value3\" }"))); } public void testSimpleBulk2() throws Exception { @@ -81,7 +81,7 @@ public class BulkRequestTests extends ESTestCase { assertThat(bulkRequest.numberOfActions(), equalTo(4)); assertThat(((UpdateRequest) bulkRequest.requests().get(0)).id(), equalTo("1")); assertThat(((UpdateRequest) bulkRequest.requests().get(0)).retryOnConflict(), equalTo(2)); - assertThat(((UpdateRequest) bulkRequest.requests().get(0)).doc().source().toUtf8(), equalTo("{\"field\":\"value\"}")); + assertThat(((UpdateRequest) bulkRequest.requests().get(0)).doc().source().utf8ToString(), equalTo("{\"field\":\"value\"}")); assertThat(((UpdateRequest) bulkRequest.requests().get(1)).id(), equalTo("0")); assertThat(((UpdateRequest) bulkRequest.requests().get(1)).type(), equalTo("type1")); assertThat(((UpdateRequest) bulkRequest.requests().get(1)).index(), equalTo("index1")); @@ -93,7 +93,7 @@ public class BulkRequestTests extends ESTestCase { assertThat(scriptParams, notNullValue()); assertThat(scriptParams.size(), equalTo(1)); assertThat(((Integer) scriptParams.get("param1")), equalTo(1)); - assertThat(((UpdateRequest) bulkRequest.requests().get(1)).upsertRequest().source().toUtf8(), equalTo("{\"counter\":1}")); + assertThat(((UpdateRequest) bulkRequest.requests().get(1)).upsertRequest().source().utf8ToString(), equalTo("{\"counter\":1}")); } public void testBulkAllowExplicitIndex() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java index b26d2531ff0..bb406366d25 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java @@ -29,11 +29,11 @@ public class BulkShardRequestTests extends ESTestCase { public void testToString() { String index = randomSimpleString(random(), 10); int count = between(1, 100); - BulkShardRequest r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.NONE, new BulkItemRequest[count]); + BulkShardRequest r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.NONE, new BulkItemRequest[count]); assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests", r.toString()); - r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.IMMEDIATE, new BulkItemRequest[count]); + r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.IMMEDIATE, new BulkItemRequest[count]); assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests and a refresh", r.toString()); - r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.WAIT_UNTIL, new BulkItemRequest[count]); + r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.WAIT_UNTIL, new BulkItemRequest[count]); assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests blocking until refresh", r.toString()); } } diff --git a/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java b/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java index 6d9987394f9..4fa640b3adc 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java @@ -149,7 +149,7 @@ public class RetryTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { this.lastFailure = e; latch.countDown(); } diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 6ae7559ba62..7c39adc76f6 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -59,7 +59,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; public class TransportBulkActionTookTests extends ESTestCase { - static private ThreadPool threadPool; + private static ThreadPool threadPool; private ClusterService clusterService; @BeforeClass @@ -201,7 +201,7 @@ public class TransportBulkActionTookTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { } }); diff --git a/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java index 451ade62584..ef259463139 100644 --- a/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java @@ -70,7 +70,7 @@ public class MultiGetShardRequestTests extends ESTestCase { out.setVersion(randomVersion(random())); multiGetShardRequest.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); in.setVersion(out.getVersion()); MultiGetShardRequest multiGetShardRequest2 = new MultiGetShardRequest(); multiGetShardRequest2.readFrom(in); diff --git a/core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java b/core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java index 3286c07e06c..9ee5036131d 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java @@ -111,7 +111,7 @@ public class BulkRequestModifierTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { } }); @@ -157,7 +157,7 @@ public class BulkRequestModifierTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { } public BulkResponse getResponse() { diff --git a/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java b/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java index a62946bf0f6..3d1a1a1c69d 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java @@ -33,7 +33,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -78,7 +78,7 @@ public class IngestProxyActionFilterTests extends ESTestCase { if (i < ingestNodes) { roles.add(DiscoveryNode.Role.INGEST); } - DiscoveryNode node = new DiscoveryNode(nodeId, nodeId, DummyTransportAddress.INSTANCE, attributes, roles, VersionUtils.randomVersion(random())); + DiscoveryNode node = new DiscoveryNode(nodeId, nodeId, LocalTransportAddress.buildUnique(), attributes, roles, VersionUtils.randomVersion(random())); builder.put(node); if (i == totalNodes - 1) { localNode = node; diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java index 323a8c0aaa6..544e2932b44 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java @@ -45,7 +45,7 @@ public class SimulateDocumentSimpleResultTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); simulateDocumentBaseResult.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); SimulateDocumentBaseResult otherSimulateDocumentBaseResult = new SimulateDocumentBaseResult(streamInput); if (isFailure) { diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java index 1376ca4280e..576e8e01724 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java @@ -73,7 +73,7 @@ public class SimulatePipelineResponseTests extends ESTestCase { SimulatePipelineResponse response = new SimulatePipelineResponse(randomAsciiOfLengthBetween(1, 10), isVerbose, results); BytesStreamOutput out = new BytesStreamOutput(); response.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); SimulatePipelineResponse otherResponse = new SimulatePipelineResponse(); otherResponse.readFrom(streamInput); diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java index f612f36c9d6..ccf3a674944 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java @@ -48,7 +48,7 @@ public class SimulateProcessorResultTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); simulateProcessorResult.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); SimulateProcessorResult otherSimulateProcessorResult = new SimulateProcessorResult(streamInput); assertThat(otherSimulateProcessorResult.getProcessorTag(), equalTo(simulateProcessorResult.getProcessorTag())); if (isFailure) { diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java index 3f252c37072..00327603ba8 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java @@ -35,7 +35,7 @@ public class WritePipelineResponseTests extends ESTestCase { response = new WritePipelineResponse(isAcknowledged); BytesStreamOutput out = new BytesStreamOutput(); response.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); WritePipelineResponse otherResponse = new WritePipelineResponse(); otherResponse.readFrom(streamInput); @@ -46,7 +46,7 @@ public class WritePipelineResponseTests extends ESTestCase { WritePipelineResponse response = new WritePipelineResponse(); BytesStreamOutput out = new BytesStreamOutput(); response.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); WritePipelineResponse otherResponse = new WritePipelineResponse(); otherResponse.readFrom(streamInput); diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index a7ce842913d..b4908846e97 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -112,7 +112,7 @@ public class WriteableIngestDocumentTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); writeableIngestDocument.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); WriteableIngestDocument otherWriteableIngestDocument = new WriteableIngestDocument(streamInput); assertIngestDocument(otherWriteableIngestDocument.getIngestDocument(), writeableIngestDocument.getIngestDocument()); } diff --git a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java index 2bff71d3c40..a8c550e01c5 100644 --- a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java @@ -64,7 +64,7 @@ public class MainActionTests extends ESTestCase { BytesStreamOutput streamOutput = new BytesStreamOutput(); mainResponse.writeTo(streamOutput); final MainResponse serialized = new MainResponse(); - serialized.readFrom(new ByteBufferStreamInput(ByteBuffer.wrap(streamOutput.bytes().toBytes()))); + serialized.readFrom(streamOutput.bytes().streamInput()); assertThat(serialized.getNodeName(), equalTo(nodeName)); assertThat(serialized.getClusterName(), equalTo(clusterName)); @@ -121,7 +121,7 @@ public class MainActionTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { logger.error("unexpected error", e); } }); diff --git a/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java index d5ed5302b97..d656e0f62a9 100644 --- a/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java @@ -38,7 +38,7 @@ public class IndicesOptionsTests extends ESTestCase { output.setVersion(outputVersion); indicesOptions.writeIndicesOptions(output); - StreamInput streamInput = StreamInput.wrap(output.bytes()); + StreamInput streamInput = output.bytes().streamInput(); streamInput.setVersion(randomVersion(random())); IndicesOptions indicesOptions2 = IndicesOptions.readIndicesOptions(streamInput); diff --git a/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java b/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java index 80492f0be61..8169a674bed 100644 --- a/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java @@ -45,15 +45,15 @@ public class ListenableActionFutureTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { error.set(e); listenerCalled.countDown(); } }); Thread networkThread = new Thread(new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - error.set(t); + public void onFailure(Exception e) { + error.set(e); listenerCalled.countDown(); } diff --git a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java index 00068c05efe..bbf1d2f1942 100644 --- a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java @@ -41,6 +41,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.IntStream; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -102,8 +104,8 @@ public class TransportActionFilterChainTests extends ESTestCase { try { assertThat(future.get(), notNullValue()); assertThat("shouldn't get here if an error is expected", errorExpected, equalTo(false)); - } catch(Throwable t) { - assertThat("shouldn't get here if an error is not expected " + t.getMessage(), errorExpected, equalTo(true)); + } catch (ExecutionException e) { + assertThat("shouldn't get here if an error is not expected " + e.getMessage(), errorExpected, equalTo(true)); } List testFiltersByLastExecution = new ArrayList<>(); @@ -182,8 +184,8 @@ public class TransportActionFilterChainTests extends ESTestCase { try { assertThat(future.get(), notNullValue()); assertThat("shouldn't get here if an error is expected", errorExpected, equalTo(false)); - } catch(Throwable t) { - assertThat("shouldn't get here if an error is not expected " + t.getMessage(), errorExpected, equalTo(true)); + } catch(ExecutionException e) { + assertThat("shouldn't get here if an error is not expected " + e.getMessage(), errorExpected, equalTo(true)); } List testFiltersByLastExecution = new ArrayList<>(); @@ -252,7 +254,7 @@ public class TransportActionFilterChainTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { failures.add(e); latch.countDown(); } @@ -309,7 +311,7 @@ public class TransportActionFilterChainTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { failures.add(e); latch.countDown(); } @@ -445,12 +447,12 @@ public class TransportActionFilterChainTests extends ESTestCase { } } - private static interface RequestCallback { + private interface RequestCallback { , Response extends ActionResponse> void execute(Task task, String action, Request request, ActionListener listener, ActionFilterChain actionFilterChain); } - private static interface ResponseCallback { + private interface ResponseCallback { void execute(String action, Response response, ActionListener listener, ActionFilterChain chain); } diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index 1d65f277e3c..603ad664ec3 100644 --- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -49,7 +49,7 @@ import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; @@ -247,7 +247,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { } static DiscoveryNode newNode(int nodeId) { - return new DiscoveryNode("node_" + nodeId, DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + return new DiscoveryNode("node_" + nodeId, LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); } @AfterClass @@ -491,7 +491,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase { } @Override - public void sendResponse(Throwable error) throws IOException { + public void sendResponse(Exception exception) throws IOException { } @Override diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java index e15c869c6ff..5d01aa369b0 100644 --- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java @@ -37,7 +37,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.MasterNotDiscoveredException; @@ -89,9 +89,9 @@ public class TransportMasterNodeActionTests extends ESTestCase { transportService = new TransportService(clusterService.getSettings(), transport, threadPool); transportService.start(); transportService.acceptIncomingRequests(); - localNode = new DiscoveryNode("local_node", DummyTransportAddress.INSTANCE, Collections.emptyMap(), + localNode = new DiscoveryNode("local_node", LocalTransportAddress.buildUnique(), Collections.emptyMap(), Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT); - remoteNode = new DiscoveryNode("remote_node", DummyTransportAddress.INSTANCE, Collections.emptyMap(), + remoteNode = new DiscoveryNode("remote_node", LocalTransportAddress.buildUnique(), Collections.emptyMap(), Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT); allNodes = new DiscoveryNode[]{localNode, remoteNode}; } @@ -136,7 +136,7 @@ public class TransportMasterNodeActionTests extends ESTestCase { @Override protected void doExecute(Task task, final Request request, ActionListener listener) { // remove unneeded threading by wrapping listener with SAME to prevent super.doExecute from wrapping it with LISTENER - super.doExecute(task, request, new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.SAME, listener)); + super.doExecute(task, request, new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.SAME, listener, false)); } @Override @@ -167,7 +167,7 @@ public class TransportMasterNodeActionTests extends ESTestCase { Request request = new Request(); PlainActionFuture listener = new PlainActionFuture<>(); - final Throwable exception = new Throwable(); + final Exception exception = new Exception(); final Response response = new Response(); setState(clusterService, ClusterStateCreationUtils.state(localNode, localNode, allNodes)); @@ -342,7 +342,7 @@ public class TransportMasterNodeActionTests extends ESTestCase { protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { // The other node has become master, simulate failures of this node while publishing cluster state through ZenDiscovery setState(clusterService, ClusterStateCreationUtils.state(localNode, remoteNode, allNodes)); - Throwable failure = randomBoolean() + Exception failure = randomBoolean() ? new Discovery.FailedToCommitClusterStateException("Fake error") : new NotMasterException("Fake error"); listener.onFailure(failure); diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java index a15f89bced4..ae8ea4a0b95 100644 --- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java @@ -32,7 +32,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.CapturingTransport; import org.elasticsearch.threadpool.TestThreadPool; @@ -236,7 +236,7 @@ public class TransportNodesActionTests extends ESTestCase { private static DiscoveryNode newNode(int nodeId, Map attributes, Set roles) { String node = "node_" + nodeId; - return new DiscoveryNode(node, node, DummyTransportAddress.INSTANCE, attributes, roles, Version.CURRENT); + return new DiscoveryNode(node, node, LocalTransportAddress.buildUnique(), attributes, roles, Version.CURRENT); } private static class TestTransportNodesAction diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index dc40fda3f8e..7496bb85faf 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -34,7 +34,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.shard.ShardId; import java.util.Arrays; @@ -220,7 +220,6 @@ public class ClusterStateCreationUtils { * Creates a cluster state with no index */ public static ClusterState stateWithNoShard() { - int numberOfNodes = 2; DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); discoBuilder.localNodeId(newNode(0).getId()); discoBuilder.masterNodeId(newNode(1).getId()); @@ -256,11 +255,11 @@ public class ClusterStateCreationUtils { } private static DiscoveryNode newNode(int nodeId) { - return new DiscoveryNode("node_" + nodeId, DummyTransportAddress.INSTANCE, Collections.emptyMap(), + return new DiscoveryNode("node_" + nodeId, LocalTransportAddress.buildUnique(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); } - static private String selectAndRemove(Set strings) { + private static String selectAndRemove(Set strings) { String selection = randomFrom(strings.toArray(new String[strings.size()])); strings.remove(selection); return selection; diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java index 55e2a9d3cf2..9f41f0e37c2 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java @@ -81,11 +81,11 @@ public class ReplicationOperationTests extends ESTestCase { final Set expectedReplicas = getExpectedReplicas(shardId, state); - final Map expectedFailures = new HashMap<>(); + final Map expectedFailures = new HashMap<>(); final Set expectedFailedShards = new HashSet<>(); for (ShardRouting replica : expectedReplicas) { if (randomBoolean()) { - Throwable t; + Exception t; boolean criticalFailure = randomBoolean(); if (criticalFailure) { t = new CorruptIndexException("simulated", (String) null); @@ -166,7 +166,7 @@ public class ReplicationOperationTests extends ESTestCase { final Set expectedReplicas = getExpectedReplicas(shardId, state); - final Map expectedFailures = new HashMap<>(); + final Map expectedFailures = new HashMap<>(); final ShardRouting failedReplica = randomFrom(new ArrayList<>(expectedReplicas)); expectedFailures.put(failedReplica, new CorruptIndexException("simulated", (String) null)); @@ -175,9 +175,9 @@ public class ReplicationOperationTests extends ESTestCase { final ClusterState finalState = state; final TestReplicaProxy replicasProxy = new TestReplicaProxy(expectedFailures) { @Override - public void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable, - Runnable onSuccess, Consumer onPrimaryDemoted, - Consumer onIgnoredFailure) { + public void failShard(ShardRouting replica, ShardRouting primary, String message, Exception exception, + Runnable onSuccess, Consumer onPrimaryDemoted, + Consumer onIgnoredFailure) { assertThat(replica, equalTo(failedReplica)); onPrimaryDemoted.accept(new ElasticsearchException("the king is dead")); } @@ -185,7 +185,7 @@ public class ReplicationOperationTests extends ESTestCase { AtomicBoolean primaryFailed = new AtomicBoolean(); final TestPrimary primary = new TestPrimary(primaryShard, primaryTerm) { @Override - public void failShard(String message, Throwable throwable) { + public void failShard(String message, Exception exception) { assertTrue(primaryFailed.compareAndSet(false, true)); } }; @@ -376,8 +376,8 @@ public class ReplicationOperationTests extends ESTestCase { } @Override - public void failShard(String message, Throwable throwable) { - throw new AssertionError("should shouldn't be failed with [" + message + "]", throwable); + public void failShard(String message, Exception exception) { + throw new AssertionError("should shouldn't be failed with [" + message + "]", exception); } @Override @@ -415,7 +415,7 @@ public class ReplicationOperationTests extends ESTestCase { static class TestReplicaProxy implements ReplicationOperation.Replicas { - final Map opFailures; + final Map opFailures; final Set failedReplicas = ConcurrentCollections.newConcurrentSet(); @@ -423,7 +423,7 @@ public class ReplicationOperationTests extends ESTestCase { this(Collections.emptyMap()); } - TestReplicaProxy(Map opFailures) { + TestReplicaProxy(Map opFailures) { this.opFailures = opFailures; } @@ -438,8 +438,8 @@ public class ReplicationOperationTests extends ESTestCase { } @Override - public void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable, Runnable onSuccess, - Consumer onPrimaryDemoted, Consumer onIgnoredFailure) { + public void failShard(ShardRouting replica, ShardRouting primary, String message, Exception exception, Runnable onSuccess, + Consumer onPrimaryDemoted, Consumer onIgnoredFailure) { if (failedReplicas.add(replica) == false) { fail("replica [" + replica + "] was failed twice"); } diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 229e3b6635e..de2ddabb0fe 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -390,7 +390,15 @@ public class TransportReplicationActionTests extends ESTestCase { PlainActionFuture listener = new PlainActionFuture<>(); ReplicationTask task = maybeTask(); AtomicBoolean executed = new AtomicBoolean(); - Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() { + + ShardRouting primaryShard = state.getRoutingTable().shardRoutingTable(shardId).primaryShard(); + boolean executeOnPrimary = true; + // whether shard has been marked as relocated already (i.e. relocation completed) + if (primaryShard.relocating() && randomBoolean()) { + isRelocated.set(true); + executeOnPrimary = false; + } + action.new AsyncPrimaryAction(request, createTransportChannel(listener), task) { @Override protected ReplicationOperation createReplicatedOperation(Request request, ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, @@ -403,15 +411,7 @@ public class TransportReplicationActionTests extends ESTestCase { } }; } - }; - ShardRouting primaryShard = state.getRoutingTable().shardRoutingTable(shardId).primaryShard(); - boolean executeOnPrimary = true; - // whether shard has been marked as relocated already (i.e. relocation completed) - if (primaryShard.relocating() && randomBoolean()) { - isRelocated.set(true); - executeOnPrimary = false; - } - primaryPhase.messageReceived(request, createTransportChannel(listener), task); + }.run(); if (executeOnPrimary) { assertTrue(executed.get()); assertTrue(listener.isDone()); @@ -445,7 +445,7 @@ public class TransportReplicationActionTests extends ESTestCase { PlainActionFuture listener = new PlainActionFuture<>(); ReplicationTask task = maybeTask(); AtomicBoolean executed = new AtomicBoolean(); - Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() { + action.new AsyncPrimaryAction(request, createTransportChannel(listener), task) { @Override protected ReplicationOperation createReplicatedOperation(Request request, ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, @@ -458,8 +458,7 @@ public class TransportReplicationActionTests extends ESTestCase { } }; } - }; - primaryPhase.messageReceived(request, createTransportChannel(listener), task); + }.run(); assertThat(executed.get(), equalTo(true)); assertPhase(task, "finished"); } @@ -579,16 +578,18 @@ public class TransportReplicationActionTests extends ESTestCase { metaData.put(IndexMetaData.builder(metaData.get(index)).settings(settings)); state = ClusterState.builder(state).metaData(metaData).build(); setState(clusterService, state); - Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() { + AtomicBoolean executed = new AtomicBoolean(); + action.new AsyncPrimaryAction(new Request(shardId), createTransportChannel(new PlainActionFuture<>()), null) { @Override protected ReplicationOperation createReplicatedOperation(Request request, ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, boolean executeOnReplicas) { assertFalse(executeOnReplicas); + assertFalse(executed.getAndSet(true)); return new NoopReplicationOperation(request, actionListener); } - }; - primaryPhase.messageReceived(new Request(shardId), createTransportChannel(new PlainActionFuture<>()), null); + }.run(); + assertThat(executed.get(), equalTo(true)); } public void testCounterOnPrimary() throws Exception { @@ -604,17 +605,16 @@ public class TransportReplicationActionTests extends ESTestCase { final boolean throwExceptionOnCreation = i == 1; final boolean throwExceptionOnRun = i == 2; final boolean respondWithError = i == 3; - Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() { - + action.new AsyncPrimaryAction(request, createTransportChannel(listener), task) { @Override protected ReplicationOperation createReplicatedOperation(Request request, - ActionListener listener, Action.PrimaryShardReference primaryShardReference, + ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, boolean executeOnReplicas) { assertIndexShardCounter(1); if (throwExceptionOnCreation) { throw new ElasticsearchException("simulated exception, during createReplicatedOperation"); } - return new NoopReplicationOperation(request, listener) { + return new NoopReplicationOperation(request, actionListener) { @Override public void execute() throws Exception { assertIndexShardCounter(1); @@ -629,18 +629,7 @@ public class TransportReplicationActionTests extends ESTestCase { } }; } - }; - try { - primaryPhase.messageReceived(request, createTransportChannel(listener), task); - } catch (ElasticsearchException e) { - if (throwExceptionOnCreation || throwExceptionOnRun) { - assertThat(e.getMessage(), containsString("simulated")); - assertIndexShardCounter(0); - return; // early terminate - } else { - throw e; - } - } + }.run(); assertIndexShardCounter(0); assertTrue(listener.isDone()); assertPhase(task, "finished"); @@ -648,7 +637,7 @@ public class TransportReplicationActionTests extends ESTestCase { try { listener.get(); } catch (ExecutionException e) { - if (respondWithError) { + if (throwExceptionOnCreation || throwExceptionOnRun || respondWithError) { Throwable cause = e.getCause(); assertThat(cause, instanceOf(ElasticsearchException.class)); assertThat(cause.getMessage(), containsString("simulated")); @@ -787,16 +776,16 @@ public class TransportReplicationActionTests extends ESTestCase { } @Override - protected PrimaryShardReference getPrimaryShardReference(ShardId shardId) { + protected void acquirePrimaryShardReference(ShardId shardId, ActionListener onReferenceAcquired) { count.incrementAndGet(); - return new PrimaryShardReference(null, null) { + PrimaryShardReference primaryShardReference = new PrimaryShardReference(null, null) { @Override public boolean isRelocated() { return isRelocated.get(); } @Override - public void failShard(String reason, @Nullable Throwable e) { + public void failShard(String reason, @Nullable Exception e) { throw new UnsupportedOperationException(); } @@ -812,13 +801,15 @@ public class TransportReplicationActionTests extends ESTestCase { public void close() { count.decrementAndGet(); } - }; + + onReferenceAcquired.onResponse(primaryShardReference); } - protected Releasable acquireReplicaOperationLock(ShardId shardId, long primaryTerm) { + @Override + protected void acquireReplicaOperationLock(ShardId shardId, long primaryTerm, ActionListener onLockAcquired) { count.incrementAndGet(); - return count::decrementAndGet; + onLockAcquired.onResponse(count::decrementAndGet); } } @@ -865,9 +856,9 @@ public class TransportReplicationActionTests extends ESTestCase { } @Override - public void sendResponse(Throwable error) throws IOException { - consumer.accept(error); - listener.onFailure(error); + public void sendResponse(Exception exception) throws IOException { + consumer.accept(exception); + listener.onFailure(exception); } @Override diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index 7b312959631..80e689743fd 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -179,7 +179,7 @@ public class TransportWriteActionTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { throw new RuntimeException(e); } } diff --git a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java index c26d376b587..37abc4d5eed 100644 --- a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java @@ -179,9 +179,9 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase { action.new AsyncSingleAction(request, listener).start(); listener.get(); fail("expected ClusterBlockException"); - } catch (Throwable t) { - if (ExceptionsHelper.unwrap(t, ClusterBlockException.class) == null) { - logger.info("expected ClusterBlockException but got ", t); + } catch (Exception e) { + if (ExceptionsHelper.unwrap(e, ClusterBlockException.class) == null) { + logger.info("expected ClusterBlockException but got ", e); fail("expected ClusterBlockException"); } } @@ -317,9 +317,9 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase { assertThat(transport.capturedRequests().length, equalTo(0)); try { listener.get(); - } catch (Throwable t) { - if (ExceptionsHelper.unwrap(t, IllegalStateException.class) == null) { - logger.info("expected IllegalStateException but got ", t); + } catch (Exception e) { + if (ExceptionsHelper.unwrap(e, IllegalStateException.class) == null) { + logger.info("expected IllegalStateException but got ", e); fail("expected and IllegalStateException"); } } diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java b/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java index 208945a6179..d9f351120b2 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java @@ -68,10 +68,10 @@ import static org.hamcrest.Matchers.equalTo; public abstract class AbstractTermVectorsTestCase extends ESIntegTestCase { protected static class TestFieldSetting { - final public String name; - final public boolean storedOffset; - final public boolean storedPayloads; - final public boolean storedPositions; + public final String name; + public final boolean storedOffset; + public final boolean storedPayloads; + public final boolean storedPositions; public TestFieldSetting(String name, boolean storedOffset, boolean storedPayloads, boolean storedPositions) { this.name = name; @@ -124,9 +124,9 @@ public abstract class AbstractTermVectorsTestCase extends ESIntegTestCase { } protected static class TestDoc { - final public String id; - final public TestFieldSetting[] fieldSettings; - final public String[] fieldContent; + public final String id; + public final TestFieldSetting[] fieldSettings; + public final String[] fieldContent; public String index = "test"; public String alias = "alias"; public String type = "type1"; @@ -163,11 +163,11 @@ public abstract class AbstractTermVectorsTestCase extends ESIntegTestCase { } protected static class TestConfig { - final public TestDoc doc; - final public String[] selectedFields; - final public boolean requestPositions; - final public boolean requestOffsets; - final public boolean requestPayloads; + public final TestDoc doc; + public final String[] selectedFields; + public final boolean requestPositions; + public final boolean requestOffsets; + public final boolean requestPayloads; public Class expectedException = null; public TestConfig(TestDoc doc, String[] selectedFields, boolean requestPositions, boolean requestOffsets, boolean requestPayloads) { diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java index 37a1bc92e9c..1611c63d2ba 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java @@ -140,7 +140,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, null); xBuilder.endObject(); - String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; @@ -196,7 +196,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, null); xBuilder.endObject(); - String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; @@ -255,7 +255,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, ToXContent.EMPTY_PARAMS); xBuilder.endObject(); - String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java index 12af9f8a2c2..d8fd7916b5a 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java @@ -39,7 +39,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.mapper.FieldMapper; -import org.hamcrest.Matcher; import java.io.IOException; import java.util.ArrayList; @@ -55,7 +54,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -391,19 +389,15 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { TestConfig[] testConfigs = generateTestConfigs(20, testDocs, testFieldSettings); for (TestConfig test : testConfigs) { - try { - TermVectorsRequestBuilder request = getRequestForConfig(test); - if (test.expectedException != null) { - assertThrows(request, test.expectedException); - continue; - } - - TermVectorsResponse response = request.get(); - Fields luceneTermVectors = getTermVectorsFromLucene(directoryReader, test.doc); - validateResponse(response, luceneTermVectors, test); - } catch (Throwable t) { - throw new Exception("Test exception while running " + test.toString(), t); + TermVectorsRequestBuilder request = getRequestForConfig(test); + if (test.expectedException != null) { + assertThrows(request, test.expectedException); + continue; } + + TermVectorsResponse response = request.get(); + Fields luceneTermVectors = getTermVectorsFromLucene(directoryReader, test.doc); + validateResponse(response, luceneTermVectors, test); } } @@ -963,21 +957,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { return randomBoolean() ? "test" : "alias"; } - private Map getFieldStatistics(Map stats, String fieldName) throws IOException { - return (Map) ((Map) stats.get(fieldName)).get("field_statistics"); - } - - private Map getTermStatistics(Map stats, String fieldName, String term) { - return (Map) ((Map) ((Map) stats.get(fieldName)).get("terms")).get(term); - } - - private Matcher equalOrLessThanTo(Integer value, boolean isEqual) { - if (isEqual) { - return equalTo(value); - } - return lessThan(value); - } - public void testTermVectorsWithVersion() { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(Settings.builder().put("index.refresh_interval", -1))); diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java index 57a89c82cc8..5ed4f3252d5 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java @@ -56,21 +56,16 @@ public class MultiTermVectorsIT extends AbstractTermVectorsTestCase { for (int i = 0; i < testConfigs.length; i++) { TestConfig test = testConfigs[i]; - try { - MultiTermVectorsItemResponse item = responseItems[i]; - if (test.expectedException != null) { - assertTrue(item.isFailed()); - continue; - } else if (item.isFailed()) { - fail(item.getFailure().getCause().getMessage()); - } - Fields luceneTermVectors = getTermVectorsFromLucene(directoryReader, test.doc); - validateResponse(item.getResponse(), luceneTermVectors, test); - } catch (Throwable t) { - throw new Exception("Test exception while running " + test.toString(), t); + MultiTermVectorsItemResponse item = responseItems[i]; + if (test.expectedException != null) { + assertTrue(item.isFailed()); + continue; + } else if (item.isFailed()) { + fail(item.getFailure().getCause().getMessage()); } + Fields luceneTermVectors = getTermVectorsFromLucene(directoryReader, test.doc); + validateResponse(item.getResponse(), luceneTermVectors, test); } - } public void testMissingIndexThrowsMissingIndex() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index 597a2a4db39..d105a4bf63b 100644 --- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -135,7 +135,7 @@ public class UpdateRequestTests extends ESTestCase { TimeValue providedTTLValue = TimeValue.parseTimeValue(randomTimeValue(), null, "ttl"); Settings settings = settings(Version.CURRENT).build(); - UpdateHelper updateHelper = new UpdateHelper(settings, null, null); + UpdateHelper updateHelper = new UpdateHelper(settings, null); // We just upsert one document with ttl IndexRequest indexRequest = new IndexRequest("test", "type1", "1") diff --git a/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java b/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java index ffe82f9388d..699b919cf05 100644 --- a/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java @@ -141,15 +141,6 @@ public class SimpleBlocksIT extends ESIntegTestCase { } } - private void canNotIndexExists(String index) { - try { - IndicesExistsResponse r = client().admin().indices().prepareExists(index).execute().actionGet(); - fail(); - } catch (ClusterBlockException e) { - // all is well - } - } - private void setIndexReadOnly(String index, Object value) { HashMap newSettings = new HashMap<>(); newSettings.put(IndexMetaData.SETTING_READ_ONLY, value); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java index 40995ff778b..be0848ed4dc 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java @@ -41,7 +41,7 @@ import static org.hamcrest.Matchers.equalTo; public class BasicAnalysisBackwardCompatibilityIT extends ESBackcompatTestCase { // This pattern match characters with Line_Break = Complex_Content. - final static Pattern complexUnicodeChars = Pattern.compile("[\u17B4\u17B5\u17D3\u17CB-\u17D1\u17DD\u1036\u17C6\u1A74\u1038\u17C7\u0E4E\u0E47-\u0E4D\u0EC8-\u0ECD\uAABF\uAAC1\u1037\u17C8-\u17CA\u1A75-\u1A7C\u1AA8-\u1AAB\uAADE\uAADF\u1AA0-\u1AA6\u1AAC\u1AAD\u109E\u109F\uAA77-\uAA79\u0E46\u0EC6\u17D7\u1AA7\uA9E6\uAA70\uAADD\u19DA\u0E01-\u0E3A\u0E40-\u0E45\u0EDE\u0E81\u0E82\u0E84\u0E87\u0E88\u0EAA\u0E8A\u0EDF\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAB\u0EDC\u0EDD\u0EAD-\u0EB9\u0EBB-\u0EBD\u0EC0-\u0EC4\uAA80-\uAABE\uAAC0\uAAC2\uAADB\uAADC\u1000\u1075\u1001\u1076\u1002\u1077\uAA60\uA9E9\u1003\uA9E0\uA9EA\u1004\u105A\u1005\u1078\uAA61\u1006\uA9E1\uAA62\uAA7E\u1007\uAA63\uA9EB\u1079\uAA72\u1008\u105B\uA9E2\uAA64\uA9EC\u1061\uAA7F\u1009\u107A\uAA65\uA9E7\u100A\u100B\uAA66\u100C\uAA67\u100D\uAA68\uA9ED\u100E\uAA69\uA9EE\u100F\u106E\uA9E3\uA9EF\u1010-\u1012\u107B\uA9FB\u1013\uAA6A\uA9FC\u1014\u107C\uAA6B\u105E\u1015\u1016\u107D\u107E\uAA6F\u108E\uA9E8\u1017\u107F\uA9FD\u1018\uA9E4\uA9FE\u1019\u105F\u101A\u103B\u101B\uAA73\uAA7A\u103C\u101C\u1060\u101D\u103D\u1082\u1080\u1050\u1051\u1065\u101E\u103F\uAA6C\u101F\u1081\uAA6D\u103E\uAA6E\uAA71\u1020\uA9FA\u105C\u105D\u106F\u1070\u1066\u1021-\u1026\u1052-\u1055\u1027-\u102A\u102C\u102B\u1083\u1072\u109C\u102D\u1071\u102E\u1033\u102F\u1073\u1074\u1030\u1056-\u1059\u1031\u1084\u1035\u1085\u1032\u109D\u1034\u1062\u1067\u1068\uA9E5\u1086\u1039\u103A\u1063\u1064\u1069-\u106D\u1087\u108B\u1088\u108C\u108D\u1089\u108A\u108F\u109A\u109B\uAA7B-\uAA7D\uAA74-\uAA76\u1780-\u17A2\u17DC\u17A3-\u17B3\u17B6-\u17C5\u17D2\u1950-\u196D\u1970-\u1974\u1980-\u199C\u19DE\u19DF\u199D-\u19AB\u19B0-\u19C9\u1A20-\u1A26\u1A58\u1A59\u1A27-\u1A3B\u1A5A\u1A5B\u1A3C-\u1A46\u1A54\u1A47-\u1A4C\u1A53\u1A6B\u1A55-\u1A57\u1A5C-\u1A5E\u1A4D-\u1A52\u1A61\u1A6C\u1A62-\u1A6A\u1A6E\u1A6F\u1A73\u1A70-\u1A72\u1A6D\u1A60]"); + static final Pattern complexUnicodeChars = Pattern.compile("[\u17B4\u17B5\u17D3\u17CB-\u17D1\u17DD\u1036\u17C6\u1A74\u1038\u17C7\u0E4E\u0E47-\u0E4D\u0EC8-\u0ECD\uAABF\uAAC1\u1037\u17C8-\u17CA\u1A75-\u1A7C\u1AA8-\u1AAB\uAADE\uAADF\u1AA0-\u1AA6\u1AAC\u1AAD\u109E\u109F\uAA77-\uAA79\u0E46\u0EC6\u17D7\u1AA7\uA9E6\uAA70\uAADD\u19DA\u0E01-\u0E3A\u0E40-\u0E45\u0EDE\u0E81\u0E82\u0E84\u0E87\u0E88\u0EAA\u0E8A\u0EDF\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAB\u0EDC\u0EDD\u0EAD-\u0EB9\u0EBB-\u0EBD\u0EC0-\u0EC4\uAA80-\uAABE\uAAC0\uAAC2\uAADB\uAADC\u1000\u1075\u1001\u1076\u1002\u1077\uAA60\uA9E9\u1003\uA9E0\uA9EA\u1004\u105A\u1005\u1078\uAA61\u1006\uA9E1\uAA62\uAA7E\u1007\uAA63\uA9EB\u1079\uAA72\u1008\u105B\uA9E2\uAA64\uA9EC\u1061\uAA7F\u1009\u107A\uAA65\uA9E7\u100A\u100B\uAA66\u100C\uAA67\u100D\uAA68\uA9ED\u100E\uAA69\uA9EE\u100F\u106E\uA9E3\uA9EF\u1010-\u1012\u107B\uA9FB\u1013\uAA6A\uA9FC\u1014\u107C\uAA6B\u105E\u1015\u1016\u107D\u107E\uAA6F\u108E\uA9E8\u1017\u107F\uA9FD\u1018\uA9E4\uA9FE\u1019\u105F\u101A\u103B\u101B\uAA73\uAA7A\u103C\u101C\u1060\u101D\u103D\u1082\u1080\u1050\u1051\u1065\u101E\u103F\uAA6C\u101F\u1081\uAA6D\u103E\uAA6E\uAA71\u1020\uA9FA\u105C\u105D\u106F\u1070\u1066\u1021-\u1026\u1052-\u1055\u1027-\u102A\u102C\u102B\u1083\u1072\u109C\u102D\u1071\u102E\u1033\u102F\u1073\u1074\u1030\u1056-\u1059\u1031\u1084\u1035\u1085\u1032\u109D\u1034\u1062\u1067\u1068\uA9E5\u1086\u1039\u103A\u1063\u1064\u1069-\u106D\u1087\u108B\u1088\u108C\u108D\u1089\u108A\u108F\u109A\u109B\uAA7B-\uAA7D\uAA74-\uAA76\u1780-\u17A2\u17DC\u17A3-\u17B3\u17B6-\u17C5\u17D2\u1950-\u196D\u1970-\u1974\u1980-\u199C\u19DE\u19DF\u199D-\u19AB\u19B0-\u19C9\u1A20-\u1A26\u1A58\u1A59\u1A27-\u1A3B\u1A5A\u1A5B\u1A3C-\u1A46\u1A54\u1A47-\u1A4C\u1A53\u1A6B\u1A55-\u1A57\u1A5C-\u1A5E\u1A4D-\u1A52\u1A61\u1A6C\u1A62-\u1A6A\u1A6E\u1A6F\u1A73\u1A70-\u1A72\u1A6D\u1A60]"); /** * Simple upgrade test for analyzers to make sure they analyze to the same tokens after upgrade diff --git a/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java index c9d5f0b622e..4601c1bcfcf 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java @@ -22,7 +22,6 @@ package org.elasticsearch.bwcompat; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESBackcompatTestCase; @@ -46,7 +45,7 @@ public class NodesStatsBasicBackwardsCompatIT extends ESBackcompatTestCase { for (NodeInfo n : nodesInfo.getNodes()) { TransportClient tc = TransportClient.builder().settings(settings).build().addTransportAddress(n.getNode().getAddress()); // Just verify that the NS can be sent and serialized/deserialized between nodes with basic indices - NodesStatsResponse ns = tc.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet(); + tc.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet(); tc.close(); } } @@ -78,7 +77,7 @@ public class NodesStatsBasicBackwardsCompatIT extends ESBackcompatTestCase { method.invoke(nsBuilder); } } - NodesStatsResponse ns = nsBuilder.execute().actionGet(); + nsBuilder.execute().actionGet(); tc.close(); } diff --git a/core/src/test/java/org/elasticsearch/cli/CommandTests.java b/core/src/test/java/org/elasticsearch/cli/CommandTests.java index 1f50ad4c13b..376c883c074 100644 --- a/core/src/test/java/org/elasticsearch/cli/CommandTests.java +++ b/core/src/test/java/org/elasticsearch/cli/CommandTests.java @@ -30,7 +30,7 @@ public class CommandTests extends ESTestCase { } @Override protected void execute(Terminal terminal, OptionSet options) throws Exception { - throw new UserError(ExitCodes.DATA_ERROR, "Bad input"); + throw new UserException(ExitCodes.DATA_ERROR, "Bad input"); } } @@ -40,7 +40,7 @@ public class CommandTests extends ESTestCase { } @Override protected void execute(Terminal terminal, OptionSet options) throws Exception { - throw new UserError(ExitCodes.USAGE, "something was no good"); + throw new UserException(ExitCodes.USAGE, "something was no good"); } } @@ -87,7 +87,7 @@ public class CommandTests extends ESTestCase { MockTerminal terminal = new MockTerminal(); NoopCommand command = new NoopCommand(); String[] args = {"-v", "-s"}; - UserError e = expectThrows(UserError.class, () -> { + UserException e = expectThrows(UserException.class, () -> { command.mainWithoutErrorHandling(args, terminal); }); assertTrue(e.getMessage(), e.getMessage().contains("Cannot specify -s and -v together")); diff --git a/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java b/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java index 4f91d378440..f4680492028 100644 --- a/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java +++ b/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java @@ -61,7 +61,7 @@ public class MultiCommandTests extends CommandTestCase { public void testUnknownCommand() throws Exception { multiCommand.subcommands.put("something", new DummySubCommand()); - UserError e = expectThrows(UserError.class, () -> { + UserException e = expectThrows(UserException.class, () -> { execute("somethingelse"); }); assertEquals(ExitCodes.USAGE, e.exitCode); @@ -70,7 +70,7 @@ public class MultiCommandTests extends CommandTestCase { public void testMissingCommand() throws Exception { multiCommand.subcommands.put("command1", new DummySubCommand()); - UserError e = expectThrows(UserError.class, () -> { + UserException e = expectThrows(UserException.class, () -> { execute(); }); assertEquals(ExitCodes.USAGE, e.exitCode); diff --git a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java index 196b053ff82..276a43581a6 100644 --- a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java +++ b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java @@ -179,7 +179,7 @@ public abstract class AbstractClientHeadersTestCase extends ESTestCase { } @Override - public void onFailure(Throwable t) { + public void onFailure(Exception t) { Throwable e = unwrap(t, InternalException.class); assertThat("expected action [" + action + "] to throw an internal exception", e, notNullValue()); assertThat(action, equalTo(((InternalException) e).action)); diff --git a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java index c085c3164a0..9d2c176dffb 100644 --- a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java +++ b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java @@ -187,19 +187,13 @@ abstract class FailAndRetryMockTransport imp } @Override - public Transport start() { - return null; - } + public void start() {} @Override - public Transport stop() { - return null; - } + public void stop() {} @Override - public void close() { - - } + public void close() {} @Override public Map profileBoundAddresses() { diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java index 5c07f5e6f25..3d50e2e44ac 100644 --- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java +++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.client.transport; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse; import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction; @@ -123,7 +122,7 @@ public class TransportClientNodesServiceTests extends ESTestCase { @SuppressWarnings("unchecked") public void handleResponse(T response) { LivenessResponse livenessResponse = new LivenessResponse(clusterName, - new DiscoveryNode(node.getName(), node.getId(), "liveness-hostname" + node.getId(), + new DiscoveryNode(node.getName(), node.getId(), node.getEphemeralId(), "liveness-hostname" + node.getId(), "liveness-hostaddress" + node.getId(), new LocalTransportAddress("liveness-address-" + node.getId()), node.getAttributes(), node.getRoles(), node.getVersion())); @@ -171,7 +170,7 @@ public class TransportClientNodesServiceTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { finalFailures.incrementAndGet(); finalFailure.set(e); latch.countDown(); diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java index 72748a59986..555f23813cb 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java @@ -30,7 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; @@ -320,7 +320,8 @@ public class ClusterChangedEventTests extends ESTestCase { // Create a new DiscoveryNode private static DiscoveryNode newNode(final String nodeId, Set roles) { - return new DiscoveryNode(nodeId, nodeId, DummyTransportAddress.INSTANCE, Collections.emptyMap(), roles, Version.CURRENT); + return new DiscoveryNode(nodeId, nodeId, nodeId, "host", "host_address", new LocalTransportAddress("_test_" + nodeId), + Collections.emptyMap(), roles, Version.CURRENT); } // Create the metadata for a cluster state. diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index b82b5e0ba60..68a0f73eb34 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.RepositoriesMetaData; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -52,6 +53,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.test.ESIntegTestCase; import java.util.Collections; @@ -121,7 +123,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase { Diff diffBeforeSerialization = clusterState.diff(previousClusterState); BytesStreamOutput os = new BytesStreamOutput(); diffBeforeSerialization.writeTo(os); - byte[] diffBytes = os.bytes().toBytes(); + byte[] diffBytes = BytesReference.toBytes(os.bytes()); Diff diff; try (StreamInput input = StreamInput.wrap(diffBytes)) { diff = previousClusterStateFromDiffs.readDiffFrom(input); @@ -190,9 +192,8 @@ public class ClusterStateDiffIT extends ESIntegTestCase { List nodeIds = randomSubsetOf(randomInt(clusterState.nodes().getNodes().size() - 1), clusterState.nodes().getNodes().keys().toArray(String.class)); for (String nodeId : nodeIds) { if (nodeId.startsWith("node-")) { + nodes.remove(nodeId); if (randomBoolean()) { - nodes.remove(nodeId); - } else { nodes.put(new DiscoveryNode(nodeId, new LocalTransportAddress(randomAsciiOfLength(10)), emptyMap(), emptySet(), randomVersion(random()))); } diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 5e272a27459..6b99e525cb2 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.test.ESTestCase; import static java.util.Collections.emptyMap; @@ -32,8 +32,9 @@ import static org.hamcrest.Matchers.equalTo; public class ClusterStateTests extends ESTestCase { public void testSupersedes() { - final DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); - final DiscoveryNode node2 = new DiscoveryNode("node2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + final Version version = Version.CURRENT; + final DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), version); + final DiscoveryNode node2 = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), version); final DiscoveryNodes nodes = DiscoveryNodes.builder().put(node1).put(node2).build(); ClusterName name = ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY); ClusterState noMaster1 = ClusterState.builder(name).version(randomInt(5)).nodes(nodes).build(); diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java index 9210e2a56bd..2073235af98 100644 --- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java @@ -23,8 +23,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.ShardStats; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -33,7 +31,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingHelper; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; @@ -200,11 +198,11 @@ public class DiskUsageTests extends ESTestCase { new FsInfo.Path("/most", "/dev/sda", 100, 90, 80), }; List nodeStats = Arrays.asList( - new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, + new NodeStats(new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0, null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null), - new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, + new NodeStats(new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0, null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null), - new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, + new NodeStats(new DiscoveryNode("node_3", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0, null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null) ); InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages); @@ -241,11 +239,11 @@ public class DiskUsageTests extends ESTestCase { new FsInfo.Path("/least", "/dev/sda", 10, -8, 0), }; List nodeStats = Arrays.asList( - new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, + new NodeStats(new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0, null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null), - new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, + new NodeStats(new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0, null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null), - new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0, + new NodeStats(new DiscoveryNode("node_3", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0, null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null) ); InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvailableUsages, newMostAvailableUsages); diff --git a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java index 61bb898acc2..aad2aa212a1 100644 --- a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java @@ -395,8 +395,8 @@ public class MinimumMasterNodesIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { - failure.set(t); + public void onFailure(String source, Exception e) { + failure.set(e); latch.countDown(); } }); diff --git a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java index 5eb5a34c44f..b0bc3ee2e4e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java @@ -26,7 +26,7 @@ import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.test.ESTestCase; @@ -64,7 +64,7 @@ public class NodeConnectionsServiceTests extends ESTestCase { List nodes = new ArrayList<>(); for (int i = randomIntBetween(20, 50); i > 0; i--) { Set roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))); - nodes.add(new DiscoveryNode("node_" + i, "" + i, DummyTransportAddress.INSTANCE, Collections.emptyMap(), + nodes.add(new DiscoveryNode("node_" + i, "" + i, LocalTransportAddress.buildUnique(), Collections.emptyMap(), roles, Version.CURRENT)); } return nodes; @@ -253,18 +253,12 @@ public class NodeConnectionsServiceTests extends ESTestCase { } @Override - public Transport start() { - return null; - } + public void start() {} @Override - public Transport stop() { - return null; - } + public void stop() {} @Override - public void close() { - - } + public void close() {} } } diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java index 1faac874114..d12b6b563b3 100644 --- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateTaskExecutor; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.node.DiscoveryNodeService; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -41,6 +40,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; @@ -305,7 +305,8 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa return randomSubsetOf(1, shards.toArray(new ShardRouting[0])).get(0); } else { return - TestShardRouting.newShardRouting(shardRouting.shardId(), DiscoveryNodeService.generateNodeId(Settings.EMPTY), randomBoolean(), randomFrom(ShardRoutingState.values())); + TestShardRouting.newShardRouting(shardRouting.shardId(), UUIDs.randomBase64UUID(random()), randomBoolean(), + randomFrom(ShardRoutingState.values())); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java index 7f0ac1a6e45..d387d6f7d43 100644 --- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java @@ -148,7 +148,7 @@ public class ShardStateActionTests extends ESTestCase { } @Override - public void onFailure(Throwable t) { + public void onFailure(Exception e) { success.set(false); latch.countDown(); assert false; @@ -196,7 +196,7 @@ public class ShardStateActionTests extends ESTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { success.set(false); latch.countDown(); assert false; @@ -245,9 +245,9 @@ public class ShardStateActionTests extends ESTestCase { } @Override - public void onFailure(Throwable t) { + public void onFailure(Exception e) { success.set(false); - throwable.set(t); + throwable.set(e); latch.countDown(); assert false; } @@ -281,7 +281,7 @@ public class ShardStateActionTests extends ESTestCase { } @Override - public void onFailure(Throwable t) { + public void onFailure(Exception e) { failure.set(true); } }); @@ -313,7 +313,7 @@ public class ShardStateActionTests extends ESTestCase { } @Override - public void onFailure(Throwable t) { + public void onFailure(Exception e) { success.set(false); latch.countDown(); assert false; @@ -348,8 +348,8 @@ public class ShardStateActionTests extends ESTestCase { } @Override - public void onFailure(Throwable t) { - failure.set(t); + public void onFailure(Exception e) { + failure.set(e); latch.countDown(); } }); @@ -401,7 +401,7 @@ public class ShardStateActionTests extends ESTestCase { } } - private Throwable getSimulatedFailure() { + private Exception getSimulatedFailure() { return new CorruptIndexException("simulated", (String) null); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java index 7f2d0828128..a7fe1b918c0 100644 --- a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java @@ -55,7 +55,7 @@ public class ClusterBlockTests extends ESTestCase { out.setVersion(version); clusterBlock.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); in.setVersion(version); ClusterBlock result = ClusterBlock.readClusterBlock(in); diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java index fd1e3e62466..8718c479216 100644 --- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java @@ -118,8 +118,8 @@ public class ClusterStateHealthTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { - logger.warn("unexpected failure", t); + public void onFailure(String source, Exception e) { + logger.warn("unexpected failure", e); } }); @@ -169,7 +169,7 @@ public class ClusterStateHealthTests extends ESTestCase { if (randomBoolean()) { BytesStreamOutput out = new BytesStreamOutput(); clusterStateHealth.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); clusterStateHealth = new ClusterStateHealth(in); } return clusterStateHealth; diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java index aec701052fb..8dd950ba8e6 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java @@ -60,8 +60,7 @@ public class IndexGraveyardTests extends ESTestCase { final IndexGraveyard graveyard = createRandom(); final BytesStreamOutput out = new BytesStreamOutput(); graveyard.writeTo(out); - final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes())); - assertThat(IndexGraveyard.fromStream(in), equalTo(graveyard)); + assertThat(IndexGraveyard.fromStream(out.bytes().streamInput()), equalTo(graveyard)); } public void testXContent() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index 0c9827587ea..5fef33be388 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -69,7 +69,7 @@ public class IndexMetaDataTests extends ESTestCase { final BytesStreamOutput out = new BytesStreamOutput(); metaData.writeTo(out); - IndexMetaData deserialized = IndexMetaData.PROTO.readFrom(StreamInput.wrap(out.bytes())); + IndexMetaData deserialized = IndexMetaData.PROTO.readFrom(out.bytes().streamInput()); assertEquals(metaData, deserialized); assertEquals(metaData.hashCode(), deserialized.hashCode()); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 04d27020273..ff31a72a5c6 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -34,7 +34,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.IndexAlreadyExistsException; import org.elasticsearch.indices.InvalidIndexNameException; @@ -181,7 +181,7 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase { } private DiscoveryNode newNode(String nodeId) { - return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(), + return new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), emptyMap(), Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), Version.CURRENT); } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index e1e3a39122c..cf040fb3c7f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.Version; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; @@ -34,7 +33,6 @@ import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.nio.ByteBuffer; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -185,8 +183,7 @@ public class MetaDataTests extends ESTestCase { final MetaData originalMeta = MetaData.builder().indexGraveyard(graveyard).build(); final BytesStreamOutput out = new BytesStreamOutput(); originalMeta.writeTo(out); - final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes())); - final MetaData fromStreamMeta = MetaData.PROTO.readFrom(in); + final MetaData fromStreamMeta = MetaData.PROTO.readFrom(out.bytes().streamInput()); assertThat(fromStreamMeta.indexGraveyard(), equalTo(fromStreamMeta.indexGraveyard())); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java index 38aa73a9935..59f058a95fb 100644 --- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java @@ -21,8 +21,8 @@ package org.elasticsearch.cluster.node; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -64,10 +64,11 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build(); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("name1", "id1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode node = new DiscoveryNode("name1", "id1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), + Version.CURRENT); assertThat(filters.match(node), equalTo(true)); - node = new DiscoveryNode("name2", "id2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + node = new DiscoveryNode("name2", "id2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); assertThat(filters.match(node), equalTo(false)); } @@ -77,10 +78,11 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build(); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("name1", "id1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode node = new DiscoveryNode("name1", "id1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), + Version.CURRENT); assertThat(filters.match(node), equalTo(true)); - node = new DiscoveryNode("name2", "id2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + node = new DiscoveryNode("name2", "id2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); assertThat(filters.match(node), equalTo(false)); } @@ -91,13 +93,14 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build()); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("name1", "id1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + final Version version = Version.CURRENT; + DiscoveryNode node = new DiscoveryNode("name1", "id1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), version); assertThat(filters.match(node), equalTo(true)); - node = new DiscoveryNode("name2", "id2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + node = new DiscoveryNode("name2", "id2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), version); assertThat(filters.match(node), equalTo(true)); - node = new DiscoveryNode("name3", "id3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + node = new DiscoveryNode("name3", "id3", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), version); assertThat(filters.match(node), equalTo(false)); } @@ -111,7 +114,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { Map attributes = new HashMap<>(); attributes.put("tag", "A"); attributes.put("group", "B"); - DiscoveryNode node = new DiscoveryNode("name1", "id1", DummyTransportAddress.INSTANCE, + DiscoveryNode node = new DiscoveryNode("name1", "id1", LocalTransportAddress.buildUnique(), attributes, emptySet(), Version.CURRENT); assertThat(filters.match(node), equalTo(true)); @@ -119,7 +122,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { attributes.put("tag", "A"); attributes.put("group", "B"); attributes.put("name", "X"); - node = new DiscoveryNode("name2", "id2", DummyTransportAddress.INSTANCE, + node = new DiscoveryNode("name2", "id2", LocalTransportAddress.buildUnique(), attributes, emptySet(), Version.CURRENT); assertThat(filters.match(node), equalTo(true)); @@ -127,11 +130,11 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { attributes.put("tag", "A"); attributes.put("group", "F"); attributes.put("name", "X"); - node = new DiscoveryNode("name3", "id3", DummyTransportAddress.INSTANCE, + node = new DiscoveryNode("name3", "id3", LocalTransportAddress.buildUnique(), attributes, emptySet(), Version.CURRENT); assertThat(filters.match(node), equalTo(false)); - node = new DiscoveryNode("name4", "id4", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + node = new DiscoveryNode("name4", "id4", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); assertThat(filters.match(node), equalTo(false)); } @@ -141,7 +144,8 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build(); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("name1", "id1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode node = new DiscoveryNode("name1", "id1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), + Version.CURRENT); assertThat(filters.match(node), equalTo(true)); } @@ -152,7 +156,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build()); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(AND, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); + DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); assertThat(filters.match(node), equalTo(true)); } @@ -163,7 +167,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build()); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(AND, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); + DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); assertThat(filters.match(node), equalTo(false)); } @@ -174,7 +178,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build()); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(AND, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); + DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); assertThat(filters.match(node), equalTo(false)); } @@ -185,7 +189,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build()); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); + DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); assertThat(filters.match(node), equalTo(true)); } @@ -196,7 +200,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build()); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); + DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); assertThat(filters.match(node), equalTo(true)); } @@ -207,7 +211,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build()); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(AND, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); + DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); assertThat(filters.match(node), equalTo(true)); } @@ -218,7 +222,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build()); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(AND, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); + DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); assertThat(filters.match(node), equalTo(false)); } @@ -229,7 +233,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build()); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); + DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); assertThat(filters.match(node), equalTo(true)); } @@ -240,7 +244,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase { .build()); DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings); - DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); + DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null); assertThat(filters.match(node), equalTo(true)); } diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java index fb38a428a76..e9f80426df6 100644 --- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java @@ -19,9 +19,9 @@ package org.elasticsearch.cluster.node; -import org.elasticsearch.Version; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -55,7 +55,8 @@ public class DiscoveryNodeServiceTests extends ESTestCase { } } DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(builder.build()); - DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(DummyTransportAddress.INSTANCE); + DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(LocalTransportAddress.buildUnique(), + () -> UUIDs.randomBase64UUID(random())); assertThat(discoveryNode.getRoles(), equalTo(selectedRoles)); assertThat(discoveryNode.getAttributes(), equalTo(expectedAttributes)); } @@ -77,7 +78,8 @@ public class DiscoveryNodeServiceTests extends ESTestCase { expectedAttributes.putAll(customAttributes); discoveryNodeService.addCustomAttributeProvider(() -> customAttributes); - DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(DummyTransportAddress.INSTANCE); + DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(LocalTransportAddress.buildUnique(), + () -> UUIDs.randomBase64UUID(random())); assertThat(discoveryNode.getAttributes(), equalTo(expectedAttributes)); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java index 984cd31b7a0..ec741a908c5 100644 --- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java @@ -19,8 +19,9 @@ package org.elasticsearch.cluster.node; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.elasticsearch.Version; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; @@ -30,10 +31,15 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.nullValue; public class DiscoveryNodesTests extends ESTestCase { @@ -53,7 +59,7 @@ public class DiscoveryNodesTests extends ESTestCase { DiscoveryNode resolvedNode = discoveryNodes.resolveNode(nodeSelector.selector); assertThat(matchingNodeIds.size(), equalTo(1)); assertThat(resolvedNode.getId(), equalTo(matchingNodeIds.iterator().next())); - } catch(IllegalArgumentException e) { + } catch (IllegalArgumentException e) { if (matchingNodeIds.size() == 0) { assertThat(e.getMessage(), equalTo("failed to resolve [" + nodeSelector.selector + "], no matching nodes")); } else if (matchingNodeIds.size() > 1) { @@ -98,26 +104,98 @@ public class DiscoveryNodesTests extends ESTestCase { assertThat(resolvedNodesIds, equalTo(expectedNodesIds)); } - private static DiscoveryNodes buildDiscoveryNodes() { - int numNodes = randomIntBetween(1, 10); - DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + public void testDeltas() { + Set nodesA = new HashSet<>(); + nodesA.addAll(randomNodes(1 + randomInt(10))); + Set nodesB = new HashSet<>(); + nodesB.addAll(randomNodes(1 + randomInt(5))); + for (DiscoveryNode node : randomSubsetOf(nodesA)) { + if (randomBoolean()) { + // change an attribute + Map attrs = new HashMap<>(node.getAttributes()); + attrs.put("new", "new"); + node = new DiscoveryNode(node.getName(), node.getId(), node.getAddress(), attrs, node.getRoles(), node.getVersion()); + } + nodesB.add(node); + } + + DiscoveryNode masterA = randomBoolean() ? null : RandomPicks.randomFrom(random(), nodesA); + DiscoveryNode masterB = randomBoolean() ? null : RandomPicks.randomFrom(random(), nodesB); + + DiscoveryNodes.Builder builderA = DiscoveryNodes.builder(); + nodesA.stream().forEach(builderA::put); + final String masterAId = masterA == null ? null : masterA.getId(); + builderA.masterNodeId(masterAId); + builderA.localNodeId(RandomPicks.randomFrom(random(), nodesA).getId()); + + DiscoveryNodes.Builder builderB = DiscoveryNodes.builder(); + nodesB.stream().forEach(builderB::put); + final String masterBId = masterB == null ? null : masterB.getId(); + builderB.masterNodeId(masterBId); + builderB.localNodeId(RandomPicks.randomFrom(random(), nodesB).getId()); + + final DiscoveryNodes discoNodesA = builderA.build(); + final DiscoveryNodes discoNodesB = builderB.build(); + logger.info("nodes A: {}", discoNodesA.prettyPrint()); + logger.info("nodes B: {}", discoNodesB.prettyPrint()); + + DiscoveryNodes.Delta delta = discoNodesB.delta(discoNodesA); + + if (masterB == null || Objects.equals(masterAId, masterBId)) { + assertFalse(delta.masterNodeChanged()); + assertThat(delta.previousMasterNode(), nullValue()); + assertThat(delta.newMasterNode(), nullValue()); + } else { + assertTrue(delta.masterNodeChanged()); + assertThat(delta.newMasterNode().getId(), equalTo(masterBId)); + assertThat(delta.previousMasterNode() != null ? delta.previousMasterNode().getId() : null, + equalTo(masterAId)); + } + + Set newNodes = new HashSet<>(nodesB); + newNodes.removeAll(nodesA); + assertThat(delta.added(), equalTo(newNodes.isEmpty() == false)); + assertThat(delta.addedNodes(), containsInAnyOrder(newNodes.stream().collect(Collectors.toList()).toArray())); + assertThat(delta.addedNodes().size(), equalTo(newNodes.size())); + + Set removedNodes = new HashSet<>(nodesA); + removedNodes.removeAll(nodesB); + assertThat(delta.removed(), equalTo(removedNodes.isEmpty() == false)); + assertThat(delta.removedNodes(), containsInAnyOrder(removedNodes.stream().collect(Collectors.toList()).toArray())); + assertThat(delta.removedNodes().size(), equalTo(removedNodes.size())); + } + + private static AtomicInteger idGenerator = new AtomicInteger(); + + private static List randomNodes(final int numNodes) { List nodesList = new ArrayList<>(); for (int i = 0; i < numNodes; i++) { Map attributes = new HashMap<>(); if (frequently()) { attributes.put("custom", randomBoolean() ? "match" : randomAsciiOfLengthBetween(3, 5)); } - final DiscoveryNode node = newNode(i, attributes, new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())))); - discoBuilder = discoBuilder.put(node); + final DiscoveryNode node = newNode(idGenerator.getAndIncrement(), attributes, + new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())))); nodesList.add(node); } + return nodesList; + } + + private static DiscoveryNodes buildDiscoveryNodes() { + int numNodes = randomIntBetween(1, 10); + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + List nodesList = randomNodes(numNodes); + for (DiscoveryNode node : nodesList) { + discoBuilder = discoBuilder.put(node); + } discoBuilder.localNodeId(randomFrom(nodesList).getId()); discoBuilder.masterNodeId(randomFrom(nodesList).getId()); return discoBuilder.build(); } private static DiscoveryNode newNode(int nodeId, Map attributes, Set roles) { - return new DiscoveryNode("name_" + nodeId, "node_" + nodeId, DummyTransportAddress.INSTANCE, attributes, roles, Version.CURRENT); + return new DiscoveryNode("name_" + nodeId, "node_" + nodeId, LocalTransportAddress.buildUnique(), attributes, roles, + Version.CURRENT); } private enum NodeSelector { @@ -152,7 +230,7 @@ public class DiscoveryNodesTests extends ESTestCase { nodes.getIngestNodes().keysIt().forEachRemaining(ids::add); return ids; } - },CUSTOM_ATTRIBUTE("attr:value") { + }, CUSTOM_ATTRIBUTE("attr:value") { @Override Set matchingNodeIds(DiscoveryNodes nodes) { Set ids = new HashSet<>(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java index 2d3e44db68a..32072282d6f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java @@ -50,7 +50,7 @@ public class PrimaryTermsTests extends ESAllocationTestCase { private RoutingTable testRoutingTable; private int numberOfShards; private int numberOfReplicas; - private final static Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + private static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); private AllocationService allocationService; private ClusterState clusterState; diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index 2d1a467a001..9da5e76ed1f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -47,7 +47,7 @@ public class RoutingTableTests extends ESAllocationTestCase { private int numberOfReplicas; private int shardsPerIndex; private int totalNumberOfShards; - private final static Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + private static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); private final AllocationService ALLOCATION_SERVICE = createAllocationService(Settings.builder() .put("cluster.routing.allocation.node_concurrent_recoveries", Integer.MAX_VALUE) // don't limit recoveries .put("cluster.routing.allocation.node_initial_primaries_recoveries", Integer.MAX_VALUE) diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java index 7267252b19f..fa9133f6d36 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java @@ -206,7 +206,7 @@ public class ShardRoutingTests extends ESTestCase { if (randomBoolean()) { BytesStreamOutput out = new BytesStreamOutput(); routing.writeTo(out); - routing = new ShardRouting(StreamInput.wrap(out.bytes())); + routing = new ShardRouting(out.bytes().streamInput()); } if (routing.initializing() || routing.relocating()) { assertEquals(routing.toString(), byteSize, routing.getExpectedShardSize()); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java index 75300a4beb8..ec33a3cd5fc 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java @@ -82,7 +82,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase { meta.writeTo(out); out.close(); - UnassignedInfo read = new UnassignedInfo(StreamInput.wrap(out.bytes())); + UnassignedInfo read = new UnassignedInfo(out.bytes().streamInput()); assertThat(read.getReason(), equalTo(meta.getReason())); assertThat(read.getUnassignedTimeInMillis(), equalTo(meta.getUnassignedTimeInMillis())); assertThat(read.getMessage(), equalTo(meta.getMessage())); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java index 28f27b8988c..f95fb687c76 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java @@ -430,7 +430,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase { ); BytesStreamOutput bytes = new BytesStreamOutput(); AllocationCommands.writeTo(commands, bytes); - StreamInput in = StreamInput.wrap(bytes.bytes()); + StreamInput in = bytes.bytes().streamInput(); // Since the commands are named writeable we need to register them and wrap the input stream NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java index 6dd4df59060..9859cd4d570 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java @@ -25,8 +25,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.EmptyClusterInfoService; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -44,12 +42,14 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.test.ESAllocationTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.gateway.NoopGatewayAllocator; @@ -307,11 +307,11 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { public void testRebalanceDoesNotAllocatePrimaryAndReplicasOnDifferentVersionNodes() { ShardId shard1 = new ShardId("test1", "_na_", 0); ShardId shard2 = new ShardId("test2", "_na_", 0); - final DiscoveryNode newNode = new DiscoveryNode("newNode", DummyTransportAddress.INSTANCE, emptyMap(), + final DiscoveryNode newNode = new DiscoveryNode("newNode", LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT); - final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", DummyTransportAddress.INSTANCE, emptyMap(), + final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, VersionUtils.getPreviousVersion()); - final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", DummyTransportAddress.INSTANCE, emptyMap(), + final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, VersionUtils.getPreviousVersion()); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder(shard1.getIndexName()).settings(settings(Version.CURRENT).put(Settings.EMPTY)).numberOfShards(1).numberOfReplicas(1)) @@ -347,11 +347,11 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { } public void testRestoreDoesNotAllocateSnapshotOnOlderNodes() { - final DiscoveryNode newNode = new DiscoveryNode("newNode", DummyTransportAddress.INSTANCE, emptyMap(), + final DiscoveryNode newNode = new DiscoveryNode("newNode", LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT); - final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", DummyTransportAddress.INSTANCE, emptyMap(), + final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, VersionUtils.getPreviousVersion()); - final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", DummyTransportAddress.INSTANCE, emptyMap(), + final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, VersionUtils.getPreviousVersion()); int numberOfShards = randomIntBetween(1, 3); @@ -407,7 +407,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { return clusterState; } - private final void assertRecoveryNodeVersions(RoutingNodes routingNodes) { + private void assertRecoveryNodeVersions(RoutingNodes routingNodes) { logger.trace("RoutingNodes: {}", routingNodes.prettyPrint()); List mutableShardRoutings = routingNodes.shardsWithState(ShardRoutingState.RELOCATING); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java index 3d0475ed137..e09d9790651 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java @@ -33,7 +33,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationD import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.test.ESAllocationTestCase; import static java.util.Collections.emptyMap; @@ -63,9 +63,9 @@ public class SameShardRoutingTests extends ESAllocationTestCase { logger.info("--> adding two nodes with the same host"); clusterState = ClusterState.builder(clusterState).nodes( DiscoveryNodes.builder() - .put(new DiscoveryNode("node1", "node1", "test1", "test1", DummyTransportAddress.INSTANCE, emptyMap(), + .put(new DiscoveryNode("node1", "node1", "node1", "test1", "test1", LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT)) - .put(new DiscoveryNode("node2", "node2", "test1", "test1", DummyTransportAddress.INSTANCE, emptyMap(), + .put(new DiscoveryNode("node2", "node2", "node2", "test1", "test1", LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT))).build(); routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); @@ -82,7 +82,7 @@ public class SameShardRoutingTests extends ESAllocationTestCase { logger.info("--> add another node, with a different host, replicas will be allocating"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) - .put(new DiscoveryNode("node3", "node3", "test2", "test2", DummyTransportAddress.INSTANCE, emptyMap(), + .put(new DiscoveryNode("node3", "node3", "node3", "test2", "test2", LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT))).build(); routingTable = strategy.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index be50c5f5331..56ca6381af9 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -41,7 +41,6 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.Index; @@ -110,9 +109,9 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase { final Index index = metaData.index("test").getIndex(); ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); - DiscoveryNode node_0 = new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, Collections.emptyMap(), + DiscoveryNode node_0 = new DiscoveryNode("node_0", LocalTransportAddress.buildUnique(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); - DiscoveryNode node_1 = new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Collections.emptyMap(), + DiscoveryNode node_1 = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); RoutingTable routingTable = RoutingTable.builder() @@ -149,9 +148,9 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase { DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss, cis, null); ImmutableOpenMap.Builder shardRoutingMap = ImmutableOpenMap.builder(); - DiscoveryNode node_0 = new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, Collections.emptyMap(), + DiscoveryNode node_0 = new DiscoveryNode("node_0", LocalTransportAddress.buildUnique(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); - DiscoveryNode node_1 = new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Collections.emptyMap(), + DiscoveryNode node_1 = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); MetaData metaData = MetaData.builder() diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index 7b81d3ece27..4fa6615ac45 100644 --- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -80,7 +80,7 @@ public class ClusterSerializationTests extends ESAllocationTestCase { BytesStreamOutput outStream = new BytesStreamOutput(); source.writeTo(outStream); - StreamInput inStream = StreamInput.wrap(outStream.bytes().toBytes()); + StreamInput inStream = outStream.bytes().streamInput(); RoutingTable target = RoutingTable.Builder.readFrom(inStream); assertThat(target.prettyPrint(), equalTo(source.prettyPrint())); diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java index 99cde60f086..9957a6d3603 100644 --- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java @@ -29,7 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.test.ESAllocationTestCase; import static java.util.Collections.emptyMap; @@ -50,7 +50,7 @@ public class ClusterStateToStringTests extends ESAllocationTestCase { .addAsNew(metaData.index("test_idx")) .build(); - DiscoveryNodes nodes = DiscoveryNodes.builder().put(new DiscoveryNode("node_foo", DummyTransportAddress.INSTANCE, + DiscoveryNodes nodes = DiscoveryNodes.builder().put(new DiscoveryNode("node_foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT)).localNodeId("node_foo").masterNodeId("node_foo").build(); ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes) diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java index 452c6054576..611c261e334 100644 --- a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java @@ -310,7 +310,7 @@ public class DiffableTests extends ESTestCase { logger.debug("--> serializing diff"); BytesStreamOutput out = new BytesStreamOutput(); diffMap.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); logger.debug("--> reading diff back"); diffMap = readDiff(in); } diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java index 991f11a4493..23713832edf 100644 --- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java @@ -94,7 +94,7 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onAllNodesAcked(@Nullable Throwable t) { + public void onAllNodesAcked(@Nullable Exception e) { allNodesAcked.set(true); latch.countDown(); } @@ -127,8 +127,8 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { - logger.error("failed to execute callback in test {}", t, source); + public void onFailure(String source, Exception e) { + logger.error("failed to execute callback in test {}", e, source); onFailure.set(true); latch.countDown(); } @@ -165,7 +165,7 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onAllNodesAcked(@Nullable Throwable t) { + public void onAllNodesAcked(@Nullable Exception e) { allNodesAcked.set(true); latch.countDown(); } @@ -198,8 +198,8 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { - logger.error("failed to execute callback in test {}", t, source); + public void onFailure(String source, Exception e) { + logger.error("failed to execute callback in test {}", e, source); onFailure.set(true); latch.countDown(); } @@ -240,7 +240,7 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onAllNodesAcked(@Nullable Throwable t) { + public void onAllNodesAcked(@Nullable Exception e) { allNodesAcked.set(true); latch.countDown(); } @@ -272,8 +272,8 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { - logger.error("failed to execute callback in test {}", t, source); + public void onFailure(String source, Exception e) { + logger.error("failed to execute callback in test {}", e, source); onFailure.set(true); latch.countDown(); } @@ -313,7 +313,7 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onAllNodesAcked(@Nullable Throwable t) { + public void onAllNodesAcked(@Nullable Exception e) { allNodesAcked.set(true); latch.countDown(); } @@ -346,8 +346,8 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { - logger.error("failed to execute callback in test {}", t, source); + public void onFailure(String source, Exception e) { + logger.error("failed to execute callback in test {}", e, source); onFailure.set(true); latch.countDown(); } @@ -388,7 +388,7 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { invoked1.countDown(); fail(); } @@ -403,7 +403,7 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { fail(); } @@ -458,7 +458,7 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { invoked3.countDown(); fail(); } @@ -473,7 +473,7 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { fail(); } }); @@ -591,7 +591,7 @@ public class ClusterServiceIT extends ESIntegTestCase { } @Singleton - public static class MasterAwareService extends AbstractLifecycleComponent implements LocalNodeMasterListener { + public static class MasterAwareService extends AbstractLifecycleComponent implements LocalNodeMasterListener { private final ClusterService clusterService; private volatile boolean master; diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java index 66f96f8cd3a..54f6233631b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java @@ -37,7 +37,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLogAppender; @@ -109,7 +109,7 @@ public class ClusterServiceTests extends ESTestCase { TimedClusterService timedClusterService = new TimedClusterService(Settings.builder().put("cluster.name", "ClusterServiceTests").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool); - timedClusterService.setLocalNode(new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(), + timedClusterService.setLocalNode(new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT)); timedClusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) { @Override @@ -149,8 +149,8 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { - throw new RuntimeException(t); + public void onFailure(String source, Exception e) { + throw new RuntimeException(e); } }); @@ -163,7 +163,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { timedOut.countDown(); } @@ -183,8 +183,8 @@ public class ClusterServiceTests extends ESTestCase { final CountDownLatch allProcessed = new CountDownLatch(1); clusterService.submitStateUpdateTask("test3", new ClusterStateUpdateTask() { @Override - public void onFailure(String source, Throwable t) { - throw new RuntimeException(t); + public void onFailure(String source, Exception e) { + throw new RuntimeException(e); } @Override @@ -212,7 +212,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { taskFailed[0] = true; latch1.countDown(); } @@ -237,7 +237,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { taskFailed[0] = true; latch2.countDown(); } @@ -286,7 +286,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { } } ); @@ -326,9 +326,9 @@ public class ClusterServiceTests extends ESTestCase { ClusterStateTaskListener listener = new ClusterStateTaskListener() { @Override - public void onFailure(String source, Throwable t) { - logger.error("unexpected failure: [{}]", t, source); - failures.add(new Tuple<>(source, t)); + public void onFailure(String source, Exception e) { + logger.error("unexpected failure: [{}]", e, source); + failures.add(new Tuple<>(source, e)); updateLatch.countDown(); } @@ -387,8 +387,8 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { - fail(ExceptionsHelper.detailedMessage(t)); + public void onFailure(String source, Exception e) { + fail(ExceptionsHelper.detailedMessage(e)); } })) ; } @@ -523,8 +523,8 @@ public class ClusterServiceTests extends ESTestCase { final CountDownLatch updateLatch = new CountDownLatch(totalTaskCount); final ClusterStateTaskListener listener = new ClusterStateTaskListener() { @Override - public void onFailure(String source, Throwable t) { - fail(ExceptionsHelper.detailedMessage(t)); + public void onFailure(String source, Exception e) { + fail(ExceptionsHelper.detailedMessage(e)); } @Override @@ -647,8 +647,8 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { - fail(ExceptionsHelper.detailedMessage(t)); + public void onFailure(String source, Exception e) { + fail(ExceptionsHelper.detailedMessage(e)); } }; @@ -693,7 +693,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { fail(); } }); @@ -710,7 +710,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { latch.countDown(); } }); @@ -727,7 +727,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { fail(); } }); @@ -745,7 +745,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { fail(); } }); @@ -788,7 +788,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { fail(); } }); @@ -807,7 +807,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { latch.countDown(); } }); @@ -824,7 +824,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { fail(); } }); @@ -841,7 +841,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { fail(); } }); @@ -859,7 +859,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { fail(); } }); @@ -902,7 +902,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { } public void close() { @@ -930,7 +930,7 @@ public class ClusterServiceTests extends ESTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { latch.countDown(); } } diff --git a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java index 4f2bad36d4a..c0cb3482b0e 100644 --- a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java +++ b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java @@ -19,14 +19,11 @@ package org.elasticsearch.common; -import org.elasticsearch.common.bytes.ByteBufferBytesReference; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Channels; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.jboss.netty.buffer.ByteBufferBackedChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffer; import org.junit.After; import org.junit.Before; @@ -85,7 +82,7 @@ public class ChannelsTests extends ESTestCase { BytesReference source = new BytesArray(randomBytes, offset + offsetToRead, lengthToRead); BytesReference read = new BytesArray(readBytes, offset + offsetToRead, lengthToRead); - assertThat("read bytes didn't match written bytes", source.toBytes(), Matchers.equalTo(read.toBytes())); + assertThat("read bytes didn't match written bytes", BytesReference.toBytes(source), Matchers.equalTo(BytesReference.toBytes(read))); } public void testBufferReadPastEOFWithException() throws Exception { @@ -157,7 +154,9 @@ public class ChannelsTests extends ESTestCase { copy.flip(); BytesReference sourceRef = new BytesArray(randomBytes, offset + offsetToRead, lengthToRead); - BytesReference copyRef = new ByteBufferBytesReference(copy); + byte[] tmp = new byte[copy.remaining()]; + copy.duplicate().get(tmp); + BytesReference copyRef = new BytesArray(tmp); assertTrue("read bytes didn't match written bytes", sourceRef.equals(copyRef)); } diff --git a/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java b/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java index bb9d23db1cb..f10a0da3029 100644 --- a/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java +++ b/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java @@ -43,7 +43,7 @@ public class MemoryCircuitBreakerTests extends ESTestCase { final int BYTES_PER_THREAD = scaledRandomIntBetween(500, 4500); final Thread[] threads = new Thread[NUM_THREADS]; final AtomicBoolean tripped = new AtomicBoolean(false); - final AtomicReference lastException = new AtomicReference<>(null); + final AtomicReference lastException = new AtomicReference<>(null); final MemoryCircuitBreaker breaker = new MemoryCircuitBreaker(new ByteSizeValue((BYTES_PER_THREAD * NUM_THREADS) - 1), 1.0, logger); @@ -60,8 +60,8 @@ public class MemoryCircuitBreakerTests extends ESTestCase { } else { assertThat(tripped.compareAndSet(false, true), equalTo(true)); } - } catch (Throwable e2) { - lastException.set(e2); + } catch (Exception e) { + lastException.set(e); } } } @@ -117,8 +117,8 @@ public class MemoryCircuitBreakerTests extends ESTestCase { } else { assertThat(tripped.compareAndSet(false, true), equalTo(true)); } - } catch (Throwable e2) { - lastException.set(e2); + } catch (Exception e) { + lastException.set(e); } } } @@ -178,8 +178,8 @@ public class MemoryCircuitBreakerTests extends ESTestCase { breaker.addEstimateBytesAndMaybeBreak(1L, "test"); } catch (CircuitBreakingException e) { tripped.incrementAndGet(); - } catch (Throwable e2) { - lastException.set(e2); + } catch (Exception e) { + lastException.set(e); } } } diff --git a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java index f31d7c69325..c998146ed74 100644 --- a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ByteArray; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; -import org.hamcrest.Matchers; import java.io.EOFException; import java.io.IOException; @@ -43,11 +42,20 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { public void testGet() throws IOException { int length = randomIntBetween(1, PAGE_SIZE * 3); BytesReference pbr = newBytesReference(length); + int sliceOffset = randomIntBetween(0, length / 2); int sliceLength = Math.max(1, length - sliceOffset - 1); BytesReference slice = pbr.slice(sliceOffset, sliceLength); assertEquals(pbr.get(sliceOffset), slice.get(0)); assertEquals(pbr.get(sliceOffset + sliceLength - 1), slice.get(sliceLength - 1)); + final int probes = randomIntBetween(20, 100); + BytesReference copy = new BytesArray(pbr.toBytesRef(), true); + for (int i = 0; i < probes; i++) { + int index = randomIntBetween(0, copy.length() - 1); + assertEquals(pbr.get(index), copy.get(index)); + index = randomIntBetween(sliceOffset, sliceOffset + sliceLength); + assertEquals(pbr.get(index), slice.get(index - sliceOffset)); + } } public void testLength() throws IOException { @@ -66,12 +74,9 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { int sliceLength = Math.max(0, length - sliceOffset - 1); BytesReference slice = pbr.slice(sliceOffset, sliceLength); assertEquals(sliceLength, slice.length()); - - if (slice.hasArray()) { - assertEquals(sliceOffset, slice.arrayOffset()); - } else { - expectThrows(IllegalStateException.class, () -> - slice.arrayOffset()); + BytesRef singlePageOrNull = getSinglePageOrNull(slice); + if (singlePageOrNull != null) { + assertEquals(sliceOffset, singlePageOrNull.offset); } } @@ -109,7 +114,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { // bulk-read all si.readFully(targetBuf); - assertArrayEquals(pbr.toBytes(), targetBuf); + assertArrayEquals(BytesReference.toBytes(pbr), targetBuf); // continuing to read should now fail with EOFException try { @@ -125,6 +130,26 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { si.readBytes(targetBuf, 0, length * 2)); } + public void testStreamInputMarkAndReset() throws IOException { + int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); + BytesReference pbr = newBytesReference(length); + StreamInput si = pbr.streamInput(); + assertNotNull(si); + + StreamInput wrap = StreamInput.wrap(BytesReference.toBytes(pbr)); + while(wrap.available() > 0) { + if (rarely()) { + wrap.mark(Integer.MAX_VALUE); + si.mark(Integer.MAX_VALUE); + } else if (rarely()) { + wrap.reset(); + si.reset(); + } + assertEquals(si.readByte(), wrap.readByte()); + assertEquals(si.available(), wrap.available()); + } + } + public void testStreamInputBulkReadWithOffset() throws IOException { final int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); BytesReference pbr = newBytesReference(length); @@ -141,7 +166,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { // now do NOT reset the stream - keep the stream's offset! // buffer to compare remaining bytes against bulk read - byte[] pbrBytesWithOffset = Arrays.copyOfRange(pbr.toBytes(), offset, length); + byte[] pbrBytesWithOffset = Arrays.copyOfRange(BytesReference.toBytes(pbr), offset, length); // randomized target buffer to ensure no stale slots byte[] targetBytes = new byte[pbrBytesWithOffset.length]; random().nextBytes(targetBytes); @@ -178,7 +203,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { } assertEquals(pbr.length(), target.length()); BytesRef targetBytes = target.get(); - assertArrayEquals(pbr.toBytes(), Arrays.copyOfRange(targetBytes.bytes, targetBytes.offset, targetBytes.length)); + assertArrayEquals(BytesReference.toBytes(pbr), Arrays.copyOfRange(targetBytes.bytes, targetBytes.offset, targetBytes.length)); } public void testSliceStreamInput() throws IOException { @@ -208,11 +233,11 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { assertEquals(sliceInput.available(), 0); // compare slice content with upper half of original - byte[] pbrSliceBytes = Arrays.copyOfRange(pbr.toBytes(), sliceOffset, length); + byte[] pbrSliceBytes = Arrays.copyOfRange(BytesReference.toBytes(pbr), sliceOffset, length); assertArrayEquals(pbrSliceBytes, sliceBytes); // compare slice bytes with bytes read from slice via streamInput :D - byte[] sliceToBytes = slice.toBytes(); + byte[] sliceToBytes = BytesReference.toBytes(slice); assertEquals(sliceBytes.length, sliceToBytes.length); assertArrayEquals(sliceBytes, sliceToBytes); @@ -233,10 +258,28 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); pbr.writeTo(out); assertEquals(pbr.length(), out.size()); - assertArrayEquals(pbr.toBytes(), out.bytes().toBytes()); + assertArrayEquals(BytesReference.toBytes(pbr), BytesReference.toBytes(out.bytes())); out.close(); } + public void testInputStreamSkip() throws IOException { + int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); + BytesReference pbr = newBytesReference(length); + final int iters = randomIntBetween(5, 50); + for (int i = 0; i < iters; i++) { + try (StreamInput input = pbr.streamInput()) { + final int offset = randomIntBetween(0, length-1); + assertEquals(offset, input.skip(offset)); + assertEquals(pbr.get(offset), input.readByte()); + final int nextOffset = randomIntBetween(offset, length-2); + assertEquals(nextOffset - offset, input.skip(nextOffset - offset)); + assertEquals(pbr.get(nextOffset+1), input.readByte()); // +1 for the one byte we read above + assertEquals(length - (nextOffset+2), input.skip(Long.MAX_VALUE)); + assertEquals(0, input.skip(randomIntBetween(0, Integer.MAX_VALUE))); + } + } + } + public void testSliceWriteToOutputStream() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 5)); BytesReference pbr = newBytesReference(length); @@ -246,7 +289,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { BytesStreamOutput sliceOut = new BytesStreamOutput(sliceLength); slice.writeTo(sliceOut); assertEquals(slice.length(), sliceOut.size()); - assertArrayEquals(slice.toBytes(), sliceOut.bytes().toBytes()); + assertArrayEquals(BytesReference.toBytes(slice), BytesReference.toBytes(sliceOut.bytes())); sliceOut.close(); } @@ -254,16 +297,19 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))}; for (int i = 0; i < sizes.length; i++) { BytesReference pbr = newBytesReference(sizes[i]); - byte[] bytes = pbr.toBytes(); + byte[] bytes = BytesReference.toBytes(pbr); assertEquals(sizes[i], bytes.length); + for (int j = 0; j < bytes.length; j++) { + assertEquals(bytes[j], pbr.get(j)); + } } } - public void testToBytesArraySharedPage() throws IOException { + public void testToBytesRefSharedPage() throws IOException { int length = randomIntBetween(10, PAGE_SIZE); BytesReference pbr = newBytesReference(length); - BytesArray ba = pbr.toBytesArray(); - BytesArray ba2 = pbr.toBytesArray(); + BytesArray ba = new BytesArray(pbr.toBytesRef()); + BytesArray ba2 = new BytesArray(pbr.toBytesRef()); assertNotNull(ba); assertNotNull(ba2); assertEquals(pbr.length(), ba.length()); @@ -272,46 +318,46 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { assertSame(ba.array(), ba2.array()); } - public void testToBytesArrayMaterializedPages() throws IOException { + public void testToBytesRefMaterializedPages() throws IOException { // we need a length != (n * pagesize) to avoid page sharing at boundaries int length = 0; while ((length % PAGE_SIZE) == 0) { length = randomIntBetween(PAGE_SIZE, PAGE_SIZE * randomIntBetween(2, 5)); } BytesReference pbr = newBytesReference(length); - BytesArray ba = pbr.toBytesArray(); - BytesArray ba2 = pbr.toBytesArray(); + BytesArray ba = new BytesArray(pbr.toBytesRef()); + BytesArray ba2 = new BytesArray(pbr.toBytesRef()); assertNotNull(ba); assertNotNull(ba2); assertEquals(pbr.length(), ba.length()); assertEquals(ba.length(), ba2.length()); } - public void testCopyBytesArray() throws IOException { + public void testCopyBytesRefSharesBytes() throws IOException { // small PBR which would normally share the first page int length = randomIntBetween(10, PAGE_SIZE); BytesReference pbr = newBytesReference(length); - BytesArray ba = pbr.copyBytesArray(); - BytesArray ba2 = pbr.copyBytesArray(); + BytesArray ba = new BytesArray(pbr.toBytesRef(), true); + BytesArray ba2 = new BytesArray(pbr.toBytesRef(), true); assertNotNull(ba); assertNotSame(ba, ba2); assertNotSame(ba.array(), ba2.array()); } - public void testSliceCopyBytesArray() throws IOException { + public void testSliceCopyBytesRef() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); BytesReference pbr = newBytesReference(length); int sliceOffset = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); BytesReference slice = pbr.slice(sliceOffset, sliceLength); - BytesArray ba1 = slice.copyBytesArray(); - BytesArray ba2 = slice.copyBytesArray(); + BytesArray ba1 = new BytesArray(slice.toBytesRef(), true); + BytesArray ba2 = new BytesArray(slice.toBytesRef(), true); assertNotNull(ba1); assertNotNull(ba2); assertNotSame(ba1.array(), ba2.array()); - assertArrayEquals(slice.toBytes(), ba1.array()); - assertArrayEquals(slice.toBytes(), ba2.array()); + assertArrayEquals(BytesReference.toBytes(slice), ba1.array()); + assertArrayEquals(BytesReference.toBytes(slice), ba2.array()); assertArrayEquals(ba1.array(), ba2.array()); } @@ -329,14 +375,14 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { while((ref = iterator.next()) != null) { builder.append(ref); } - assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); + assertArrayEquals(BytesReference.toBytes(pbr), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); } public void testSliceIterator() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); BytesReference pbr = newBytesReference(length); int sliceOffset = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); BytesReference slice = pbr.slice(sliceOffset, sliceLength); BytesRefIterator iterator = slice.iterator(); BytesRef ref = null; @@ -344,7 +390,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { while((ref = iterator.next()) != null) { builder.append(ref); } - assertArrayEquals(slice.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); + assertArrayEquals(BytesReference.toBytes(slice), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); } public void testIteratorRandom() throws IOException { @@ -352,12 +398,12 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { BytesReference pbr = newBytesReference(length); if (randomBoolean()) { int sliceOffset = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); pbr = pbr.slice(sliceOffset, sliceLength); } if (randomBoolean()) { - pbr = pbr.toBytesArray(); + pbr = new BytesArray(pbr.toBytesRef()); } BytesRefIterator iterator = pbr.iterator(); BytesRef ref = null; @@ -365,29 +411,15 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { while((ref = iterator.next()) != null) { builder.append(ref); } - assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); - } - - public void testArray() throws IOException { - int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))}; - - for (int i = 0; i < sizes.length; i++) { - BytesReference pbr = newBytesReference(sizes[i]); - byte[] array = pbr.array(); - assertNotNull(array); - assertEquals(sizes[i], array.length); - assertSame(array, pbr.array()); - } + assertArrayEquals(BytesReference.toBytes(pbr), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); } public void testArrayOffset() throws IOException { int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5)); BytesReference pbr = newBytesReference(length); - if (pbr.hasArray()) { - assertEquals(0, pbr.arrayOffset()); - } else { - expectThrows(IllegalStateException.class, () -> - pbr.arrayOffset()); + BytesRef singlePageOrNull = getSinglePageOrNull(pbr); + if (singlePageOrNull != null) { + assertEquals(0, singlePageOrNull.offset); } } @@ -395,20 +427,24 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5)); BytesReference pbr = newBytesReference(length); int sliceOffset = randomIntBetween(0, pbr.length() - 1); // an offset to the end would be len 0 - int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); BytesReference slice = pbr.slice(sliceOffset, sliceLength); - if (slice.hasArray()) { - assertEquals(sliceOffset, slice.arrayOffset()); - } else { - expectThrows(IllegalStateException.class, () -> - slice.arrayOffset()); + BytesRef singlePageOrNull = getSinglePageOrNull(slice); + if (singlePageOrNull != null) { + if (getSinglePageOrNull(pbr) == null) { + // original reference has pages + assertEquals(sliceOffset % PAGE_SIZE, singlePageOrNull.offset); + } else { + // orig ref has no pages ie. BytesArray + assertEquals(sliceOffset, singlePageOrNull.offset); + } } } public void testToUtf8() throws IOException { // test empty BytesReference pbr = newBytesReference(0); - assertEquals("", pbr.toUtf8()); + assertEquals("", pbr.utf8ToString()); // TODO: good way to test? } @@ -417,7 +453,6 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { BytesReference pbr = newBytesReference(length); BytesRef ref = pbr.toBytesRef(); assertNotNull(ref); - assertEquals(pbr.arrayOffset(), ref.offset); assertEquals(pbr.length(), ref.length); } @@ -426,21 +461,13 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { BytesReference pbr = newBytesReference(length); // get a BytesRef from a slice int sliceOffset = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); BytesRef sliceRef = pbr.slice(sliceOffset, sliceLength).toBytesRef(); // note that these are only true if we have <= than a page, otherwise offset/length are shifted assertEquals(sliceOffset, sliceRef.offset); assertEquals(sliceLength, sliceRef.length); } - public void testCopyBytesRef() throws IOException { - int length = randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5)); - BytesReference pbr = newBytesReference(length); - BytesRef ref = pbr.copyBytesRef(); - assertNotNull(ref); - assertEquals(pbr.length(), ref.length); - } - public void testHashCode() throws IOException { // empty content must have hash 1 (JDK compat) BytesReference pbr = newBytesReference(0); @@ -448,40 +475,36 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { // test with content pbr = newBytesReference(randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5))); - int jdkHash = Arrays.hashCode(pbr.toBytes()); + int jdkHash = Arrays.hashCode(BytesReference.toBytes(pbr)); int pbrHash = pbr.hashCode(); assertEquals(jdkHash, pbrHash); // test hashes of slices int sliceFrom = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom); + int sliceLength = randomIntBetween(0, pbr.length() - sliceFrom); BytesReference slice = pbr.slice(sliceFrom, sliceLength); - int sliceJdkHash = Arrays.hashCode(slice.toBytes()); + int sliceJdkHash = Arrays.hashCode(BytesReference.toBytes(slice)); int sliceHash = slice.hashCode(); assertEquals(sliceJdkHash, sliceHash); } - public void testEquals() { - int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5)); - ByteArray ba1 = bigarrays.newByteArray(length, false); - ByteArray ba2 = bigarrays.newByteArray(length, false); - - // copy contents - for (long i = 0; i < length; i++) { - ba2.set(i, ba1.get(i)); - } + public void testEquals() throws IOException { + BytesReference bytesReference = newBytesReference(randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5))); + BytesReference copy = bytesReference.slice(0, bytesReference.length()); // get refs & compare - BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length); - BytesReference pbr2 = new PagedBytesReference(bigarrays, ba2, length); - assertEquals(pbr, pbr2); - } + assertEquals(copy, bytesReference); + int sliceFrom = randomIntBetween(0, bytesReference.length()); + int sliceLength = randomIntBetween(0, bytesReference.length() - sliceFrom); + assertEquals(copy.slice(sliceFrom, sliceLength), bytesReference.slice(sliceFrom, sliceLength)); - public void testEqualsPeerClass() throws IOException { - int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5)); - BytesReference pbr = newBytesReference(length); - BytesReference ba = new BytesArray(pbr.toBytes()); - assertEquals(pbr, ba); + BytesRef bytesRef = BytesRef.deepCopyOf(copy.toBytesRef()); + assertEquals(new BytesArray(bytesRef), copy); + + int offsetToFlip = randomIntBetween(0, bytesRef.length - 1); + int value = ~Byte.toUnsignedInt(bytesRef.bytes[bytesRef.offset+offsetToFlip]); + bytesRef.bytes[bytesRef.offset+offsetToFlip] = (byte)value; + assertNotEquals(new BytesArray(bytesRef), copy); } public void testSliceEquals() { @@ -491,19 +514,118 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { // test equality of slices int sliceFrom = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom); + int sliceLength = randomIntBetween(0, pbr.length() - sliceFrom); BytesReference slice1 = pbr.slice(sliceFrom, sliceLength); BytesReference slice2 = pbr.slice(sliceFrom, sliceLength); - assertArrayEquals(slice1.toBytes(), slice2.toBytes()); + assertArrayEquals(BytesReference.toBytes(slice1), BytesReference.toBytes(slice2)); // test a slice with same offset but different length, // unless randomized testing gave us a 0-length slice. if (sliceLength > 0) { BytesReference slice3 = pbr.slice(sliceFrom, sliceLength / 2); - assertFalse(Arrays.equals(slice1.toBytes(), slice3.toBytes())); + assertFalse(Arrays.equals(BytesReference.toBytes(slice1), BytesReference.toBytes(slice3))); } } protected abstract BytesReference newBytesReference(int length) throws IOException; + public void testCompareTo() throws IOException { + final int iters = randomIntBetween(5, 10); + for (int i = 0; i < iters; i++) { + int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); + BytesReference bytesReference = newBytesReference(length); + assertTrue(bytesReference.compareTo(new BytesArray("")) > 0); + assertTrue(new BytesArray("").compareTo(bytesReference) < 0); + + + assertEquals(0, bytesReference.compareTo(bytesReference)); + int sliceFrom = randomIntBetween(0, bytesReference.length()); + int sliceLength = randomIntBetween(0, bytesReference.length() - sliceFrom); + BytesReference slice = bytesReference.slice(sliceFrom, sliceLength); + + assertEquals(bytesReference.toBytesRef().compareTo(slice.toBytesRef()), + new BytesArray(bytesReference.toBytesRef(), true).compareTo(new BytesArray(slice.toBytesRef(), true))); + + assertEquals(bytesReference.toBytesRef().compareTo(slice.toBytesRef()), + bytesReference.compareTo(slice)); + assertEquals(slice.toBytesRef().compareTo(bytesReference.toBytesRef()), + slice.compareTo(bytesReference)); + + assertEquals(0, slice.compareTo(new BytesArray(slice.toBytesRef()))); + assertEquals(0, new BytesArray(slice.toBytesRef()).compareTo(slice)); + + final int crazyLength = length + randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); + ReleasableBytesStreamOutput crazyStream = new ReleasableBytesStreamOutput(length, bigarrays); + final int offset = randomIntBetween(0, crazyLength - length); + for (int j = 0; j < offset; j++) { + crazyStream.writeByte((byte) random().nextInt(1 << 8)); + } + bytesReference.writeTo(crazyStream); + for (int j = crazyStream.size(); j < crazyLength; j++) { + crazyStream.writeByte((byte) random().nextInt(1 << 8)); + } + PagedBytesReference crazyReference = crazyStream.bytes(); + + assertFalse(crazyReference.compareTo(bytesReference) == 0); + assertEquals(0, crazyReference.slice(offset, length).compareTo( + bytesReference)); + assertEquals(0, bytesReference.compareTo( + crazyReference.slice(offset, length))); + } + } + + public static BytesRef getSinglePageOrNull(BytesReference ref) throws IOException { + if (ref.length() > 0) { + BytesRefIterator iterator = ref.iterator(); + BytesRef next = iterator.next(); + BytesRef retVal = next.clone(); + if (iterator.next() == null) { + return retVal; + } + } else { + return new BytesRef(); + } + return null; + } + + public static int getNumPages(BytesReference ref) throws IOException { + int num = 0; + if (ref.length() > 0) { + BytesRefIterator iterator = ref.iterator(); + while(iterator.next() != null) { + num++; + } + } + return num; + } + + + public void testBasicEquals() { + final int len = randomIntBetween(0, randomBoolean() ? 10: 100000); + final int offset1 = randomInt(5); + final byte[] array1 = new byte[offset1 + len + randomInt(5)]; + random().nextBytes(array1); + final int offset2 = randomInt(offset1); + final byte[] array2 = Arrays.copyOfRange(array1, offset1 - offset2, array1.length); + + final BytesArray b1 = new BytesArray(array1, offset1, len); + final BytesArray b2 = new BytesArray(array2, offset2, len); + assertEquals(b1, b2); + assertEquals(Arrays.hashCode(BytesReference.toBytes(b1)), b1.hashCode()); + assertEquals(Arrays.hashCode(BytesReference.toBytes(b2)), b2.hashCode()); + + // test same instance + assertEquals(b1, b1); + assertEquals(b2, b2); + + if (len > 0) { + // test different length + BytesArray differentLen = new BytesArray(array1, offset1, randomInt(len - 1)); + assertNotEquals(b1, differentLen); + + // test changed bytes + array1[offset1 + randomInt(len - 1)] += 13; + assertNotEquals(b1, b2); + } + } } diff --git a/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java b/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java index 61d24ef44c3..fff030200b7 100644 --- a/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java +++ b/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java @@ -32,10 +32,28 @@ public class BytesArrayTests extends AbstractBytesReferenceTestCase { out.writeByte((byte) random().nextInt(1 << 8)); } assertEquals(length, out.size()); - BytesArray ref = out.bytes().toBytesArray(); + BytesArray ref = new BytesArray(out.bytes().toBytesRef()); assertEquals(length, ref.length()); assertTrue(ref instanceof BytesArray); assertThat(ref.length(), Matchers.equalTo(length)); return ref; } + + public void testArray() throws IOException { + int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))}; + + for (int i = 0; i < sizes.length; i++) { + BytesArray pbr = (BytesArray) newBytesReference(sizes[i]); + byte[] array = pbr.array(); + assertNotNull(array); + assertEquals(sizes[i], array.length); + assertSame(array, pbr.array()); + } + } + + public void testArrayOffset() throws IOException { + int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5)); + BytesArray pbr = (BytesArray) newBytesReference(length); + assertEquals(0, pbr.offset()); + } } diff --git a/core/src/test/java/org/elasticsearch/common/bytes/BytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/BytesReferenceTests.java deleted file mode 100644 index 60f4983dd19..00000000000 --- a/core/src/test/java/org/elasticsearch/common/bytes/BytesReferenceTests.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.bytes; - - -import org.elasticsearch.test.ESTestCase; - -import java.util.Arrays; - -public class BytesReferenceTests extends ESTestCase { - - public void testEquals() { - final int len = randomIntBetween(0, randomBoolean() ? 10: 100000); - final int offset1 = randomInt(5); - final byte[] array1 = new byte[offset1 + len + randomInt(5)]; - random().nextBytes(array1); - final int offset2 = randomInt(offset1); - final byte[] array2 = Arrays.copyOfRange(array1, offset1 - offset2, array1.length); - - final BytesArray b1 = new BytesArray(array1, offset1, len); - final BytesArray b2 = new BytesArray(array2, offset2, len); - assertTrue(BytesReference.Helper.bytesEqual(b1, b2)); - assertTrue(BytesReference.Helper.bytesEquals(b1, b2)); - assertEquals(Arrays.hashCode(b1.toBytes()), b1.hashCode()); - assertEquals(BytesReference.Helper.bytesHashCode(b1), BytesReference.Helper.slowHashCode(b2)); - - // test same instance - assertTrue(BytesReference.Helper.bytesEqual(b1, b1)); - assertTrue(BytesReference.Helper.bytesEquals(b1, b1)); - assertEquals(BytesReference.Helper.bytesHashCode(b1), BytesReference.Helper.slowHashCode(b1)); - - if (len > 0) { - // test different length - BytesArray differentLen = new BytesArray(array1, offset1, randomInt(len - 1)); - assertFalse(BytesReference.Helper.bytesEqual(b1, differentLen)); - - // test changed bytes - array1[offset1 + randomInt(len - 1)] += 13; - assertFalse(BytesReference.Helper.bytesEqual(b1, b2)); - assertFalse(BytesReference.Helper.bytesEquals(b1, b2)); - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java new file mode 100644 index 00000000000..aec957aba68 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java @@ -0,0 +1,110 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.bytes; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class CompositeBytesReferenceTests extends AbstractBytesReferenceTestCase { + @Override + protected BytesReference newBytesReference(int length) throws IOException { + // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content + List referenceList = newRefList(length); + BytesReference ref = new CompositeBytesReference(referenceList.toArray(new BytesReference[0])); + assertEquals(length, ref.length()); + return ref; + } + + private List newRefList(int length) throws IOException { + List referenceList = new ArrayList<>(); + for (int i = 0; i < length;) { + int remaining = length-i; + int sliceLength = randomIntBetween(1, remaining); + ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(sliceLength, bigarrays); + for (int j = 0; j < sliceLength; j++) { + out.writeByte((byte) random().nextInt(1 << 8)); + } + assertEquals(sliceLength, out.size()); + referenceList.add(out.bytes()); + i+=sliceLength; + } + return referenceList; + } + + public void testCompositeBuffer() throws IOException { + List referenceList = newRefList(randomIntBetween(1, PAGE_SIZE * 2)); + BytesReference ref = new CompositeBytesReference(referenceList.toArray(new BytesReference[0])); + BytesRefIterator iterator = ref.iterator(); + BytesRefBuilder builder = new BytesRefBuilder(); + + for (BytesReference reference : referenceList) { + BytesRefIterator innerIter = reference.iterator(); // sometimes we have a paged ref - pull an iter and walk all pages! + BytesRef scratch; + while ((scratch = innerIter.next()) != null) { + BytesRef next = iterator.next(); + assertNotNull(next); + assertEquals(next, scratch); + builder.append(next); + } + + } + assertNull(iterator.next()); + + int offset = 0; + for (BytesReference reference : referenceList) { + assertEquals(reference, ref.slice(offset, reference.length())); + int probes = randomIntBetween(Math.min(10, reference.length()), reference.length()); + for (int i = 0; i < probes; i++) { + int index = randomIntBetween(0, reference.length()-1); + assertEquals(ref.get(offset + index), reference.get(index)); + } + offset += reference.length(); + } + + BytesArray array = new BytesArray(builder.toBytesRef()); + assertEquals(array, ref); + assertEquals(array.hashCode(), ref.hashCode()); + + BytesStreamOutput output = new BytesStreamOutput(); + ref.writeTo(output); + assertEquals(array, output.bytes()); + } + + @Override + public void testToBytesRefSharedPage() throws IOException { + // CompositeBytesReference doesn't share pages + } + + @Override + public void testSliceArrayOffset() throws IOException { + // the assertions in this test only work on no-composite buffers + } + + @Override + public void testSliceToBytesRef() throws IOException { + // CompositeBytesReference shifts offsets + } +} diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java index 5a299d82de8..6ae2b3cf943 100644 --- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java +++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java @@ -50,15 +50,15 @@ public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase { return ref; } - public void testToBytesArrayMaterializedPages() throws IOException { + public void testToBytesRefMaterializedPages() throws IOException { // we need a length != (n * pagesize) to avoid page sharing at boundaries int length = 0; while ((length % PAGE_SIZE) == 0) { length = randomIntBetween(PAGE_SIZE, PAGE_SIZE * randomIntBetween(2, 5)); } BytesReference pbr = newBytesReference(length); - BytesArray ba = pbr.toBytesArray(); - BytesArray ba2 = pbr.toBytesArray(); + BytesArray ba = new BytesArray(pbr.toBytesRef()); + BytesArray ba2 = new BytesArray(pbr.toBytesRef()); assertNotNull(ba); assertNotNull(ba2); assertEquals(pbr.length(), ba.length()); @@ -67,23 +67,23 @@ public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase { assertNotSame(ba.array(), ba2.array()); } - public void testArray() throws IOException { + public void testSinglePage() throws IOException { int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))}; for (int i = 0; i < sizes.length; i++) { BytesReference pbr = newBytesReference(sizes[i]); // verify that array() is cheap for small payloads if (sizes[i] <= PAGE_SIZE) { - byte[] array = pbr.array(); + BytesRef page = getSinglePageOrNull(pbr); + assertNotNull(page); + byte[] array = page.bytes; assertNotNull(array); assertEquals(sizes[i], array.length); - assertSame(array, pbr.array()); + assertSame(array, page.bytes); } else { - try { - pbr.array(); - fail("expected IllegalStateException"); - } catch (IllegalStateException isx) { - // expected + BytesRef page = getSinglePageOrNull(pbr); + if (pbr.length() > 0) { + assertNull(page); } } } @@ -94,22 +94,42 @@ public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase { for (int i = 0; i < sizes.length; i++) { BytesReference pbr = newBytesReference(sizes[i]); - byte[] bytes = pbr.toBytes(); + byte[] bytes = BytesReference.toBytes(pbr); assertEquals(sizes[i], bytes.length); // verify that toBytes() is cheap for small payloads if (sizes[i] <= PAGE_SIZE) { - assertSame(bytes, pbr.toBytes()); + assertSame(bytes, BytesReference.toBytes(pbr)); } else { - assertNotSame(bytes, pbr.toBytes()); + assertNotSame(bytes, BytesReference.toBytes(pbr)); } } } - public void testHasArray() throws IOException { + public void testHasSinglePage() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(1, 3)); BytesReference pbr = newBytesReference(length); // must return true for <= pagesize - assertEquals(length <= PAGE_SIZE, pbr.hasArray()); + assertEquals(length <= PAGE_SIZE, getNumPages(pbr) == 1); + } + + public void testEquals() { + int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5)); + ByteArray ba1 = bigarrays.newByteArray(length, false); + ByteArray ba2 = bigarrays.newByteArray(length, false); + + // copy contents + for (long i = 0; i < length; i++) { + ba2.set(i, ba1.get(i)); + } + + // get refs & compare + BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length); + BytesReference pbr2 = new PagedBytesReference(bigarrays, ba2, length); + assertEquals(pbr, pbr2); + int offsetToFlip = randomIntBetween(0, length - 1); + int value = ~Byte.toUnsignedInt(ba1.get(offsetToFlip)); + ba2.set(offsetToFlip, (byte)value); + assertNotEquals(pbr, pbr2); } } diff --git a/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java b/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java index 72866d082ae..0ce95077965 100644 --- a/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java +++ b/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java @@ -91,8 +91,8 @@ public class DeflateCompressedXContentTests extends ESTestCase { // of different size are being used assertFalse(b1.equals(b2)); // we used the compressed representation directly and did not recompress - assertArrayEquals(b1.toBytes(), new CompressedXContent(b1).compressed()); - assertArrayEquals(b2.toBytes(), new CompressedXContent(b2).compressed()); + assertArrayEquals(BytesReference.toBytes(b1), new CompressedXContent(b1).compressed()); + assertArrayEquals(BytesReference.toBytes(b2), new CompressedXContent(b2).compressed()); // but compressedstring instances are still equal assertEquals(new CompressedXContent(b1), new CompressedXContent(b2)); } diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java index 407c9790dbe..416299f8e7e 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java @@ -46,7 +46,7 @@ public class GeoDistanceTests extends ESTestCase { GeoDistance geoDistance = randomFrom(GeoDistance.PLANE, GeoDistance.FACTOR, GeoDistance.ARC, GeoDistance.SLOPPY_ARC); try (BytesStreamOutput out = new BytesStreamOutput()) { geoDistance.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) {; + try (StreamInput in = out.bytes().streamInput()) {; GeoDistance copy = GeoDistance.readFromStream(in); assertEquals(copy.toString() + " vs. " + geoDistance.toString(), copy, geoDistance); } @@ -60,7 +60,7 @@ public class GeoDistanceTests extends ESTestCase { } else { out.writeVInt(randomIntBetween(Integer.MIN_VALUE, -1)); } - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { GeoDistance.readFromStream(in); } catch (IOException e) { assertThat(e.getMessage(), containsString("Unknown GeoDistance ordinal [")); diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java index 566d2148cae..76376a4d30d 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java @@ -56,7 +56,7 @@ import static org.elasticsearch.common.geo.builders.ShapeBuilder.SPATIAL_CONTEXT */ public class GeoJSONShapeParserTests extends ESTestCase { - private final static GeometryFactory GEOMETRY_FACTORY = SPATIAL_CONTEXT.getGeometryFactory(); + private static final GeometryFactory GEOMETRY_FACTORY = SPATIAL_CONTEXT.getGeometryFactory(); public void testParse_simplePoint() throws IOException { String pointGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Point") diff --git a/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java b/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java index 6ee6a4fff83..e4eaa17874c 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java @@ -39,21 +39,21 @@ public class ShapeRelationTests extends ESTestCase { public void testwriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { ShapeRelation.INTERSECTS.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { ShapeRelation.DISJOINT.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { ShapeRelation.WITHIN.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(2)); } } @@ -62,19 +62,19 @@ public class ShapeRelationTests extends ESTestCase { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(ShapeRelation.readFromStream(in), equalTo(ShapeRelation.INTERSECTS)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(ShapeRelation.readFromStream(in), equalTo(ShapeRelation.DISJOINT)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(2); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(ShapeRelation.readFromStream(in), equalTo(ShapeRelation.WITHIN)); } } @@ -83,7 +83,7 @@ public class ShapeRelationTests extends ESTestCase { public void testInvalidReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(randomIntBetween(3, Integer.MAX_VALUE)); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { ShapeRelation.readFromStream(in); fail("Expected IOException"); } catch(IOException e) { diff --git a/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java b/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java index c2f29e6ecd7..b6eae97932f 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java @@ -38,14 +38,14 @@ public class SpatialStrategyTests extends ESTestCase { public void testwriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { SpatialStrategy.TERM.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { SpatialStrategy.RECURSIVE.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } @@ -54,13 +54,13 @@ public class SpatialStrategyTests extends ESTestCase { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(SpatialStrategy.readFromStream(in), equalTo(SpatialStrategy.TERM)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(SpatialStrategy.readFromStream(in), equalTo(SpatialStrategy.RECURSIVE)); } } @@ -69,7 +69,7 @@ public class SpatialStrategyTests extends ESTestCase { public void testInvalidReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE)); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { SpatialStrategy.readFromStream(in); fail("Expected IOException"); } catch(IOException e) { diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java index 9cbd4bb769d..4003a96e26f 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java @@ -137,7 +137,7 @@ public abstract class AbstractShapeBuilderTestCase exte static ShapeBuilder copyShape(ShapeBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return namedWriteableRegistry.getReader(ShapeBuilder.class, original.getWriteableName()).read(in); } } diff --git a/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java index 5c6c1e1789b..76b52c08a85 100644 --- a/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java @@ -84,7 +84,7 @@ public class StreamsTests extends ESTestCase { byte stuff[] = new byte[] { 0, 1, 2, 3 }; BytesRef stuffRef = new BytesRef(stuff, 2, 2); BytesArray stuffArray = new BytesArray(stuffRef); - StreamInput input = StreamInput.wrap(stuffArray); + StreamInput input = stuffArray.streamInput(); assertEquals(2, input.read()); assertEquals(3, input.read()); assertEquals(-1, input.read()); diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java b/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java index a4d15173a7c..dc57b0c70d4 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java @@ -60,7 +60,7 @@ public abstract class AbstractWriteableEnumTestCase extends ESTestCase { protected static void assertWriteToStream(final Writeable writeableEnum, final int ordinal) throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { writeableEnum.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(ordinal)); } } @@ -70,7 +70,7 @@ public abstract class AbstractWriteableEnumTestCase extends ESTestCase { protected void assertReadFromStream(final int ordinal, final Writeable expected) throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(ordinal); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(reader.read(in), equalTo(expected)); } } diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java index 9fcbb708156..94f07369770 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.common.io.stream; import org.apache.lucene.util.Constants; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.util.BigArrays; @@ -48,7 +49,7 @@ public class BytesStreamsTests extends ESTestCase { // test empty stream to array assertEquals(0, out.size()); - assertEquals(0, out.bytes().toBytes().length); + assertEquals(0, out.bytes().length()); out.close(); } @@ -63,7 +64,7 @@ public class BytesStreamsTests extends ESTestCase { // write single byte out.writeByte(expectedData[0]); assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -80,7 +81,7 @@ public class BytesStreamsTests extends ESTestCase { } assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -108,14 +109,14 @@ public class BytesStreamsTests extends ESTestCase { byte[] expectedData = randomizedByteArrayWithSize(expectedSize); out.writeBytes(expectedData); assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); // bulk-write again with actual bytes expectedSize = 10; expectedData = randomizedByteArrayWithSize(expectedSize); out.writeBytes(expectedData); assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -130,7 +131,7 @@ public class BytesStreamsTests extends ESTestCase { out.writeBytes(expectedData); assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -149,7 +150,7 @@ public class BytesStreamsTests extends ESTestCase { // now write the rest - more than fits into the remaining first page out.writeBytes(expectedData, initialOffset, additionalLength); assertEquals(expectedData.length, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -168,7 +169,7 @@ public class BytesStreamsTests extends ESTestCase { // ie. we cross over into a third out.writeBytes(expectedData, initialOffset, additionalLength); assertEquals(expectedData.length, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -185,7 +186,7 @@ public class BytesStreamsTests extends ESTestCase { } assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -202,7 +203,7 @@ public class BytesStreamsTests extends ESTestCase { } assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -219,7 +220,7 @@ public class BytesStreamsTests extends ESTestCase { } assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -235,7 +236,7 @@ public class BytesStreamsTests extends ESTestCase { out.seek(position += BigArrays.BYTE_PAGE_SIZE + 10); out.seek(position += BigArrays.BYTE_PAGE_SIZE * 2); assertEquals(position, out.position()); - assertEquals(position, out.bytes().toBytes().length); + assertEquals(position, BytesReference.toBytes(out.bytes()).length); out.close(); } @@ -288,8 +289,8 @@ public class BytesStreamsTests extends ESTestCase { out.writeTimeZone(DateTimeZone.forID("CET")); out.writeOptionalTimeZone(DateTimeZone.getDefault()); out.writeOptionalTimeZone(null); - final byte[] bytes = out.bytes().toBytes(); - StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + final byte[] bytes = BytesReference.toBytes(out.bytes()); + StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); assertEquals(in.available(), bytes.length); assertThat(in.readBoolean(), equalTo(false)); assertThat(in.readByte(), equalTo((byte)1)); @@ -328,7 +329,7 @@ public class BytesStreamsTests extends ESTestCase { namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new); TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); - byte[] bytes = out.bytes().toBytes(); + byte[] bytes = BytesReference.toBytes(out.bytes()); StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); assertEquals(in.available(), bytes.length); BaseNamedWriteable namedWriteableOut = in.readNamedWriteable(BaseNamedWriteable.class); @@ -348,7 +349,7 @@ public class BytesStreamsTests extends ESTestCase { public void testNamedWriteableUnknownCategory() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); out.writeNamedWriteable(new TestNamedWriteable("test1", "test2")); - StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), new NamedWriteableRegistry()); + StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), new NamedWriteableRegistry()); //no named writeable registered with given name, can write but cannot read it back IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class)); assertThat(e.getMessage(), equalTo("unknown named writeable category [" + BaseNamedWriteable.class.getName() + "]")); @@ -368,7 +369,7 @@ public class BytesStreamsTests extends ESTestCase { public void writeTo(StreamOutput out) throws IOException { } }); - StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), namedWriteableRegistry); + StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(BytesReference.toBytes(out.bytes())), namedWriteableRegistry); try { //no named writeable registered with given name under test category, can write but cannot read it back in.readNamedWriteable(BaseNamedWriteable.class); @@ -382,7 +383,7 @@ public class BytesStreamsTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); TestNamedWriteable testNamedWriteable = new TestNamedWriteable("test1", "test2"); out.writeNamedWriteable(testNamedWriteable); - StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); try { in.readNamedWriteable(BaseNamedWriteable.class); fail("Expected UnsupportedOperationException"); @@ -397,7 +398,7 @@ public class BytesStreamsTests extends ESTestCase { namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, (StreamInput in) -> null); TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); - byte[] bytes = out.bytes().toBytes(); + byte[] bytes = BytesReference.toBytes(out.bytes()); StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); assertEquals(in.available(), bytes.length); IOException e = expectThrows(IOException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class)); @@ -407,7 +408,7 @@ public class BytesStreamsTests extends ESTestCase { public void testOptionalWriteableReaderReturnsNull() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); out.writeOptionalWriteable(new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10))); - StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); IOException e = expectThrows(IOException.class, () -> in.readOptionalWriteable((StreamInput ignored) -> null)); assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream.")); } @@ -423,7 +424,7 @@ public class BytesStreamsTests extends ESTestCase { }); TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); - byte[] bytes = out.bytes().toBytes(); + byte[] bytes = BytesReference.toBytes(out.bytes()); StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); assertEquals(in.available(), bytes.length); AssertionError e = expectThrows(AssertionError.class, () -> in.readNamedWriteable(BaseNamedWriteable.class)); @@ -442,7 +443,7 @@ public class BytesStreamsTests extends ESTestCase { final BytesStreamOutput out = new BytesStreamOutput(); out.writeStreamableList(expected); - final StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + final StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); List loaded = in.readStreamableList(TestStreamable::new); @@ -458,7 +459,7 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - private static abstract class BaseNamedWriteable implements NamedWriteable { + private abstract static class BaseNamedWriteable implements NamedWriteable { } @@ -537,7 +538,7 @@ public class BytesStreamsTests extends ESTestCase { // toByteArray() must fail try { - out.bytes().toBytes(); + BytesReference.toBytes(out.bytes()); fail("expected IllegalStateException: stream closed"); } catch (IllegalStateException iex1) { @@ -558,7 +559,7 @@ public class BytesStreamsTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); GeoPoint geoPoint = new GeoPoint(randomDouble(), randomDouble()); out.writeGenericValue(geoPoint); - StreamInput wrap = StreamInput.wrap(out.bytes()); + StreamInput wrap = out.bytes().streamInput(); GeoPoint point = (GeoPoint) wrap.readGenericValue(); assertEquals(point, geoPoint); } @@ -566,7 +567,7 @@ public class BytesStreamsTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); GeoPoint geoPoint = new GeoPoint(randomDouble(), randomDouble()); out.writeGeoPoint(geoPoint); - StreamInput wrap = StreamInput.wrap(out.bytes()); + StreamInput wrap = out.bytes().streamInput(); GeoPoint point = wrap.readGeoPoint(); assertEquals(point, geoPoint); } diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java index aa6016774b0..06d39398c8e 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java @@ -19,13 +19,13 @@ package org.elasticsearch.common.io.stream; -import org.elasticsearch.common.bytes.ByteBufferBytesReference; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.test.ESTestCase; import java.io.ByteArrayInputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -61,8 +61,8 @@ public class StreamTests extends ESTestCase { for (Tuple value : values) { BytesStreamOutput out = new BytesStreamOutput(); out.writeZLong(value.v1()); - assertArrayEquals(Long.toString(value.v1()), value.v2(), out.bytes().toBytes()); - ByteBufferBytesReference bytes = new ByteBufferBytesReference(ByteBuffer.wrap(value.v2())); + assertArrayEquals(Long.toString(value.v1()), value.v2(), BytesReference.toBytes(out.bytes())); + BytesReference bytes = new BytesArray(value.v2()); assertEquals(Arrays.toString(value.v2()), (long)value.v1(), bytes.streamInput().readZLong()); } } @@ -143,7 +143,7 @@ public class StreamTests extends ESTestCase { assertThat(targetArray, equalTo(sourceArray)); } - final static class WriteableString implements Writeable { + static final class WriteableString implements Writeable { final String string; public WriteableString(String string) { diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java index 9ae0beadf59..131636e2011 100644 --- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java +++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java @@ -59,7 +59,7 @@ public class NetworkModuleTests extends ModuleTestCase { } } - static class FakeHttpTransport extends AbstractLifecycleComponent implements HttpServerTransport { + static class FakeHttpTransport extends AbstractLifecycleComponent implements HttpServerTransport { public FakeHttpTransport() { super(null); } @@ -95,7 +95,7 @@ public class NetworkModuleTests extends ModuleTestCase { static class FakeCatRestHandler extends AbstractCatAction { public FakeCatRestHandler() { - super(null, null); + super(null); } @Override protected void doRequest(RestRequest request, RestChannel channel, NodeClient client) {} diff --git a/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java b/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java index 45db5a33d21..1a3fa4db137 100644 --- a/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java +++ b/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java @@ -51,7 +51,7 @@ public class BoundTransportAddressTests extends ESTestCase { // serialize BytesStreamOutput streamOutput = new BytesStreamOutput(); transportAddress.writeTo(streamOutput); - StreamInput in = ByteBufferStreamInput.wrap(streamOutput.bytes()); + StreamInput in = streamOutput.bytes().streamInput(); BoundTransportAddress serializedAddress; if (randomBoolean()) { diff --git a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java index f9a4d3f22af..7c5463baed2 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java @@ -82,7 +82,7 @@ public class DistanceUnitTests extends ESTestCase { for (DistanceUnit unit : DistanceUnit.values()) { try (BytesStreamOutput out = new BytesStreamOutput()) { unit.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat("Roundtrip serialisation failed.", DistanceUnit.readFromStream(in), equalTo(unit)); } } diff --git a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java index 2b5a7c00e5d..3f6f1848fd8 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java @@ -145,7 +145,7 @@ public class FuzzinessTests extends ESTestCase { private static Fuzziness doSerializeRoundtrip(Fuzziness in) throws IOException { BytesStreamOutput output = new BytesStreamOutput(); in.writeTo(output); - StreamInput streamInput = StreamInput.wrap(output.bytes()); + StreamInput streamInput = output.bytes().streamInput(); return new Fuzziness(streamInput); } } diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java index 78afc9e514f..003d78ce42e 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java @@ -161,7 +161,7 @@ public class TimeValueTests extends ESTestCase { value.writeTo(out); assertEquals(expectedSize, out.size()); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); TimeValue inValue = new TimeValue(in); assertThat(inValue, equalTo(value)); diff --git a/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java b/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java index a89cb48c37a..729c431d2b2 100644 --- a/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java @@ -132,7 +132,7 @@ public class CancellableThreadsTests extends ESTestCase { public void testCancellableThreads() throws InterruptedException { Thread[] threads = new Thread[randomIntBetween(3, 10)]; final TestPlan[] plans = new TestPlan[threads.length]; - final Throwable[] throwables = new Throwable[threads.length]; + final Exception[] exceptions = new Exception[threads.length]; final boolean[] interrupted = new boolean[threads.length]; final CancellableThreads cancellableThreads = new CancellableThreads(); final CountDownLatch readyForCancel = new CountDownLatch(threads.length); @@ -153,8 +153,8 @@ public class CancellableThreadsTests extends ESTestCase { } else { cancellableThreads.execute(new TestRunnable(plan, readyForCancel)); } - } catch (Throwable t) { - throwables[plan.id] = t; + } catch (Exception e) { + exceptions[plan.id] = e; } if (plan.exceptBeforeCancel || plan.exitBeforeCancel) { // we have to mark we're ready now (actually done). @@ -176,19 +176,19 @@ public class CancellableThreadsTests extends ESTestCase { TestPlan plan = plans[i]; final Class exceptionClass = plan.ioException ? IOCustomException.class : CustomException.class; if (plan.exceptBeforeCancel) { - assertThat(throwables[i], Matchers.instanceOf(exceptionClass)); + assertThat(exceptions[i], Matchers.instanceOf(exceptionClass)); } else if (plan.exitBeforeCancel) { - assertNull(throwables[i]); + assertNull(exceptions[i]); } else { // in all other cases, we expect a cancellation exception. - assertThat(throwables[i], Matchers.instanceOf(CancellableThreads.ExecutionCancelledException.class)); + assertThat(exceptions[i], Matchers.instanceOf(CancellableThreads.ExecutionCancelledException.class)); if (plan.exceptAfterCancel) { - assertThat(throwables[i].getSuppressed(), + assertThat(exceptions[i].getSuppressed(), Matchers.arrayContaining( Matchers.instanceOf(exceptionClass) )); } else { - assertThat(throwables[i].getSuppressed(), Matchers.emptyArray()); + assertThat(exceptions[i].getSuppressed(), Matchers.emptyArray()); } } assertThat(interrupted[plan.id], Matchers.equalTo(plan.presetInterrupt)); diff --git a/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java b/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java index 26d6af1cd5f..5302ba8d55c 100644 --- a/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java @@ -67,7 +67,7 @@ public class IndexFolderUpgraderTests extends ESTestCase { public void testUpgradeCustomDataPath() throws IOException { Path customPath = createTempDir(); final Settings nodeSettings = Settings.builder() - .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()) + .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()) .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), customPath.toAbsolutePath().toString()).build(); try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) { final Index index = new Index(randomAsciiOfLength(10), UUIDs.randomBase64UUID()); @@ -96,7 +96,7 @@ public class IndexFolderUpgraderTests extends ESTestCase { public void testPartialUpgradeCustomDataPath() throws IOException { Path customPath = createTempDir(); final Settings nodeSettings = Settings.builder() - .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()) + .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()) .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), customPath.toAbsolutePath().toString()).build(); try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) { final Index index = new Index(randomAsciiOfLength(10), UUIDs.randomBase64UUID()); @@ -136,7 +136,7 @@ public class IndexFolderUpgraderTests extends ESTestCase { public void testUpgrade() throws IOException { final Settings nodeSettings = Settings.builder() - .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build(); + .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build(); try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) { final Index index = new Index(randomAsciiOfLength(10), UUIDs.randomBase64UUID()); Settings settings = Settings.builder() @@ -159,7 +159,7 @@ public class IndexFolderUpgraderTests extends ESTestCase { public void testUpgradeIndices() throws IOException { final Settings nodeSettings = Settings.builder() - .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build(); + .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build(); try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) { Map> indexSettingsMap = new HashMap<>(); for (int i = 0; i < randomIntBetween(2, 5); i++) { @@ -256,7 +256,7 @@ public class IndexFolderUpgraderTests extends ESTestCase { .numberOfReplicas(0) .build(); try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) { - IndexMetaData.FORMAT.write(indexState, 1, nodeEnvironment.indexPaths(index)); + IndexMetaData.FORMAT.write(indexState, nodeEnvironment.indexPaths(index)); assertFalse(IndexFolderUpgrader.needsUpgrade(index, index.getUUID())); } } @@ -305,7 +305,7 @@ public class IndexFolderUpgraderTests extends ESTestCase { for (int i = 0; i < nodePaths.length; i++) { oldIndexPaths[i] = nodePaths[i].indicesPath.resolve(indexSettings.getIndex().getName()); } - IndexMetaData.FORMAT.write(indexSettings.getIndexMetaData(), 1, oldIndexPaths); + IndexMetaData.FORMAT.write(indexSettings.getIndexMetaData(), oldIndexPaths); for (int id = 0; id < indexSettings.getNumberOfShards(); id++) { Path oldIndexPath = randomFrom(oldIndexPaths); ShardId shardId = new ShardId(indexSettings.getIndex(), id); @@ -316,7 +316,7 @@ public class IndexFolderUpgraderTests extends ESTestCase { writeShard(shardId, oldIndexPath, numIdxFiles, numTranslogFiles); } ShardStateMetaData state = new ShardStateMetaData(true, indexSettings.getUUID(), AllocationId.newInitializing()); - ShardStateMetaData.FORMAT.write(state, 1, oldIndexPath.resolve(String.valueOf(shardId.getId()))); + ShardStateMetaData.FORMAT.write(state, oldIndexPath.resolve(String.valueOf(shardId.getId()))); } } diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java index 4c2e4700943..02adb783197 100644 --- a/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java @@ -48,7 +48,7 @@ public class AbstractLifecycleRunnableTests extends ESTestCase { AbstractLifecycleRunnable runnable = new AbstractLifecycleRunnable(lifecycle, logger) { @Override - public void onFailure(Throwable t) { + public void onFailure(Exception e) { fail("It should not fail"); } @@ -77,7 +77,7 @@ public class AbstractLifecycleRunnableTests extends ESTestCase { AbstractLifecycleRunnable runnable = new AbstractLifecycleRunnable(lifecycle, logger) { @Override - public void onFailure(Throwable t) { + public void onFailure(Exception e) { fail("It should not fail"); } @@ -106,7 +106,7 @@ public class AbstractLifecycleRunnableTests extends ESTestCase { AbstractLifecycleRunnable runnable = new AbstractLifecycleRunnable(lifecycle, logger) { @Override - public void onFailure(Throwable t) { + public void onFailure(Exception e) { fail("It should not fail"); } @@ -145,7 +145,7 @@ public class AbstractLifecycleRunnableTests extends ESTestCase { AbstractLifecycleRunnable runnable = new AbstractLifecycleRunnable(lifecycle, logger) { @Override - public void onFailure(Throwable t) { + public void onFailure(Exception e) { fail("It should not fail"); } diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractRunnableTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractRunnableTests.java index 54491aade6f..2373b30e1b2 100644 --- a/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractRunnableTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractRunnableTests.java @@ -37,8 +37,8 @@ public class AbstractRunnableTests extends ESTestCase { AbstractRunnable runnable = new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - fail("It should not fail"); + public void onFailure(Exception e) { + fail(e.toString()); } @Override @@ -57,8 +57,8 @@ public class AbstractRunnableTests extends ESTestCase { AbstractRunnable runnable = new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - assertSame(exception, t); + public void onFailure(Exception e) { + assertSame(exception, e); } @Override @@ -76,8 +76,8 @@ public class AbstractRunnableTests extends ESTestCase { AbstractRunnable runnable = new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - fail("It should not fail"); + public void onFailure(Exception e) { + fail(e.toString()); } @Override @@ -91,7 +91,7 @@ public class AbstractRunnableTests extends ESTestCase { afterCallable.call(); } catch (Exception e) { - fail("Unexpected for mock."); + fail(e.toString()); } } }; @@ -111,8 +111,8 @@ public class AbstractRunnableTests extends ESTestCase { AbstractRunnable runnable = new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - assertSame(exception, t); + public void onFailure(Exception e) { + assertSame(exception, e); } @Override @@ -126,7 +126,7 @@ public class AbstractRunnableTests extends ESTestCase { afterCallable.call(); } catch (Exception e) { - fail("Unexpected for mock."); + fail(e.toString()); } } }; @@ -142,14 +142,15 @@ public class AbstractRunnableTests extends ESTestCase { AbstractRunnable runnable = new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - assertSame(exception, t); + public void onFailure(Exception e) { + assertSame(exception, e); try { failureCallable.call(); } - catch (Exception e) { - fail("Unexpected for mock."); + catch (Exception inner) { + inner.addSuppressed(e); + fail(inner.toString()); } } @@ -165,8 +166,8 @@ public class AbstractRunnableTests extends ESTestCase { public void testIsForceExecutuonDefaultsFalse() { AbstractRunnable runnable = new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - fail("Not tested"); + public void onFailure(Exception e) { + fail(e.toString()); } @Override diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java index 57da614e689..72db2911fc0 100644 --- a/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java @@ -88,8 +88,8 @@ public class EsExecutorsTests extends ESTestCase { } @Override - public void onFailure(Throwable t) { - throw new AssertionError(t); + public void onFailure(Exception e) { + throw new AssertionError(e); } }); @@ -178,7 +178,7 @@ public class EsExecutorsTests extends ESTestCase { try { barrier.await(); barrier.await(); - } catch (Throwable e) { + } catch (Exception e) { barrier.reset(e); } } @@ -214,7 +214,7 @@ public class EsExecutorsTests extends ESTestCase { try { barrier.await(); barrier.await(); - } catch (Throwable e) { + } catch (Exception e) { barrier.reset(e); } } diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java index 9338beccb9a..c5d0ec4257e 100644 --- a/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java @@ -88,7 +88,7 @@ public class RefCountedTests extends ESTestCase { final MyRefCounted counted = new MyRefCounted(); Thread[] threads = new Thread[randomIntBetween(2, 5)]; final CountDownLatch latch = new CountDownLatch(1); - final CopyOnWriteArrayList exceptions = new CopyOnWriteArrayList<>(); + final CopyOnWriteArrayList exceptions = new CopyOnWriteArrayList<>(); for (int i = 0; i < threads.length; i++) { threads[i] = new Thread() { @Override @@ -103,7 +103,7 @@ public class RefCountedTests extends ESTestCase { counted.decRef(); } } - } catch (Throwable e) { + } catch (Exception e) { exceptions.add(e); } } diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainerTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainerTests.java deleted file mode 100644 index 83db2d4a7c6..00000000000 --- a/core/src/test/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainerTests.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.util.concurrent; - -import org.elasticsearch.common.lease.Releasable; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.test.ESTestCase; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.nullValue; - -public class SuspendableRefContainerTests extends ESTestCase { - - public void testBasicAcquire() throws InterruptedException { - SuspendableRefContainer refContainer = new SuspendableRefContainer(); - assertThat(refContainer.activeRefs(), equalTo(0)); - - Releasable lock1 = randomLockingMethod(refContainer); - assertThat(refContainer.activeRefs(), equalTo(1)); - Releasable lock2 = randomLockingMethod(refContainer); - assertThat(refContainer.activeRefs(), equalTo(2)); - lock1.close(); - assertThat(refContainer.activeRefs(), equalTo(1)); - lock1.close(); // check idempotence - assertThat(refContainer.activeRefs(), equalTo(1)); - lock2.close(); - assertThat(refContainer.activeRefs(), equalTo(0)); - } - - public void testAcquisitionBlockingBlocksNewAcquisitions() throws InterruptedException { - SuspendableRefContainer refContainer = new SuspendableRefContainer(); - assertThat(refContainer.activeRefs(), equalTo(0)); - - try (Releasable block = refContainer.blockAcquisition()) { - assertThat(refContainer.activeRefs(), equalTo(0)); - assertThat(refContainer.tryAcquire(), nullValue()); - assertThat(refContainer.activeRefs(), equalTo(0)); - } - try (Releasable lock = refContainer.tryAcquire()) { - assertThat(refContainer.activeRefs(), equalTo(1)); - } - - // same with blocking acquire - AtomicBoolean acquired = new AtomicBoolean(); - Thread t = new Thread(() -> { - try (Releasable lock = randomBoolean() ? refContainer.acquire() : refContainer.acquireUninterruptibly()) { - acquired.set(true); - assertThat(refContainer.activeRefs(), equalTo(1)); - } catch (InterruptedException e) { - fail("Interrupted"); - } - }); - try (Releasable block = refContainer.blockAcquisition()) { - assertThat(refContainer.activeRefs(), equalTo(0)); - t.start(); - // check that blocking acquire really blocks - assertThat(acquired.get(), equalTo(false)); - assertThat(refContainer.activeRefs(), equalTo(0)); - } - t.join(); - assertThat(acquired.get(), equalTo(true)); - assertThat(refContainer.activeRefs(), equalTo(0)); - } - - public void testAcquisitionBlockingWaitsOnExistingAcquisitions() throws InterruptedException { - SuspendableRefContainer refContainer = new SuspendableRefContainer(); - - AtomicBoolean acquired = new AtomicBoolean(); - Thread t = new Thread(() -> { - try (Releasable block = refContainer.blockAcquisition()) { - acquired.set(true); - assertThat(refContainer.activeRefs(), equalTo(0)); - } - }); - try (Releasable lock = randomLockingMethod(refContainer)) { - assertThat(refContainer.activeRefs(), equalTo(1)); - t.start(); - assertThat(acquired.get(), equalTo(false)); - assertThat(refContainer.activeRefs(), equalTo(1)); - } - t.join(); - assertThat(acquired.get(), equalTo(true)); - assertThat(refContainer.activeRefs(), equalTo(0)); - } - - private Releasable randomLockingMethod(SuspendableRefContainer refContainer) throws InterruptedException { - switch (randomInt(2)) { - case 0: return refContainer.tryAcquire(); - case 1: return refContainer.acquire(); - case 2: return refContainer.acquireUninterruptibly(); - } - throw new IllegalArgumentException("randomLockingMethod inconsistent"); - } -} diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java index 1a582d48f6b..e6726879513 100644 --- a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java @@ -154,7 +154,7 @@ public class ThreadContextTests extends ESTestCase { assertNull(threadContext.getTransient("ctx.foo")); assertEquals("1", threadContext.getHeader("default")); - threadContext.readHeaders(StreamInput.wrap(out.bytes())); + threadContext.readHeaders(out.bytes().streamInput()); assertEquals("bar", threadContext.getHeader("foo")); assertNull(threadContext.getTransient("ctx.foo")); } @@ -179,14 +179,14 @@ public class ThreadContextTests extends ESTestCase { { Settings otherSettings = Settings.builder().put("request.headers.default", "5").build(); ThreadContext otherhreadContext = new ThreadContext(otherSettings); - otherhreadContext.readHeaders(StreamInput.wrap(out.bytes())); + otherhreadContext.readHeaders(out.bytes().streamInput()); assertEquals("bar", otherhreadContext.getHeader("foo")); assertNull(otherhreadContext.getTransient("ctx.foo")); assertEquals("1", otherhreadContext.getHeader("default")); } } - + public void testSerializeInDifferentContextNoDefaults() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); { @@ -202,7 +202,7 @@ public class ThreadContextTests extends ESTestCase { { Settings otherSettings = Settings.builder().put("request.headers.default", "5").build(); ThreadContext otherhreadContext = new ThreadContext(otherSettings); - otherhreadContext.readHeaders(StreamInput.wrap(out.bytes())); + otherhreadContext.readHeaders(out.bytes().streamInput()); assertEquals("bar", otherhreadContext.getHeader("foo")); assertNull(otherhreadContext.getTransient("ctx.foo")); @@ -294,8 +294,8 @@ public class ThreadContextTests extends ESTestCase { } return new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - throw new RuntimeException(t); + public void onFailure(Exception e) { + throw new RuntimeException(e); } @Override diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java index f3592936765..bef4a047ef5 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java @@ -68,7 +68,7 @@ public class ConstructingObjectParserTests extends ESTestCase { assertEquals(expected.b, parsed.b); assertEquals(expected.c, parsed.c); assertEquals(expected.d, parsed.d); - } catch (Throwable e) { + } catch (Exception e) { // It is convenient to decorate the error message with the json throw new Exception("Error parsing: [" + builder.string() + "]", e); } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java index 159d8a97be4..a8d26e87ecf 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java @@ -35,7 +35,7 @@ import org.elasticsearch.test.ESTestCase; public class ObjectParserTests extends ESTestCase { - private final static ParseFieldMatcherSupplier STRICT_PARSING = () -> ParseFieldMatcher.STRICT; + private static final ParseFieldMatcherSupplier STRICT_PARSING = () -> ParseFieldMatcher.STRICT; public void testBasics() throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser( diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java index 583234461b3..8319873878a 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java @@ -57,11 +57,10 @@ public class XContentFactoryTests extends ESTestCase { builder.endObject(); assertThat(XContentFactory.xContentType(builder.bytes()), equalTo(type)); - BytesArray bytesArray = builder.bytes().toBytesArray(); - assertThat(XContentFactory.xContentType(StreamInput.wrap(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length())), equalTo(type)); + assertThat(XContentFactory.xContentType(builder.bytes().streamInput()), equalTo(type)); // CBOR is binary, cannot use String - if (type != XContentType.CBOR) { + if (type != XContentType.CBOR && type != XContentType.SMILE) { assertThat(XContentFactory.xContentType(builder.string()), equalTo(type)); } } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java index 34944e713bd..fe69fc1f05d 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java @@ -94,7 +94,7 @@ public class XContentBuilderTests extends ESTestCase { xContentBuilder.startObject(); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.endObject(); - assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"foo\":{\"test\":\"value\"}}")); + assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"foo\":{\"test\":\"value\"}}")); } { XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); @@ -102,7 +102,7 @@ public class XContentBuilderTests extends ESTestCase { xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.rawField("foo1", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.endObject(); - assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"}}")); + assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"}}")); } { XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); @@ -110,7 +110,7 @@ public class XContentBuilderTests extends ESTestCase { xContentBuilder.field("test", "value"); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.endObject(); - assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"}}")); + assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"}}")); } { XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); @@ -119,7 +119,7 @@ public class XContentBuilderTests extends ESTestCase { xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.field("test1", "value1"); xContentBuilder.endObject(); - assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"test1\":\"value1\"}")); + assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"test1\":\"value1\"}")); } { XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); @@ -129,7 +129,7 @@ public class XContentBuilderTests extends ESTestCase { xContentBuilder.rawField("foo1", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.field("test1", "value1"); xContentBuilder.endObject(); - assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"},\"test1\":\"value1\"}")); + assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"},\"test1\":\"value1\"}")); } } @@ -161,15 +161,14 @@ public class XContentBuilderTests extends ESTestCase { gen.writeEndObject(); gen.close(); - byte[] data = bos.bytes().toBytes(); - String sData = new String(data, "UTF8"); + String sData = bos.bytes().utf8ToString(); assertThat(sData, equalTo("{\"name\":\"something\", source : { test : \"value\" },\"name2\":\"something2\"}")); } public void testByteConversion() throws Exception { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.startObject().field("test_name", (Byte)(byte)120).endObject(); - assertThat(builder.bytes().toUtf8(), equalTo("{\"test_name\":120}")); + assertThat(builder.bytes().utf8ToString(), equalTo("{\"test_name\":120}")); } public void testDateTypesConversion() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java index bf2dd442b64..efbca114aac 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java @@ -63,7 +63,8 @@ public class JsonVsCborTests extends ESTestCase { xsonGen.close(); jsonGen.close(); - verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes().toBytes()), XContentFactory.xContent(XContentType.CBOR).createParser(xsonOs.bytes().toBytes())); + verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes()), + XContentFactory.xContent(XContentType.CBOR).createParser(xsonOs.bytes())); } private void verifySameTokens(XContentParser parser1, XContentParser parser2) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java index 9e686fe78f1..63b19a63822 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java @@ -63,7 +63,8 @@ public class JsonVsSmileTests extends ESTestCase { xsonGen.close(); jsonGen.close(); - verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes().toBytes()), XContentFactory.xContent(XContentType.SMILE).createParser(xsonOs.bytes().toBytes())); + verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes()), + XContentFactory.xContent(XContentType.SMILE).createParser(xsonOs.bytes())); } private void verifySameTokens(XContentParser parser1, XContentParser parser2) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java index e3d8735e05e..b8b38a543f6 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import java.io.ByteArrayInputStream; import java.io.IOException; import static org.hamcrest.CoreMatchers.is; @@ -45,7 +44,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertNotNull(expected); // Verify that the result is equal to the expected string - assertThat(builder.bytes().toUtf8(), is(expected.bytes().toUtf8())); + assertThat(builder.bytes().utf8ToString(), is(expected.bytes().utf8ToString())); } protected void assertBinary(XContentBuilder expected, XContentBuilder builder) { @@ -1166,15 +1165,15 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase // Test method: rawField(String fieldName, InputStream content) assertXContentBuilder(expectedRawField, - newXContentBuilder().startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject()); assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*", true).startObject().field("foo", 0) - .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + .rawField("raw", raw.streamInput()).endObject()); assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("r*", false).startObject().field("foo", 0) - .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + .rawField("raw", raw.streamInput()).endObject()); assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*", true).startObject().field("foo", 0) - .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + .rawField("raw", raw.streamInput()).endObject()); assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("f*", false).startObject().field("foo", 0) - .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + .rawField("raw", raw.streamInput()).endObject()); } public void testArrays() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java index dd2fe42eb8e..8dbefedb249 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java @@ -142,7 +142,7 @@ public class FilterPathGeneratorFilteringTests extends ESTestCase { } } } - assertThat(os.bytes().toUtf8(), equalTo(replaceQuotes(expected))); + assertThat(os.bytes().utf8ToString(), equalTo(replaceQuotes(expected))); } } diff --git a/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java b/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java index 4efedd9154a..c25a0a6503b 100644 --- a/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java +++ b/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java @@ -56,8 +56,7 @@ public class JacksonLocationTests extends ESTestCase { gen.close(); - byte[] data = os.bytes().toBytes(); - JsonParser parser = new JsonFactory().createParser(data); + JsonParser parser = new JsonFactory().createParser(os.bytes().streamInput()); assertThat(parser.nextToken(), equalTo(JsonToken.START_OBJECT)); assertThat(parser.nextToken(), equalTo(JsonToken.FIELD_NAME)); // "index" diff --git a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java index f6aac190c4b..4ff4c4cd035 100644 --- a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java @@ -21,7 +21,7 @@ package org.elasticsearch.discovery; import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.test.ESTestCase; @@ -40,7 +40,7 @@ import static org.hamcrest.Matchers.not; public class BlockingClusterStatePublishResponseHandlerTests extends ESTestCase { - static private class PublishResponder extends AbstractRunnable { + private static class PublishResponder extends AbstractRunnable { final boolean fail; final DiscoveryNode node; @@ -58,8 +58,8 @@ public class BlockingClusterStatePublishResponseHandlerTests extends ESTestCase } @Override - public void onFailure(Throwable t) { - logger.error("unexpected error", t); + public void onFailure(Exception e) { + logger.error("unexpected error", e); } @Override @@ -77,7 +77,7 @@ public class BlockingClusterStatePublishResponseHandlerTests extends ESTestCase int nodeCount = scaledRandomIntBetween(10, 20); DiscoveryNode[] allNodes = new DiscoveryNode[nodeCount]; for (int i = 0; i < nodeCount; i++) { - DiscoveryNode node = new DiscoveryNode("node_" + i, DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode node = new DiscoveryNode("node_" + i, LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); allNodes[i] = node; } diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java index 7f89acd169e..a1bac928daf 100644 --- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java @@ -170,7 +170,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { return nodes; } - final static Settings DEFAULT_SETTINGS = Settings.builder() + static final Settings DEFAULT_SETTINGS = Settings.builder() .put(FaultDetection.PING_TIMEOUT_SETTING.getKey(), "1s") // for hitting simulated network failures quickly .put(FaultDetection.PING_RETRIES_SETTING.getKey(), "1") // for hitting simulated network failures quickly .put("discovery.zen.join_timeout", "10s") // still long to induce failures but to long so test won't time out @@ -503,8 +503,8 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { } } catch (InterruptedException e) { // fine - semaphore interrupt - } catch (Throwable t) { - logger.info("unexpected exception in background thread of [{}]", t, node); + } catch (AssertionError | Exception e) { + logger.info("unexpected exception in background thread of [{}]", e, node); } } }); @@ -690,8 +690,8 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { - logger.warn("failure [{}]", t, source); + public void onFailure(String source, Exception e) { + logger.warn("failure [{}]", e, source); } }); @@ -960,7 +960,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable t) { + public void onFailure(Exception e) { success.set(false); latch.countDown(); assert false; diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java index 0f93e5d460c..b31b0cbaa55 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java @@ -22,7 +22,7 @@ package org.elasticsearch.discovery.zen; import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.test.ESTestCase; @@ -46,7 +46,7 @@ public class ElectMasterServiceTests extends ESTestCase { if (randomBoolean()) { roles.add(DiscoveryNode.Role.MASTER); } - DiscoveryNode node = new DiscoveryNode("n_" + i, "n_" + i, DummyTransportAddress.INSTANCE, Collections.emptyMap(), + DiscoveryNode node = new DiscoveryNode("n_" + i, "n_" + i, LocalTransportAddress.buildUnique(), Collections.emptyMap(), roles, Version.CURRENT); nodes.add(node); } diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java index 135352343b6..cd2b4eaf2e4 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.RoutingService; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; @@ -32,7 +31,6 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -41,6 +39,7 @@ import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.discovery.zen.membership.MembershipAction; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -50,6 +49,8 @@ import org.junit.Before; import org.junit.BeforeClass; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -68,6 +69,7 @@ import static java.util.Collections.emptySet; import static java.util.Collections.shuffle; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.elasticsearch.test.ClusterServiceUtils.setState; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -99,10 +101,9 @@ public class NodeJoinControllerTests extends ESTestCase { // make sure we have a master setState(clusterService, ClusterState.builder(clusterService.state()).nodes( DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.getId()))); - nodeJoinController = new NodeJoinController(clusterService, new NoopRoutingService(Settings.EMPTY), - new ElectMasterService(Settings.EMPTY), - new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), - Settings.EMPTY); + nodeJoinController = new NodeJoinController(clusterService, new NoopAllocationService(Settings.EMPTY), + new ElectMasterService(Settings.EMPTY), new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), Settings.EMPTY); } @After @@ -198,17 +199,19 @@ public class NodeJoinControllerTests extends ESTestCase { final SimpleFuture electionFuture = new SimpleFuture("master election"); final Thread masterElection = new Thread(new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - logger.error("unexpected error from waitToBeElectedAsMaster", t); - electionFuture.markAsFailed(t); + public void onFailure(Exception e) { + logger.error("unexpected error from waitToBeElectedAsMaster", e); + electionFuture.markAsFailed(e); } @Override protected void doRun() throws Exception { - nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30), new NodeJoinController.ElectionCallback() { + nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30), + new NodeJoinController.ElectionCallback() { @Override public void onElectedAsMaster(ClusterState state) { - assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true)); + assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), + equalTo(true)); electionFuture.markAsDone(); } @@ -246,17 +249,19 @@ public class NodeJoinControllerTests extends ESTestCase { final SimpleFuture electionFuture = new SimpleFuture("master election"); final Thread masterElection = new Thread(new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - logger.error("unexpected error from waitToBeElectedAsMaster", t); - electionFuture.markAsFailed(t); + public void onFailure(Exception e) { + logger.error("unexpected error from waitToBeElectedAsMaster", e); + electionFuture.markAsFailed(e); } @Override protected void doRun() throws Exception { - nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30), new NodeJoinController.ElectionCallback() { + nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30), + new NodeJoinController.ElectionCallback() { @Override public void onElectedAsMaster(ClusterState state) { - assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true)); + assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), + equalTo(true)); electionFuture.markAsDone(); } @@ -298,7 +303,8 @@ public class NodeJoinControllerTests extends ESTestCase { } logger.debug("--> asserting master election didn't finish yet"); - assertThat("election finished after [" + initialJoins + "] master nodes but required joins is [" + requiredJoins + "]", electionFuture.isDone(), equalTo(false)); + assertThat("election finished after [" + initialJoins + "] master nodes but required joins is [" + requiredJoins + "]", + electionFuture.isDone(), equalTo(false)); final int finalJoins = requiredJoins - initialJoins + randomInt(5); nodesToJoin.clear(); @@ -374,7 +380,8 @@ public class NodeJoinControllerTests extends ESTestCase { nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueMillis(1), new NodeJoinController.ElectionCallback() { @Override public void onElectedAsMaster(ClusterState state) { - assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true)); + assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), + equalTo(true)); latch.countDown(); } @@ -403,7 +410,7 @@ public class NodeJoinControllerTests extends ESTestCase { public void testNewClusterStateOnExistingNodeJoin() throws InterruptedException, ExecutionException { ClusterState state = clusterService.state(); final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(state.nodes()); - final DiscoveryNode other_node = new DiscoveryNode("other_node", DummyTransportAddress.INSTANCE, + final DiscoveryNode other_node = new DiscoveryNode("other_node", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); nodesBuilder.put(other_node); setState(clusterService, ClusterState.builder(state).nodes(nodesBuilder)); @@ -425,9 +432,9 @@ public class NodeJoinControllerTests extends ESTestCase { nodes.add(node); threads[i] = new Thread(new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - logger.error("unexpected error in join thread", t); - backgroundExceptions.add(t); + public void onFailure(Exception e) { + logger.error("unexpected error in join thread", e); + backgroundExceptions.add(e); } @Override @@ -468,9 +475,9 @@ public class NodeJoinControllerTests extends ESTestCase { nodes.add(node); threads[i] = new Thread(new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - logger.error("unexpected error in join thread", t); - backgroundExceptions.add(t); + public void onFailure(Exception e) { + logger.error("unexpected error in join thread", e); + backgroundExceptions.add(e); } @Override @@ -492,7 +499,8 @@ public class NodeJoinControllerTests extends ESTestCase { nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30), new NodeJoinController.ElectionCallback() { @Override public void onElectedAsMaster(ClusterState state) { - assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true)); + assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), + equalTo(true)); latch.countDown(); } @@ -515,17 +523,37 @@ public class NodeJoinControllerTests extends ESTestCase { assertNodesInCurrentState(nodes); } + public void testRejectingJoinWithSameAddressButDifferentId() throws InterruptedException, ExecutionException { + ClusterState state = clusterService.state(); + final DiscoveryNode other_node = new DiscoveryNode("other_node", state.nodes().getLocalNode().getAddress(), + emptyMap(), emptySet(), Version.CURRENT); - static class NoopRoutingService extends RoutingService { + ExecutionException e = expectThrows(ExecutionException.class, () -> joinNode(other_node)); + assertThat(e.getMessage(), containsString("found existing node")); + } - public NoopRoutingService(Settings settings) { - super(settings, null, new NoopAllocationService(settings)); - } + public void testRejectingJoinWithSameIdButDifferentAddress() throws InterruptedException, ExecutionException { + ClusterState state = clusterService.state(); + final DiscoveryNode other_node = new DiscoveryNode(state.nodes().getLocalNode().getId(), + new LocalTransportAddress(randomAsciiOfLength(20)), emptyMap(), emptySet(), Version.CURRENT); - @Override - protected void performReroute(String reason) { + ExecutionException e = expectThrows(ExecutionException.class, () -> joinNode(other_node)); + assertThat(e.getMessage(), containsString("found existing node")); + } - } + public void testJoinWithSameIdSameAddressButDifferentMeta() throws InterruptedException, ExecutionException { + ClusterState state = clusterService.state(); + final DiscoveryNode localNode = state.nodes().getLocalNode(); + final DiscoveryNode other_node = new DiscoveryNode( + randomBoolean() ? localNode.getName() : "other_name", + localNode.getId(), localNode.getAddress(), + randomBoolean() ? localNode.getAttributes() : Collections.singletonMap("attr", "other"), + randomBoolean() ? localNode.getRoles() : new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))), + randomBoolean() ? localNode.getVersion() : VersionUtils.randomVersion(random())); + + joinNode(other_node); + + assertThat(clusterService.localNode(), equalTo(other_node)); } static class NoopAllocationService extends AllocationService { @@ -535,12 +563,14 @@ public class NodeJoinControllerTests extends ESTestCase { } @Override - public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, List startedShards, boolean withReroute) { + public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, List startedShards, + boolean withReroute) { return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } @Override - public RoutingAllocation.Result applyFailedShards(ClusterState clusterState, List failedShards) { + public RoutingAllocation.Result applyFailedShards(ClusterState clusterState, + List failedShards) { return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } @@ -581,7 +611,7 @@ public class NodeJoinControllerTests extends ESTestCase { } } - final static AtomicInteger joinId = new AtomicInteger(); + static final AtomicInteger joinId = new AtomicInteger(); private SimpleFuture joinNodeAsync(final DiscoveryNode node) throws InterruptedException { final SimpleFuture future = new SimpleFuture("join of " + node + " (id [" + joinId.incrementAndGet() + "]"); @@ -596,9 +626,9 @@ public class NodeJoinControllerTests extends ESTestCase { } @Override - public void onFailure(Throwable t) { - logger.error("unexpected error for {}", t, future); - future.markAsFailed(t); + public void onFailure(Exception e) { + logger.error("unexpected error for {}", e, future); + future.markAsFailed(e); } }); return future; @@ -608,8 +638,8 @@ public class NodeJoinControllerTests extends ESTestCase { * creates an object clone of node, so it will be a different object instance */ private DiscoveryNode cloneNode(DiscoveryNode node) { - return new DiscoveryNode(node.getName(), node.getId(), node.getHostName(), node.getHostAddress(), node.getAddress(), - node.getAttributes(), node.getRoles(), node.getVersion()); + return new DiscoveryNode(node.getName(), node.getId(), node.getEphemeralId(), node.getHostName(), node.getHostAddress(), + node.getAddress(), node.getAttributes(), node.getRoles(), node.getVersion()); } private void joinNode(final DiscoveryNode node) throws InterruptedException, ExecutionException { diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java index fd0b11eae01..3d0d9ddd8b1 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java @@ -261,8 +261,8 @@ public class ZenDiscoveryIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable t) { - holder.set((IllegalStateException) t); + public void onFailure(Exception e) { + holder.set((IllegalStateException) e); } }); @@ -309,8 +309,8 @@ public class ZenDiscoveryIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable t) { - holder.set((IllegalStateException) t); + public void onFailure(Exception e) { + holder.set((IllegalStateException) e); } }); diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java index a6638eb19cf..9db83f48f0e 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java @@ -24,8 +24,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.test.ESTestCase; @@ -51,9 +50,9 @@ public class ZenDiscoveryUnitTests extends ESTestCase { ClusterName clusterName = new ClusterName("abc"); DiscoveryNodes.Builder currentNodes = DiscoveryNodes.builder(); - currentNodes.masterNodeId("a").put(new DiscoveryNode("a", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT)); + currentNodes.masterNodeId("a").put(new DiscoveryNode("a", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT)); DiscoveryNodes.Builder newNodes = DiscoveryNodes.builder(); - newNodes.masterNodeId("a").put(new DiscoveryNode("a", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT)); + newNodes.masterNodeId("a").put(new DiscoveryNode("a", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT)); ClusterState.Builder currentState = ClusterState.builder(clusterName); currentState.nodes(currentNodes); @@ -71,7 +70,7 @@ public class ZenDiscoveryUnitTests extends ESTestCase { assertFalse("should not ignore, because new state's version is higher to current state's version", shouldIgnoreOrRejectNewClusterState(logger, currentState.build(), newState.build())); currentNodes = DiscoveryNodes.builder(); - currentNodes.masterNodeId("b").put(new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT)); + currentNodes.masterNodeId("b").put(new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT)); ; // version isn't taken into account, so randomize it to ensure this. if (randomBoolean()) { @@ -109,7 +108,7 @@ public class ZenDiscoveryUnitTests extends ESTestCase { ArrayList allNodes = new ArrayList<>(); for (int i = randomIntBetween(10, 20); i >= 0; i--) { Set roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))); - DiscoveryNode node = new DiscoveryNode("node_" + i, "id_" + i, DummyTransportAddress.INSTANCE, Collections.emptyMap(), + DiscoveryNode node = new DiscoveryNode("node_" + i, "id_" + i, LocalTransportAddress.buildUnique(), Collections.emptyMap(), roles, Version.CURRENT); responses.add(new ZenPing.PingResponse(node, randomBoolean() ? null : node, new ClusterName("test"), randomBoolean())); allNodes.add(node); diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java index 8aa5114c387..72674f44e3d 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.test.ESTestCase; @@ -42,7 +42,7 @@ public class ZenPingTests extends ESTestCase { boolean hasJoinedOncePerNode[] = new boolean[nodes.length]; ArrayList pings = new ArrayList<>(); for (int i = 0; i < nodes.length; i++) { - nodes[i] = new DiscoveryNode("" + i, DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + nodes[i] = new DiscoveryNode("" + i, LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); } for (int pingCount = scaledRandomIntBetween(10, nodes.length * 10); pingCount > 0; pingCount--) { diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java index f072c5faf8a..06ac6aeb570 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java @@ -20,7 +20,6 @@ package org.elasticsearch.discovery.zen.ping.unicast; import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java index ab9aed6ba44..42aa792c95f 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java @@ -25,7 +25,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.discovery.zen.publish.PendingClusterStatesQueue.ClusterStateContext; import org.elasticsearch.test.ESTestCase; @@ -237,7 +237,7 @@ public class PendingClusterStatesQueueTests extends ESTestCase { ClusterState state = lastClusterStatePerMaster[masterIndex]; if (state == null) { state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(DiscoveryNodes.builder() - .put(new DiscoveryNode(masters[masterIndex], DummyTransportAddress.INSTANCE, + .put(new DiscoveryNode(masters[masterIndex], LocalTransportAddress.buildUnique(), emptyMap(), emptySet(),Version.CURRENT)).masterNodeId(masters[masterIndex]).build() ).build(); } else { @@ -259,8 +259,8 @@ public class PendingClusterStatesQueueTests extends ESTestCase { } @Override - public void onNewClusterStateFailed(Throwable t) { - failure = t; + public void onNewClusterStateFailed(Exception e) { + failure = e; } } diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java index 61374cc0d8f..0d4274a5a53 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.DiscoveryNodesProvider; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; @@ -152,16 +153,17 @@ public class PublishClusterStateActionTests extends ESTestCase { return createMockNode(name, settings, null); } - public MockNode createMockNode(String name, Settings settings, @Nullable ClusterStateListener listener) throws Exception { - settings = Settings.builder() + public MockNode createMockNode(String name, final Settings basSettings, @Nullable ClusterStateListener listener) throws Exception { + final Settings settings = Settings.builder() .put("name", name) .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "", TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING") - .put(settings) + .put(basSettings) .build(); MockTransportService service = buildTransportService(settings); DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(settings); - DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(service.boundAddress().publishAddress()); + DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(service.boundAddress().publishAddress(), + () -> NodeEnvironment.generateNodeId(settings)); MockNode node = new MockNode(discoveryNode, service, listener, logger); node.action = buildPublishClusterStateAction(settings, service, () -> node.clusterState, node); final CountDownLatch latch = new CountDownLatch(nodes.size() * 2 + 1); @@ -797,9 +799,9 @@ public class PublishClusterStateActionTests extends ESTestCase { } @Override - public void onNodeAck(DiscoveryNode node, @Nullable Throwable t) { - if (t != null) { - errors.add(new Tuple<>(node, t)); + public void onNodeAck(DiscoveryNode node, @Nullable Exception e) { + if (e != null) { + errors.add(new Tuple<>(node, e)); } countDown.countDown(); } @@ -910,8 +912,8 @@ public class PublishClusterStateActionTests extends ESTestCase { } @Override - public void sendResponse(Throwable error) throws IOException { - this.error.set(error); + public void sendResponse(Exception exception) throws IOException { + this.error.set(exception); assertThat(response.get(), nullValue()); } diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index ad425d8afc9..50e05d97985 100644 --- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -22,7 +22,6 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.io.PathUtils; @@ -48,12 +47,11 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.not; @LuceneTestCase.SuppressFileSystems("ExtrasFS") // TODO: fix test to allow extras public class NodeEnvironmentTests extends ESTestCase { @@ -269,9 +267,9 @@ public class NodeEnvironmentTests extends ESTestCase { if (randomBoolean()) { Thread t = new Thread(new AbstractRunnable() { @Override - public void onFailure(Throwable t) { - logger.error("unexpected error", t); - threadException.set(t); + public void onFailure(Exception e) { + logger.error("unexpected error", e); + threadException.set(e); latch.countDown(); blockLatch.countDown(); } @@ -392,7 +390,7 @@ public class NodeEnvironmentTests extends ESTestCase { env.close(); NodeEnvironment env2 = newNodeEnvironment(dataPaths, "/tmp", - Settings.builder().put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), false).build()); + Settings.builder().put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), false).build()); assertThat(env2.availableShardPaths(sid), equalTo(env2.availableShardPaths(sid))); assertThat(env2.resolveCustomLocation(s2, sid), equalTo(PathUtils.get("/tmp/foo/" + index.getUUID() + "/0"))); @@ -450,6 +448,27 @@ public class NodeEnvironmentTests extends ESTestCase { } } + public void testPersistentNodeId() throws IOException { + String[] paths = tmpPaths(); + NodeEnvironment env = newNodeEnvironment(paths, Settings.builder() + .put("node.local_storage", false) + .put("node.master", false) + .put("node.data", false) + .build()); + String nodeID = env.nodeId(); + env.close(); + env = newNodeEnvironment(paths, Settings.EMPTY); + assertThat("previous node didn't have local storage enabled, id should change", env.nodeId(), not(equalTo(nodeID))); + nodeID = env.nodeId(); + env.close(); + env = newNodeEnvironment(paths, Settings.EMPTY); + assertThat(env.nodeId(), equalTo(nodeID)); + env.close(); + env = newNodeEnvironment(Settings.EMPTY); + assertThat(env.nodeId(), not(equalTo(nodeID))); + env.close(); + } + /** Converts an array of Strings to an array of Paths, adding an additional child if specified */ private Path[] stringsToPaths(String[] strings, String additional) { Path[] locations = new Path[strings.length]; diff --git a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java index 948f4820439..092e6eaff8a 100644 --- a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java @@ -24,7 +24,7 @@ import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; @@ -45,11 +45,11 @@ import static org.hamcrest.Matchers.sameInstance; /** */ public class AsyncShardFetchTests extends ESTestCase { - private final DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, Collections.emptyMap(), + private final DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), Collections.emptyMap(), Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT); private final Response response1 = new Response(node1); private final Throwable failure1 = new Throwable("simulated failure 1"); - private final DiscoveryNode node2 = new DiscoveryNode("node2", DummyTransportAddress.INSTANCE, Collections.emptyMap(), + private final DiscoveryNode node2 = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), Collections.emptyMap(), Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT); private final Response response2 = new Response(node2); private final Throwable failure2 = new Throwable("simulate failure 2"); @@ -292,7 +292,7 @@ public class AsyncShardFetchTests extends ESTestCase { } else { processAsyncFetch(shardId, Collections.singletonList(entry.response), null); } - } catch (Throwable e) { + } catch (Exception e) { logger.error("unexpected failure", e); } finally { if (entry != null) { diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index 129495ea15e..f86b56f1052 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -346,7 +346,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { logger.info("--> created temp data path for shadow replicas [{}]", dataPath); logger.info("--> starting a cluster with " + numNodes + " nodes"); final Settings nodeSettings = Settings.builder() - .put("node.add_id_to_custom_path", false) + .put("node.add_lock_id_to_custom_path", false) .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), dataPath.toString()) .put("index.store.fs.fs_lock", randomFrom("native", "simple")) .build(); @@ -426,7 +426,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { // this one is not validated ahead of time and breaks allocation .put("index.analysis.filter.myCollator.type", "icu_collation") ).build(); - IndexMetaData.FORMAT.write(brokenMeta, brokenMeta.getVersion(), services.indexPaths(brokenMeta.getIndex())); + IndexMetaData.FORMAT.write(brokenMeta, services.indexPaths(brokenMeta.getIndex())); } internalCluster().fullRestart(); // ensureGreen(closedIndex) waits for the index to show up in the metadata @@ -483,7 +483,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { for (NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) { IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings() .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build(); - IndexMetaData.FORMAT.write(brokenMeta, brokenMeta.getVersion(), services.indexPaths(brokenMeta.getIndex())); + IndexMetaData.FORMAT.write(brokenMeta, services.indexPaths(brokenMeta.getIndex())); } internalCluster().fullRestart(); // ensureGreen(closedIndex) waits for the index to show up in the metadata @@ -521,7 +521,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase { MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder() .put(metaData.persistentSettings()).put("this.is.unknown", true) .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build(); - MetaData.FORMAT.write(brokenMeta, metaData.version(), nodeEnv.nodeDataPaths()); + MetaData.FORMAT.write(brokenMeta, nodeEnv.nodeDataPaths()); } internalCluster().fullRestart(); ensureYellow("test"); // wait for state recovery diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java index 4f65c5fafdd..1b62f5d330a 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.gateway; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -40,7 +39,7 @@ public class GatewayServiceTests extends ESTestCase { .put("http.enabled", "false") .put("discovery.type", "local") .put(settings.build()).build(), - null, clusterService, null, null, null, null, new NoopDiscovery(), null, null); + null, clusterService, null, null, null, new NoopDiscovery(), null, null); } public void testDefaultRecoverAfterTime() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java index 41eba406009..4cf505d839a 100644 --- a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java @@ -104,7 +104,7 @@ public class MetaDataStateFormatTests extends ESTestCase { Format format = new Format(randomFrom(XContentType.values()), "foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); int version = between(0, Integer.MAX_VALUE/2); - format.write(state, version, dirs); + format.write(state, dirs); for (Path file : dirs) { Path[] list = content("*", file); assertEquals(list.length, 1); @@ -119,7 +119,7 @@ public class MetaDataStateFormatTests extends ESTestCase { } final int version2 = between(version, Integer.MAX_VALUE); DummyState state2 = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); - format.write(state2, version2, dirs); + format.write(state2, dirs); for (Path file : dirs) { Path[] list = content("*", file); @@ -146,7 +146,7 @@ public class MetaDataStateFormatTests extends ESTestCase { Format format = new Format(randomFrom(XContentType.values()), "foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); int version = between(0, Integer.MAX_VALUE/2); - format.write(state, version, dirs); + format.write(state, dirs); for (Path file : dirs) { Path[] list = content("*", file); assertEquals(list.length, 1); @@ -170,7 +170,7 @@ public class MetaDataStateFormatTests extends ESTestCase { Format format = new Format(randomFrom(XContentType.values()), "foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); int version = between(0, Integer.MAX_VALUE/2); - format.write(state, version, dirs); + format.write(state, dirs); for (Path file : dirs) { Path[] list = content("*", file); assertEquals(list.length, 1); @@ -261,7 +261,7 @@ public class MetaDataStateFormatTests extends ESTestCase { } } for (int j = numLegacy; j < numStates; j++) { - format.write(meta.get(j), j, dirs[i]); + format.write(meta.get(j), dirs[i]); if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) { // corrupt a file that we do not necessarily need here.... Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j + ".st"); corruptedFiles.add(file); diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java index 4999ef5eac5..795046ba10c 100644 --- a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java @@ -161,8 +161,8 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase { logger.info("checking if meta state exists..."); try { assertTrue("Expecting meta state of index " + indexName + " to be on node " + nodeName, getIndicesMetaDataOnNode(nodeName).containsKey(indexName)); - } catch (Throwable t) { - logger.info("failed to load meta state", t); + } catch (Exception e) { + logger.info("failed to load meta state", e); fail("could not load meta state"); } } diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java index 96e360550af..e64c816c4bf 100644 --- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RestoreSource; @@ -41,6 +40,7 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardStateMetaData; import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.test.ESAllocationTestCase; import org.junit.Before; @@ -547,7 +547,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { return addData(node, version, allocationId, primary, null); } - public TestAllocator addData(DiscoveryNode node, long version, String allocationId, boolean primary, @Nullable Throwable storeException) { + public TestAllocator addData(DiscoveryNode node, long version, String allocationId, boolean primary, @Nullable Exception storeException) { if (data == null) { data = new HashMap<>(); } diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java index d6e8d61a7a6..1e35bcdd469 100644 --- a/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java @@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.hasItem; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class RecoverAfterNodesIT extends ESIntegTestCase { - private final static TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10); + private static final TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10); public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException { long start = System.currentTimeMillis(); diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java index ce0c78c1c92..9a93cb4d7ea 100644 --- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java +++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java @@ -18,9 +18,9 @@ */ package org.elasticsearch.http; -import org.elasticsearch.cluster.service.ClusterService; +import java.util.Map; + import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.bytes.ByteBufferBytesReference; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -33,7 +33,6 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; -import org.elasticsearch.node.service.NodeService; import org.elasticsearch.rest.AbstractRestChannel; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestController; @@ -43,10 +42,6 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Map; - public class HttpServerTests extends ESTestCase { private static final ByteSizeValue BREAKER_LIMIT = new ByteSizeValue(20); private HttpServer httpServer; @@ -72,11 +67,7 @@ public class HttpServerTests extends ESTestCase { throw new IllegalArgumentException("test error"); }); - ClusterService clusterService = new ClusterService(Settings.EMPTY, - new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null); - NodeService nodeService = new NodeService(Settings.EMPTY, null, null, null, null, null, null, null, null, - clusterService, null); - httpServer = new HttpServer(settings, httpServerTransport, restController, nodeService, null, circuitBreakerService); + httpServer = new HttpServer(settings, httpServerTransport, restController, null, circuitBreakerService); httpServer.start(); } @@ -129,7 +120,7 @@ public class HttpServerTests extends ESTestCase { assertEquals(0, inFlightRequestsBreaker.getUsed()); } - private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements + private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport { public TestHttpServerTransport() { @@ -202,7 +193,7 @@ public class HttpServerTests extends ESTestCase { private TestRestRequest(String path, String content) { this.path = path; - this.content = new ByteBufferBytesReference(ByteBuffer.wrap(content.getBytes(StandardCharsets.UTF_8))); + this.content = new BytesArray(content); } @Override diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java index 264876b7963..0aeb00914e2 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java @@ -104,7 +104,7 @@ public class NettyHttpClient implements Closeable { } @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods. - private final Collection processRequestsWithBody(HttpMethod method, SocketAddress remoteAddress, Tuple processRequestsWithBody(HttpMethod method, SocketAddress remoteAddress, Tuple... urisAndBodies) throws InterruptedException { Collection requests = new ArrayList<>(urisAndBodies.length); for (Tuple uriAndBody : urisAndBodies) { diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java index 6fc9a4e674a..ef8621dfcd3 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java @@ -142,9 +142,9 @@ public class NettyHttpServerPipeliningTests extends ESTestCase { } @Override - public HttpServerTransport stop() { + public void stop() { executorService.shutdownNow(); - return super.stop(); + super.stop(); } } diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 2769534aee0..91e9bb2c016 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -193,7 +193,6 @@ public class IndexModuleTests extends ESTestCase { IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings); IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); - Consumer listener = (s) -> {}; module.addIndexEventListener(eventListener); IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, this.listener)); diff --git a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java index c41051ec59c..f95e8408a87 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java @@ -22,6 +22,7 @@ package org.elasticsearch.index; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; @@ -43,7 +44,7 @@ public class IndexRequestBuilderIT extends ESIntegTestCase { client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}"), client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")), client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")), - client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}").toBytes()), + client().prepareIndex("test", "test").setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}"))), client().prepareIndex("test", "test").setSource(map) }; indexRandom(true, builders); diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java index 97258b12a3b..22324e1ff2b 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -77,7 +77,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase { public void testFilteringAliases() throws Exception { IndexService indexService = createIndex("test", Settings.EMPTY); - IndexShard shard = indexService.getShard(0); add(indexService, "cats", filter(termQuery("animal", "cat"))); add(indexService, "dogs", filter(termQuery("animal", "dog"))); add(indexService, "all", null); @@ -101,7 +100,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase { public void testAliasFilters() throws Exception { IndexService indexService = createIndex("test", Settings.EMPTY); - IndexShard shard = indexService.getShard(0); add(indexService, "cats", filter(termQuery("animal", "cat"))); add(indexService, "dogs", filter(termQuery("animal", "dog"))); @@ -118,7 +116,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase { public void testRemovedAliasFilter() throws Exception { IndexService indexService = createIndex("test", Settings.EMPTY); - IndexShard shard = indexService.getShard(0); add(indexService, "cats", filter(termQuery("animal", "cat"))); remove(indexService, "cats"); @@ -132,7 +129,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase { public void testUnknownAliasFilter() throws Exception { IndexService indexService = createIndex("test", Settings.EMPTY); - IndexShard shard = indexService.getShard(0); add(indexService, "cats", filter(termQuery("animal", "cat"))); add(indexService, "dogs", filter(termQuery("animal", "dog"))); diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java index f0e12abeac8..bd78607c617 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java @@ -94,7 +94,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { private Settings nodeSettings(String dataPath) { return Settings.builder() - .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), false) + .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), false) .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), dataPath) .put(FsDirectoryService.INDEX_LOCK_FACTOR_SETTING.getKey(), randomFrom("native", "simple")) .build(); @@ -379,7 +379,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar")); } - public void testPrimaryRelocationWithConcurrentIndexing() throws Throwable { + public void testPrimaryRelocationWithConcurrentIndexing() throws Exception { Path dataPath = createTempDir(); Settings nodeSettings = nodeSettings(dataPath); @@ -408,7 +408,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { final int numPhase2Docs = scaledRandomIntBetween(25, 200); final CountDownLatch phase1finished = new CountDownLatch(1); final CountDownLatch phase2finished = new CountDownLatch(1); - final CopyOnWriteArrayList exceptions = new CopyOnWriteArrayList<>(); + final CopyOnWriteArrayList exceptions = new CopyOnWriteArrayList<>(); Thread thread = new Thread() { @Override public void run() { @@ -418,8 +418,8 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { final IndexResponse indexResponse = client().prepareIndex(IDX, "doc", Integer.toString(counter.incrementAndGet())).setSource("foo", "bar").get(); assertTrue(indexResponse.isCreated()); - } catch (Throwable t) { - exceptions.add(t); + } catch (Exception e) { + exceptions.add(e); } final int docCount = counter.get(); if (docCount == numPhase1Docs) { @@ -454,7 +454,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { public void testPrimaryRelocationWhereRecoveryFails() throws Exception { Path dataPath = createTempDir(); Settings nodeSettings = Settings.builder() - .put("node.add_id_to_custom_path", false) + .put("node.add_lock_id_to_custom_path", false) .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), dataPath) .build(); @@ -677,7 +677,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { client().prepareIndex(IDX, "doc", "4").setSource("foo", "eggplant").get(); flushAndRefresh(IDX); - SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).addFieldDataField("foo").addSort("foo", SortOrder.ASC).get(); + SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).addDocValueField("foo").addSort("foo", SortOrder.ASC).get(); assertHitCount(resp, 4); assertOrderedSearchHits(resp, "2", "3", "4", "1"); SearchHit[] hits = resp.getHits().hits(); diff --git a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java index 000d2509ea8..8e2b8f68963 100644 --- a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java +++ b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java @@ -45,8 +45,6 @@ public class SettingsListenerIT extends ESIntegTestCase { public static class SettingsListenerPlugin extends Plugin { private final SettingsTestingService service = new SettingsTestingService(); - private static final Setting SETTING = Setting.intSetting("index.test.new.setting", 0, - Property.Dynamic, Property.IndexScope); @Override public List> getSettings() { diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index a0396b7abc6..a3fd266f603 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -297,7 +297,7 @@ public class InternalEngineTests extends ESTestCase { } Engine.EventListener listener = new Engine.EventListener() { @Override - public void onFailedEngine(String reason, @Nullable Throwable t) { + public void onFailedEngine(String reason, @Nullable Exception e) { // we don't need to notify anybody in this test } }; @@ -539,7 +539,7 @@ public class InternalEngineTests extends ESTestCase { public void testCommitStats() { Document document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.index(new Engine.Index(newUid("1"), doc)); @@ -716,7 +716,7 @@ public class InternalEngineTests extends ESTestCase { // create a document Document document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.index(new Engine.Index(newUid("1"), doc)); @@ -729,7 +729,7 @@ public class InternalEngineTests extends ESTestCase { // but, we can still get it (in realtime) Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); - assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray())); + assertThat(getResult.source().source, equalTo(B_1)); assertThat(getResult.docIdAndVersion(), nullValue()); getResult.release(); @@ -755,7 +755,7 @@ public class InternalEngineTests extends ESTestCase { // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); - document.add(new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null); engine.index(new Engine.Index(newUid("1"), doc)); @@ -769,7 +769,7 @@ public class InternalEngineTests extends ESTestCase { // but, we can still get it (in realtime) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); - assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray())); + assertThat(getResult.source().source, equalTo(B_2)); assertThat(getResult.docIdAndVersion(), nullValue()); getResult.release(); @@ -808,7 +808,7 @@ public class InternalEngineTests extends ESTestCase { // add it back document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.index(new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED)); @@ -2092,7 +2092,7 @@ public class InternalEngineTests extends ESTestCase { } public void testShardNotAvailableExceptionWhenEngineClosedConcurrently() throws IOException, InterruptedException { - AtomicReference throwable = new AtomicReference<>(); + AtomicReference exception = new AtomicReference<>(); String operation = randomFrom("optimize", "refresh", "flush"); Thread mergeThread = new Thread() { @Override @@ -2115,8 +2115,8 @@ public class InternalEngineTests extends ESTestCase { break; } } - } catch (Throwable t) { - throwable.set(t); + } catch (Exception e) { + exception.set(e); stop = true; } } @@ -2125,8 +2125,8 @@ public class InternalEngineTests extends ESTestCase { mergeThread.start(); engine.close(); mergeThread.join(); - logger.info("exception caught: ", throwable.get()); - assertTrue("expected an Exception that signals shard is not available", TransportActions.isShardNotAvailableException(throwable.get())); + logger.info("exception caught: ", exception.get()); + assertTrue("expected an Exception that signals shard is not available", TransportActions.isShardNotAvailableException(exception.get())); } public void testCurrentTranslogIDisCommitted() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java index ef443d1e102..672686926bd 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java @@ -242,7 +242,7 @@ public class ShadowEngineTests extends ESTestCase { } Engine.EventListener eventListener = new Engine.EventListener() { @Override - public void onFailedEngine(String reason, @Nullable Throwable t) { + public void onFailedEngine(String reason, @Nullable Exception e) { // we don't need to notify anybody in this test } }; @@ -500,7 +500,7 @@ public class ShadowEngineTests extends ESTestCase { public void testShadowEngineIgnoresWriteOperations() throws Exception { // create a document ParseContext.Document document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); try { replicaEngine.index(new Engine.Index(newUid("1"), doc)); @@ -538,7 +538,7 @@ public class ShadowEngineTests extends ESTestCase { // Now, add a document to the primary so we can test shadow engine deletes document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); primaryEngine.index(new Engine.Index(newUid("1"), doc)); primaryEngine.flush(); @@ -593,7 +593,7 @@ public class ShadowEngineTests extends ESTestCase { // create a document ParseContext.Document document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); primaryEngine.index(new Engine.Index(newUid("1"), doc)); @@ -612,7 +612,7 @@ public class ShadowEngineTests extends ESTestCase { // but, we can still get it (in realtime) Engine.GetResult getResult = primaryEngine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); - assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray())); + assertThat(getResult.source().source, equalTo(B_1)); assertThat(getResult.docIdAndVersion(), nullValue()); getResult.release(); @@ -649,7 +649,7 @@ public class ShadowEngineTests extends ESTestCase { // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); - document.add(new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null); primaryEngine.index(new Engine.Index(newUid("1"), doc)); @@ -663,7 +663,7 @@ public class ShadowEngineTests extends ESTestCase { // but, we can still get it (in realtime) getResult = primaryEngine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); - assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray())); + assertThat(getResult.source().source, equalTo(B_2)); assertThat(getResult.docIdAndVersion(), nullValue()); getResult.release(); @@ -720,7 +720,7 @@ public class ShadowEngineTests extends ESTestCase { // add it back document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); primaryEngine.index(new Engine.Index(newUid("1"), doc)); @@ -971,7 +971,7 @@ public class ShadowEngineTests extends ESTestCase { // create a document ParseContext.Document document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); pEngine.index(new Engine.Index(newUid("1"), doc)); pEngine.flush(true, true); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java index f9fb5e77b70..9a8815e9398 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; @@ -116,7 +115,7 @@ public class DynamicMappingDisabledTests extends ESSingleNodeTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { onFailureCalled.set(true); assertThat(e, instanceOf(IndexNotFoundException.class)); assertEquals(e.getMessage(), "no such index"); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIntegrationIT.java index 8afdea27451..71628c06128 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIntegrationIT.java @@ -96,8 +96,8 @@ public class DynamicMappingIntegrationIT extends ESIntegTestCase { try { startLatch.await(); assertTrue(client().prepareIndex("index", "type", id).setSource("field" + id, "bar").get().isCreated()); - } catch (Throwable t) { - error.compareAndSet(null, t); + } catch (Exception e) { + error.compareAndSet(null, e); } } }); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index 68e59527982..37d0436c9db 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -32,7 +32,7 @@ import java.util.List; public abstract class FieldTypeTestCase extends ESTestCase { /** Abstraction for mutating a property of a MappedFieldType */ - public static abstract class Modifier { + public abstract static class Modifier { /** The name of the property that is being modified. Used in test failure messages. */ public final String property; /** true if this modifier only makes types incompatible in strict mode, false otherwise */ diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index c93c181f860..2afeb02499d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -103,7 +103,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase { // 2. already existing index IndexService indexService = createIndex("index2"); - expectThrows(ExecutionException.class, () -> { + e = expectThrows(ExecutionException.class, () -> { client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "2").setSource().execute().get(); }); throwable = ExceptionsHelper.unwrapCause(e.getCause()); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java index 165b49d3145..817dc6e50df 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -292,17 +293,17 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase { } DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8(); + String mapping = mappingBuilder.endObject().endObject().bytes().utf8ToString(); logger.info("Mapping: {}", mapping); DocumentMapper docMapper = parser.parse("test", new CompressedXContent(mapping)); String builtMapping = docMapper.mappingSource().string(); // reparse it DocumentMapper builtDocMapper = parser.parse("test", new CompressedXContent(builtMapping)); - byte[] json = jsonBuilder().startObject() + byte[] json = BytesReference.toBytes(jsonBuilder().startObject() .field("foo", "bar") .field("foobar", "foobar") - .endObject().bytes().toBytes(); + .endObject().bytes()); Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc(); IndexableField[] fields = doc.getFields("_all"); if (enabled) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java index fc8e2ba1872..4bf1d0c68f7 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.binary; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -79,7 +80,7 @@ public class BinaryMappingTests extends ESSingleNodeTestCase { try (StreamOutput compressed = CompressorFactory.COMPRESSOR.streamOutput(out)) { new BytesArray(binaryValue1).writeTo(compressed); } - final byte[] binaryValue2 = out.bytes().toBytes(); + final byte[] binaryValue2 = BytesReference.toBytes(out.bytes()); assertTrue(CompressorFactory.isCompressed(new BytesArray(binaryValue2))); for (byte[] value : Arrays.asList(binaryValue1, binaryValue2)) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java index 31e4e2d0923..da0c3d081af 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java @@ -166,9 +166,9 @@ public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase { private SearchRequestBuilder prepareSearch() { SearchRequestBuilder request = client().prepareSearch("test").setTypes("test"); - request.addField("foo.token_count"); + request.addStoredField("foo.token_count"); if (loadCountedFields) { - request.addField("foo"); + request.addStoredField("foo"); } return request; } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java index 202afd7a4b1..14dd370fbfd 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java @@ -816,7 +816,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { .field("lon", -74.0059731).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get(); // match all search with geohash field - SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet(); + SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet(); Map m = searchResponse.getHits().getAt(0).getFields(); // ensure single geohash was indexed @@ -841,7 +841,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { .field("lon", -74.0059731).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get(); // match all search with geohash field (includes prefixes) - SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet(); + SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet(); Map m = searchResponse.getHits().getAt(0).getFields(); List hashes = m.get("location.geohash").values(); @@ -872,11 +872,11 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { } // query by geohash subfield - SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet(); + SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet(); assertEquals(numDocs, searchResponse.getHits().totalHits()); // query by latlon subfield - searchResponse = client().prepareSearch().addField("location.latlon").setQuery(matchAllQuery()).execute().actionGet(); + searchResponse = client().prepareSearch().addStoredField("location.latlon").setQuery(matchAllQuery()).execute().actionGet(); assertEquals(numDocs, searchResponse.getHits().totalHits()); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/merge/TestMergeMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/merge/TestMergeMapperTests.java index 627f268545a..0133d3e5943 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/merge/TestMergeMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/merge/TestMergeMapperTests.java @@ -159,7 +159,7 @@ public class TestMergeMapperTests extends ESSingleNodeTestCase { final AtomicBoolean stopped = new AtomicBoolean(false); final CyclicBarrier barrier = new CyclicBarrier(2); final AtomicReference lastIntroducedFieldName = new AtomicReference<>(); - final AtomicReference error = new AtomicReference<>(); + final AtomicReference error = new AtomicReference<>(); final Thread updater = new Thread() { @Override public void run() { @@ -173,8 +173,8 @@ public class TestMergeMapperTests extends ESSingleNodeTestCase { lastIntroducedFieldName.set(fieldName); mapperService.merge("test", new CompressedXContent(update.toString()), MapperService.MergeReason.MAPPING_UPDATE, false); } - } catch (Throwable t) { - error.set(t); + } catch (Exception e) { + error.set(e); } finally { stopped.set(true); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java index 78da5abb746..8f38e2be576 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java @@ -297,7 +297,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { out.close(); BytesReference bytes = out.bytes(); - MappingMetaData metaData = MappingMetaData.PROTO.readFrom(StreamInput.wrap(bytes)); + MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput()); assertThat(metaData, is(expected)); } @@ -314,7 +314,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { out.close(); BytesReference bytes = out.bytes(); - MappingMetaData metaData = MappingMetaData.PROTO.readFrom(StreamInput.wrap(bytes)); + MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput()); assertThat(metaData, is(expected)); } @@ -331,7 +331,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { out.close(); BytesReference bytes = out.bytes(); - MappingMetaData metaData = MappingMetaData.PROTO.readFrom(StreamInput.wrap(bytes)); + MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput()); assertThat(metaData, is(expected)); } diff --git a/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java b/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java index 695330c21e2..667efbc8bac 100644 --- a/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java @@ -40,41 +40,41 @@ public class CombineFunctionTests extends ESTestCase { public void testWriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.MULTIPLY.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.REPLACE.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.SUM.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(2)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.AVG.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(3)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.MIN.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(4)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.MAX.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(5)); } } @@ -83,37 +83,37 @@ public class CombineFunctionTests extends ESTestCase { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.MULTIPLY)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.REPLACE)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(2); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.SUM)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(3); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.AVG)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(4); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.MIN)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(5); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.MAX)); } } diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index 6460d8505ee..da8610f29f6 100644 --- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -218,8 +218,8 @@ public class InnerHitBuilderTests extends ESTestCase { innerHits.setExplain(randomBoolean()); innerHits.setVersion(randomBoolean()); innerHits.setTrackScores(randomBoolean()); - innerHits.setFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16))); - innerHits.setFieldDataFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16))); + innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16))); + innerHits.setDocValueFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16))); // Random script fields deduped on their field name. Map scriptFields = new HashMap<>(); for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) { @@ -294,11 +294,11 @@ public class InnerHitBuilderTests extends ESTestCase { break; case 6: if (randomBoolean()) { - instance.setFieldDataFields(randomValueOtherThan(instance.getFieldDataFields(), () -> { + instance.setDocValueFields(randomValueOtherThan(instance.getDocValueFields(), () -> { return randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)); })); } else { - instance.addFieldDataField(randomAsciiOfLengthBetween(1, 16)); + instance.addDocValueField(randomAsciiOfLengthBetween(1, 16)); } break; case 7: @@ -341,12 +341,12 @@ public class InnerHitBuilderTests extends ESTestCase { HighlightBuilderTests::randomHighlighterBuilder)); break; case 11: - if (instance.getFieldNames() == null || randomBoolean()) { - instance.setFieldNames(randomValueOtherThan(instance.getFieldNames(), () -> { + if (instance.getStoredFieldNames() == null || randomBoolean()) { + instance.setStoredFieldNames(randomValueOtherThan(instance.getStoredFieldNames(), () -> { return randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)); })); } else { - instance.getFieldNames().add(randomAsciiOfLengthBetween(1, 16)); + instance.getStoredFieldNames().add(randomAsciiOfLengthBetween(1, 16)); } break; default: @@ -381,7 +381,7 @@ public class InnerHitBuilderTests extends ESTestCase { private static InnerHitBuilder serializedCopy(InnerHitBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return new InnerHitBuilder(in); } } diff --git a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java index 91a42d70809..3c5bfed86dd 100644 --- a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lucene.search.MoreLikeThisQuery; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -208,7 +207,7 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase indexMapping = Collections.singletonMap("type", "{ \"type\": {} }"); - protected final static RecoveryTargetService.RecoveryListener recoveryListener = new RecoveryTargetService.RecoveryListener() { + private final Index index = new Index("test", "uuid"); + private final ShardId shardId = new ShardId(index, 0); + private final Map indexMapping = Collections.singletonMap("type", "{ \"type\": {} }"); + protected static final RecoveryTargetService.RecoveryListener recoveryListener = new RecoveryTargetService.RecoveryListener() { @Override public void onRecoveryDone(RecoveryState state) { @@ -198,7 +198,7 @@ public abstract class ESIndexLevelReplicationTestCase extends ESTestCase { } private DiscoveryNode getDiscoveryNode(String id) { - return new DiscoveryNode(id, id, DummyTransportAddress.INSTANCE, Collections.emptyMap(), + return new DiscoveryNode(id, id, LocalTransportAddress.buildUnique(), Collections.emptyMap(), Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT); } @@ -408,7 +408,7 @@ public abstract class ESIndexLevelReplicationTestCase extends ESTestCase { } @Override - public void failShard(String message, Throwable throwable) { + public void failShard(String message, Exception exception) { throw new UnsupportedOperationException(); } @@ -436,14 +436,14 @@ public abstract class ESIndexLevelReplicationTestCase extends ESTestCase { .filter(s -> replicaRouting.isSameAllocation(s.routingEntry())).findFirst().get(); TransportIndexAction.executeIndexRequestOnReplica(request, replica); listener.onResponse(TransportResponse.Empty.INSTANCE); - } catch (Throwable t) { + } catch (Exception t) { listener.onFailure(t); } } @Override - public void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable, Runnable onSuccess, - Consumer onPrimaryDemoted, Consumer onIgnoredFailure) { + public void failShard(ShardRouting replica, ShardRouting primary, String message, Exception exception, Runnable onSuccess, + Consumer onPrimaryDemoted, Consumer onIgnoredFailure) { throw new UnsupportedOperationException(); } } diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java new file mode 100644 index 00000000000..c9bb9e19866 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java @@ -0,0 +1,219 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.shard; + +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.inject.internal.Nullable; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class IndexShardOperationsLockTests extends ESTestCase { + + private static ThreadPool threadPool; + + private IndexShardOperationsLock block; + + @BeforeClass + public static void setupThreadPool() { + threadPool = new TestThreadPool("IndexShardOperationsLockTests"); + } + + @AfterClass + public static void shutdownThreadPool() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + + @Before + public void createIndexShardOperationsLock() { + block = new IndexShardOperationsLock(new ShardId("blubb", "id", 0), logger, threadPool); + } + + @After + public void checkNoInflightOperations() { + assertThat(block.semaphore.availablePermits(), equalTo(Integer.MAX_VALUE)); + assertThat(block.getActiveOperationsCount(), equalTo(0)); + } + + public void testAllOperationsInvoked() throws InterruptedException, TimeoutException, ExecutionException { + int numThreads = 10; + + List> futures = new ArrayList<>(); + List operationThreads = new ArrayList<>(); + CountDownLatch latch = new CountDownLatch(numThreads / 2); + for (int i = 0; i < numThreads; i++) { + PlainActionFuture future = new PlainActionFuture() { + @Override + public void onResponse(Releasable releasable) { + releasable.close(); + super.onResponse(releasable); + } + }; + Thread thread = new Thread() { + public void run() { + latch.countDown(); + block.acquire(future, ThreadPool.Names.GENERIC, true); + } + }; + futures.add(future); + operationThreads.add(thread); + } + + CountDownLatch blockFinished = new CountDownLatch(1); + threadPool.generic().execute(() -> { + try { + latch.await(); + blockAndWait().close(); + blockFinished.countDown(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + + for (Thread thread : operationThreads) { + thread.start(); + } + + for (PlainActionFuture future : futures) { + assertNotNull(future.get(1, TimeUnit.MINUTES)); + } + + for (Thread thread : operationThreads) { + thread.join(); + } + + blockFinished.await(); + } + + + public void testOperationsInvokedImmediatelyIfNoBlock() throws ExecutionException, InterruptedException { + PlainActionFuture future = new PlainActionFuture<>(); + block.acquire(future, ThreadPool.Names.GENERIC, true); + assertTrue(future.isDone()); + future.get().close(); + } + + public void testOperationsIfClosed() throws ExecutionException, InterruptedException { + PlainActionFuture future = new PlainActionFuture<>(); + block.close(); + block.acquire(future, ThreadPool.Names.GENERIC, true); + ExecutionException exception = expectThrows(ExecutionException.class, future::get); + assertThat(exception.getCause(), instanceOf(IndexShardClosedException.class)); + } + + public void testBlockIfClosed() throws ExecutionException, InterruptedException { + block.close(); + expectThrows(IndexShardClosedException.class, () -> block.blockOperations(randomInt(10), TimeUnit.MINUTES, + () -> { throw new IllegalArgumentException("fake error"); })); + } + + public void testOperationsDelayedIfBlock() throws ExecutionException, InterruptedException, TimeoutException { + PlainActionFuture future = new PlainActionFuture<>(); + try (Releasable releasable = blockAndWait()) { + block.acquire(future, ThreadPool.Names.GENERIC, true); + assertFalse(future.isDone()); + } + future.get(1, TimeUnit.MINUTES).close(); + } + + protected Releasable blockAndWait() throws InterruptedException { + CountDownLatch blockAcquired = new CountDownLatch(1); + CountDownLatch releaseBlock = new CountDownLatch(1); + CountDownLatch blockReleased = new CountDownLatch(1); + boolean throwsException = randomBoolean(); + IndexShardClosedException exception = new IndexShardClosedException(new ShardId("blubb", "id", 0)); + threadPool.generic().execute(() -> { + try { + block.blockOperations(1, TimeUnit.MINUTES, () -> { + try { + blockAcquired.countDown(); + releaseBlock.await(); + if (throwsException) { + throw exception; + } + } catch (InterruptedException e) { + throw new RuntimeException(); + } + }); + } catch (Exception e) { + if (e != exception) { + throw new RuntimeException(e); + } + } finally { + blockReleased.countDown(); + } + }); + blockAcquired.await(); + return () -> { + releaseBlock.countDown(); + try { + blockReleased.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }; + } + + public void testActiveOperationsCount() throws ExecutionException, InterruptedException { + PlainActionFuture future1 = new PlainActionFuture<>(); + block.acquire(future1, ThreadPool.Names.GENERIC, true); + assertTrue(future1.isDone()); + assertThat(block.getActiveOperationsCount(), equalTo(1)); + + PlainActionFuture future2 = new PlainActionFuture<>(); + block.acquire(future2, ThreadPool.Names.GENERIC, true); + assertTrue(future2.isDone()); + assertThat(block.getActiveOperationsCount(), equalTo(2)); + + future1.get().close(); + assertThat(block.getActiveOperationsCount(), equalTo(1)); + future1.get().close(); // check idempotence + assertThat(block.getActiveOperationsCount(), equalTo(1)); + future2.get().close(); + assertThat(block.getActiveOperationsCount(), equalTo(0)); + + try (Releasable releasable = blockAndWait()) { + assertThat(block.getActiveOperationsCount(), equalTo(0)); + } + + PlainActionFuture future3 = new PlainActionFuture<>(); + block.acquire(future3, ThreadPool.Names.GENERIC, true); + assertTrue(future3.isDone()); + assertThat(block.getActiveOperationsCount(), equalTo(1)); + future3.get().close(); + assertThat(block.getActiveOperationsCount(), equalTo(0)); + } +} diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 40a23ee66cf..a0813fb572f 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -44,6 +44,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.TransportIndexAction; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.InternalClusterInfoService; @@ -68,7 +69,7 @@ import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -105,6 +106,7 @@ import org.elasticsearch.test.FieldMaskingReader; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; import java.nio.file.Files; @@ -121,6 +123,7 @@ import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -295,13 +298,13 @@ public class IndexShardTests extends ESSingleNodeTestCase { client().admin().indices().prepareDelete("test").get(); assertThat(indexShard.getActiveOperationsCount(), equalTo(0)); try { - indexShard.acquirePrimaryOperationLock(); + indexShard.acquirePrimaryOperationLock(null, ThreadPool.Names.INDEX); fail("we should not be able to increment anymore"); } catch (IndexShardClosedException e) { // expected } try { - indexShard.acquireReplicaOperationLock(indexShard.getPrimaryTerm()); + indexShard.acquireReplicaOperationLock(indexShard.getPrimaryTerm(), null, ThreadPool.Names.INDEX); fail("we should not be able to increment anymore"); } catch (IndexShardClosedException e) { // expected @@ -339,21 +342,33 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertEquals(0, indexShard.getActiveOperationsCount()); if (newPrimaryShardRouting.isRelocationTarget() == false) { try { - indexShard.acquireReplicaOperationLock(primaryTerm); + indexShard.acquireReplicaOperationLock(primaryTerm, null, ThreadPool.Names.INDEX); fail("shard shouldn't accept operations as replica"); } catch (IllegalStateException ignored) { } } - Releasable operation1 = indexShard.acquirePrimaryOperationLock(); + Releasable operation1 = acquirePrimaryOperationLockBlockingly(indexShard); assertEquals(1, indexShard.getActiveOperationsCount()); - Releasable operation2 = indexShard.acquirePrimaryOperationLock(); + Releasable operation2 = acquirePrimaryOperationLockBlockingly(indexShard); assertEquals(2, indexShard.getActiveOperationsCount()); Releasables.close(operation1, operation2); assertEquals(0, indexShard.getActiveOperationsCount()); } + private Releasable acquirePrimaryOperationLockBlockingly(IndexShard indexShard) throws ExecutionException, InterruptedException { + PlainActionFuture fut = new PlainActionFuture<>(); + indexShard.acquirePrimaryOperationLock(fut, ThreadPool.Names.INDEX); + return fut.get(); + } + + private Releasable acquireReplicaOperationLockBlockingly(IndexShard indexShard, long opPrimaryTerm) throws ExecutionException, InterruptedException { + PlainActionFuture fut = new PlainActionFuture<>(); + indexShard.acquireReplicaOperationLock(opPrimaryTerm, fut, ThreadPool.Names.INDEX); + return fut.get(); + } + public void testOperationLocksOnReplicaShards() throws InterruptedException, ExecutionException, IOException { assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get()); ensureGreen("test"); @@ -399,20 +414,20 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertEquals(0, indexShard.getActiveOperationsCount()); if (newShardRouting.primary() == false) { try { - indexShard.acquirePrimaryOperationLock(); + indexShard.acquirePrimaryOperationLock(null, ThreadPool.Names.INDEX); fail("shard shouldn't accept primary ops"); } catch (IllegalStateException ignored) { } } - Releasable operation1 = indexShard.acquireReplicaOperationLock(primaryTerm); + Releasable operation1 = acquireReplicaOperationLockBlockingly(indexShard, primaryTerm); assertEquals(1, indexShard.getActiveOperationsCount()); - Releasable operation2 = indexShard.acquireReplicaOperationLock(primaryTerm); + Releasable operation2 = acquireReplicaOperationLockBlockingly(indexShard, primaryTerm); assertEquals(2, indexShard.getActiveOperationsCount()); try { - indexShard.acquireReplicaOperationLock(primaryTerm - 1); + indexShard.acquireReplicaOperationLock(primaryTerm - 1, null, ThreadPool.Names.INDEX); fail("you can not increment the operation counter with an older primary term"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("operation term")); @@ -420,7 +435,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { } // but you can increment with a newer one.. - indexShard.acquireReplicaOperationLock(primaryTerm + 1 + randomInt(20)).close(); + acquireReplicaOperationLockBlockingly(indexShard, primaryTerm + 1 + randomInt(20)).close(); Releasables.close(operation1, operation2); assertEquals(0, indexShard.getActiveOperationsCount()); } @@ -448,7 +463,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { public static void write(ShardStateMetaData shardStateMetaData, Path... shardPaths) throws IOException { - ShardStateMetaData.FORMAT.write(shardStateMetaData, shardStateMetaData.legacyVersion, shardPaths); + ShardStateMetaData.FORMAT.write(shardStateMetaData, shardPaths); } public void testDurableFlagHasEffect() { @@ -653,7 +668,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { if (randomBoolean() || true) { // try to serialize it to ensure values survive the serialization BytesStreamOutput out = new BytesStreamOutput(); stats.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); stats = ShardStats.readShardStats(in); } XContentBuilder builder = XContentFactory.jsonBuilder(); @@ -710,7 +725,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { } @Override - public void postIndex(Engine.Index index, Throwable ex) { + public void postIndex(Engine.Index index, Exception ex) { postIndexException.incrementAndGet(); } @@ -726,7 +741,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { } @Override - public void postDelete(Engine.Delete delete, Throwable ex) { + public void postDelete(Engine.Delete delete, Exception ex) { postDeleteException.incrementAndGet(); } @@ -882,13 +897,18 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService(resolveIndex("test")); final IndexShard shard = test.getShardOrNull(0); + assertBusy(() -> assertThat(shard.state(), equalTo(IndexShardState.STARTED))); CountDownLatch latch = new CountDownLatch(1); Thread recoveryThread = new Thread(() -> { latch.countDown(); - shard.relocated("simulated recovery"); + try { + shard.relocated("simulated recovery"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } }); - try (Releasable ignored = shard.acquirePrimaryOperationLock()) { + try (Releasable ignored = acquirePrimaryOperationLockBlockingly(shard)) { // start finalization of recovery recoveryThread.start(); latch.await(); @@ -898,12 +918,50 @@ public class IndexShardTests extends ESSingleNodeTestCase { // recovery can be now finalized recoveryThread.join(); assertThat(shard.state(), equalTo(IndexShardState.RELOCATED)); - try (Releasable ignored = shard.acquirePrimaryOperationLock()) { + try (Releasable ignored = acquirePrimaryOperationLockBlockingly(shard)) { // lock can again be acquired assertThat(shard.state(), equalTo(IndexShardState.RELOCATED)); } } + public void testDelayedOperationsBeforeAndAfterRelocated() throws Exception { + assertAcked(client().admin().indices().prepareCreate("test").setSettings( + Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0) + ).get()); + ensureGreen(); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService test = indicesService.indexService(resolveIndex("test")); + final IndexShard shard = test.getShardOrNull(0); + assertBusy(() -> assertThat(shard.state(), equalTo(IndexShardState.STARTED))); + Thread recoveryThread = new Thread(() -> { + try { + shard.relocated("simulated recovery"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + + recoveryThread.start(); + List> onLockAcquiredActions = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + PlainActionFuture onLockAcquired = new PlainActionFuture() { + @Override + public void onResponse(Releasable releasable) { + releasable.close(); + super.onResponse(releasable); + } + }; + shard.acquirePrimaryOperationLock(onLockAcquired, ThreadPool.Names.INDEX); + onLockAcquiredActions.add(onLockAcquired); + } + + for (PlainActionFuture onLockAcquired : onLockAcquiredActions) { + assertNotNull(onLockAcquired.get(30, TimeUnit.SECONDS)); + } + + recoveryThread.join(); + } + public void testStressRelocated() throws Exception { assertAcked(client().admin().indices().prepareCreate("test").setSettings( Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0) @@ -920,10 +978,10 @@ public class IndexShardTests extends ESSingleNodeTestCase { indexThreads[i] = new Thread() { @Override public void run() { - try (Releasable operationLock = shard.acquirePrimaryOperationLock()) { + try (Releasable operationLock = acquirePrimaryOperationLockBlockingly(shard)) { allPrimaryOperationLocksAcquired.countDown(); barrier.await(); - } catch (InterruptedException | BrokenBarrierException e) { + } catch (InterruptedException | BrokenBarrierException | ExecutionException e) { throw new RuntimeException(e); } } @@ -932,7 +990,11 @@ public class IndexShardTests extends ESSingleNodeTestCase { } AtomicBoolean relocated = new AtomicBoolean(); final Thread recoveryThread = new Thread(() -> { - shard.relocated("simulated recovery"); + try { + shard.relocated("simulated recovery"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } relocated.set(true); }); // ensure we wait for all primary operation locks to be acquired @@ -974,7 +1036,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { routing = ShardRoutingHelper.reinit(routing); IndexShard newShard = test.createShard(routing); newShard.updateRoutingEntry(routing); - DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); assertTrue(newShard.recoverFromStore()); assertEquals(translogOps, newShard.recoveryState().getTranslog().recoveredOperations()); @@ -1001,7 +1063,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { routing = ShardRoutingHelper.reinit(routing, UnassignedInfo.Reason.INDEX_CREATED); IndexShard newShard = test.createShard(routing); newShard.updateRoutingEntry(routing); - DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); assertTrue(newShard.recoverFromStore()); @@ -1018,7 +1080,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { createIndex("test"); ensureGreen(); IndicesService indicesService = getInstanceFromNode(IndicesService.class); - DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); IndexService test = indicesService.indexService(resolveIndex("test")); final IndexShard shard = test.getShardOrNull(0); @@ -1114,7 +1176,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { Store targetStore = test_target_shard.store(); test_target_shard.updateRoutingEntry(routing); - DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); test_target_shard.markAsRecovering("store", new RecoveryState(routing.shardId(), routing.primary(), RecoveryState.Type.SNAPSHOT, routing.restoreSource(), localNode)); assertTrue(test_target_shard.restoreFromRepository(new IndexShardRepository() { @Override @@ -1321,7 +1383,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {}; shard.close("simon says", false); AtomicReference shardRef = new AtomicReference<>(); - List failures = new ArrayList<>(); + List failures = new ArrayList<>(); IndexingOperationListener listener = new IndexingOperationListener() { @Override @@ -1331,9 +1393,9 @@ public class IndexShardTests extends ESSingleNodeTestCase { // this is all IMC needs to do - check current memory and refresh assertTrue(shardRef.get().getIndexBufferRAMBytesUsed() > 0); shardRef.get().refresh("test"); - } catch (Throwable t) { - failures.add(t); - throw t; + } catch (Exception e) { + failures.add(e); + throw e; } } @@ -1345,9 +1407,9 @@ public class IndexShardTests extends ESSingleNodeTestCase { // this is all IMC needs to do - check current memory and refresh assertTrue(shardRef.get().getIndexBufferRAMBytesUsed() > 0); shardRef.get().refresh("test"); - } catch (Throwable t) { - failures.add(t); - throw t; + } catch (Exception e) { + failures.add(e); + throw e; } } }; @@ -1398,7 +1460,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { } public static final IndexShard recoverShard(IndexShard newShard) throws IOException { - DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), newShard.routingEntry().primary(), RecoveryState.Type.STORE, localNode, localNode)); assertTrue(newShard.recoverFromStore()); newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted()); @@ -1410,8 +1472,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndexShard newShard = new IndexShard(initializingShardRouting, indexService.getIndexSettings(), shard.shardPath(), shard.store(), indexService.cache(), indexService.mapperService(), indexService.similarityService(), indexService.fieldData(), shard.getEngineFactory(), indexService.getIndexEventListener(), wrapper, - indexService.getThreadPool(), indexService.getBigArrays(), null, Collections.emptyList(), Arrays.asList(listeners) - ); + indexService.getThreadPool(), indexService.getBigArrays(), null, Collections.emptyList(), Arrays.asList(listeners)); return newShard; } @@ -1439,10 +1500,10 @@ public class IndexShardTests extends ESSingleNodeTestCase { ShardRouting routing = getInitializingShardRouting(shard.routingEntry()); test.removeShard(0, "b/c britta says so"); IndexShard newShard = test.createShard(routing); - DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode)); List operations = new ArrayList<>(); - operations.add(new Translog.Index("testtype", "1", jsonBuilder().startObject().field("foo", "bar").endObject().bytes().toBytes())); + operations.add(new Translog.Index("testtype", "1", BytesReference.toBytes(jsonBuilder().startObject().field("foo", "bar").endObject().bytes()))); newShard.prepareForIndexRecovery(); newShard.recoveryState().getTranslog().totalOperations(operations.size()); newShard.skipTranslogRecovery(); @@ -1467,7 +1528,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { test.removeShard(0, "b/c britta says so"); IndexShard newShard = test.createShard(routing); newShard.shardRouting = routing; - DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode)); // Shard is still inactive since we haven't started recovering yet assertFalse(newShard.isActive()); @@ -1495,12 +1556,12 @@ public class IndexShardTests extends ESSingleNodeTestCase { ShardRouting routing = getInitializingShardRouting(shard.routingEntry()); test.removeShard(0, "b/c britta says so"); IndexShard newShard = test.createShard(routing); - DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode)); // Shard is still inactive since we haven't started recovering yet assertFalse(newShard.isActive()); List operations = new ArrayList<>(); - operations.add(new Translog.Index("testtype", "1", jsonBuilder().startObject().field("foo", "bar").endObject().bytes().toBytes())); + operations.add(new Translog.Index("testtype", "1", BytesReference.toBytes(jsonBuilder().startObject().field("foo", "bar").endObject().bytes()))); newShard.prepareForIndexRecovery(); newShard.skipTranslogRecovery(); // Shard is still inactive since we haven't started recovering yet @@ -1530,7 +1591,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndexShard shard = test.getShardOrNull(0); ShardRouting routing = ShardRoutingHelper.initWithSameId(shard.routingEntry()); test.removeShard(0, "b/c simon says so"); - DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); { final IndexShard newShard = test.createShard(routing); newShard.updateRoutingEntry(routing); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java index 8d86e64a391..d1cf8b32f58 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java @@ -51,7 +51,7 @@ public class IndexingOperationListenerTests extends ESTestCase{ } @Override - public void postIndex(Engine.Index index, Throwable ex) { + public void postIndex(Engine.Index index, Exception ex) { postIndexException.incrementAndGet(); } @@ -67,7 +67,7 @@ public class IndexingOperationListenerTests extends ESTestCase{ } @Override - public void postDelete(Engine.Delete delete, Throwable ex) { + public void postDelete(Engine.Delete delete, Exception ex) { postDeleteException.incrementAndGet(); } }; @@ -83,7 +83,7 @@ public class IndexingOperationListenerTests extends ESTestCase{ throw new RuntimeException(); } @Override - public void postIndex(Engine.Index index, Throwable ex) { + public void postIndex(Engine.Index index, Exception ex) { throw new RuntimeException(); } @Override @@ -96,7 +96,7 @@ public class IndexingOperationListenerTests extends ESTestCase{ throw new RuntimeException(); } @Override - public void postDelete(Engine.Delete delete, Throwable ex) { + public void postDelete(Engine.Delete delete, Exception ex) { throw new RuntimeException(); } }; diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index 4938f686f60..79b0773481e 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -115,7 +115,7 @@ public class RefreshListenersTests extends ESTestCase { BigArrays.NON_RECYCLING_INSTANCE); Engine.EventListener eventListener = new Engine.EventListener() { @Override - public void onFailedEngine(String reason, @Nullable Throwable t) { + public void onFailedEngine(String reason, @Nullable Exception e) { // we don't need to notify anybody in this test } }; @@ -251,7 +251,7 @@ public class RefreshListenersTests extends ESTestCase { getResult.docIdAndVersion().context.reader().document(getResult.docIdAndVersion().docId, visitor); assertEquals(Arrays.asList(testFieldValue), visitor.fields().get("test")); } - } catch (Throwable t) { + } catch (Exception t) { throw new RuntimeException("failure on the [" + iteration + "] iteration of thread [" + threadId + "]", t); } } @@ -279,7 +279,7 @@ public class RefreshListenersTests extends ESTestCase { document.add(uidField); document.add(versionField); BytesReference source = new BytesArray(new byte[] { 1 }); - ParsedDocument doc = new ParsedDocument(versionField, id, type, null, -1, -1, Arrays.asList(document), source, null); + ParsedDocument doc = new ParsedDocument(versionField, id, type, null, -1, -1, Arrays.asList(document), source, null); Engine.Index index = new Engine.Index(new Term("_uid", uid), doc); engine.index(index); return index; @@ -290,7 +290,7 @@ public class RefreshListenersTests extends ESTestCase { * When the listener is called this captures it's only argument. */ AtomicReference forcedRefresh = new AtomicReference<>(); - private volatile Throwable error; + private volatile Exception error; @Override public void accept(Boolean forcedRefresh) { @@ -298,7 +298,7 @@ public class RefreshListenersTests extends ESTestCase { assertNotNull(forcedRefresh); Boolean oldValue = this.forcedRefresh.getAndSet(forcedRefresh); assertNull("Listener called twice", oldValue); - } catch (Throwable e) { + } catch (Exception e) { error = e; } } diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java index b2bd7e1f9ff..749b1621e4d 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java @@ -45,7 +45,7 @@ public class ShardPathTests extends ESTestCase { ShardId shardId = new ShardId("foo", "0xDEADBEEF", 0); Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), path); ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings)); assertEquals(path, shardPath.getDataPath()); assertEquals("0xDEADBEEF", shardPath.getShardId().getIndex().getUUID()); @@ -65,7 +65,7 @@ public class ShardPathTests extends ESTestCase { Path[] paths = env.availableShardPaths(shardId); assumeTrue("This test tests multi data.path but we only got one", paths.length > 1); int id = randomIntBetween(1, 10); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, indexUUID, AllocationId.newInitializing()), id, paths); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, indexUUID, AllocationId.newInitializing()), paths); ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings)); fail("Expected IllegalStateException"); } catch (IllegalStateException e) { @@ -82,7 +82,7 @@ public class ShardPathTests extends ESTestCase { Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); int id = randomIntBetween(1, 10); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), id, path); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), path); ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings)); fail("Expected IllegalStateException"); } catch (IllegalStateException e) { @@ -124,7 +124,7 @@ public class ShardPathTests extends ESTestCase { final boolean includeNodeId = randomBoolean(); indexSettings = indexSettingsBuilder.put(IndexMetaData.SETTING_DATA_PATH, "custom").build(); nodeSettings = Settings.builder().put(Environment.PATH_SHARED_DATA_SETTING.getKey(), path.toAbsolutePath().toAbsolutePath()) - .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), includeNodeId).build(); + .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), includeNodeId).build(); if (includeNodeId) { customPath = path.resolve("custom").resolve("0"); } else { @@ -139,7 +139,7 @@ public class ShardPathTests extends ESTestCase { ShardId shardId = new ShardId("foo", indexUUID, 0); Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, indexUUID, AllocationId.newInitializing()), 2, path); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, indexUUID, AllocationId.newInitializing()), path); ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), indexSettings)); boolean found = false; for (Path p : env.nodeDataPaths()) { diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java index 70eacaafedb..abaebb88c5e 100644 --- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java +++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.snapshots.blobstore; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -52,7 +53,7 @@ public class FileInfoTests extends ESTestCase { BlobStoreIndexShardSnapshot.FileInfo info = new BlobStoreIndexShardSnapshot.FileInfo("_foobar", meta, size); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); BlobStoreIndexShardSnapshot.FileInfo.toXContent(info, builder, ToXContent.EMPTY_PARAMS); - byte[] xcontent = shuffleXContent(builder).bytes().toBytes(); + byte[] xcontent = BytesReference.toBytes(shuffleXContent(builder).bytes()); final BlobStoreIndexShardSnapshot.FileInfo parsedInfo; try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(xcontent)) { @@ -111,7 +112,7 @@ public class FileInfoTests extends ESTestCase { builder.field(FileInfo.WRITTEN_BY, Version.LATEST.toString()); builder.field(FileInfo.CHECKSUM, "666"); builder.endObject(); - byte[] xContent = builder.bytes().toBytes(); + byte[] xContent = BytesReference.toBytes(builder.bytes()); if (failure == null) { // No failures should read as usual diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java index 6508336d9f8..a8f8a9f802d 100644 --- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -22,6 +22,7 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -192,7 +193,7 @@ public class CorruptedFileIT extends ESIntegTestCase { * run the checkindex. if the corruption is still there we will catch it. */ final CountDownLatch latch = new CountDownLatch(numShards * 3); // primary + 2 replicas - final CopyOnWriteArrayList exception = new CopyOnWriteArrayList<>(); + final CopyOnWriteArrayList exception = new CopyOnWriteArrayList<>(); final IndexEventListener listener = new IndexEventListener() { @Override public void afterIndexShardClosed(ShardId sid, @Nullable IndexShard indexShard, Settings indexSettings) { @@ -210,12 +211,12 @@ public class CorruptedFileIT extends ESIntegTestCase { out.flush(); CheckIndex.Status status = checkIndex.checkIndex(); if (!status.clean) { - logger.warn("check index [failure]\n{}", new String(os.bytes().toBytes(), StandardCharsets.UTF_8)); + logger.warn("check index [failure]\n{}", os.bytes().utf8ToString()); throw new IOException("index check failure"); } } - } catch (Throwable t) { - exception.add(t); + } catch (Exception e) { + exception.add(e); } finally { store.decRef(); latch.countDown(); @@ -346,7 +347,7 @@ public class CorruptedFileIT extends ESIntegTestCase { public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { if (corrupt.get() && action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) { RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; - byte[] array = req.content().array(); + byte[] array = BytesRef.deepCopyOf(req.content().toBytesRef()).bytes; int i = randomIntBetween(0, req.content().length() - 1); array[i] = (byte) ~array[i]; // flip one byte in the content hasCorrupted.countDown(); @@ -419,10 +420,12 @@ public class CorruptedFileIT extends ESIntegTestCase { if (action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) { RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; if (truncate && req.length() > 1) { - BytesArray array = new BytesArray(req.content().array(), req.content().arrayOffset(), (int) req.length() - 1); + BytesRef bytesRef = req.content().toBytesRef(); + BytesArray array = new BytesArray(bytesRef.bytes, bytesRef.offset, (int) req.length() - 1); request = new RecoveryFileChunkRequest(req.recoveryId(), req.shardId(), req.metadata(), req.position(), array, req.lastChunk(), req.totalTranslogOps(), req.sourceThrottleTimeInNanos()); } else { - byte[] array = req.content().array(); + assert req.content().toBytesRef().bytes == req.content().toBytesRef().bytes : "no internal reference!!"; + final byte[] array = req.content().toBytesRef().bytes; int i = randomIntBetween(0, req.content().length() - 1); array[i] = (byte) ~array[i]; // flip one byte in the content } @@ -643,12 +646,12 @@ public class CorruptedFileIT extends ESIntegTestCase { return shardRouting; } - private static final boolean isPerCommitFile(String fileName) { + private static boolean isPerCommitFile(String fileName) { // .liv and segments_N are per commit files and might change after corruption return fileName.startsWith("segments") || fileName.endsWith(".liv"); } - private static final boolean isPerSegmentFile(String fileName) { + private static boolean isPerSegmentFile(String fileName) { return isPerCommitFile(fileName) == false; } diff --git a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java index 163d72f4553..590c5c624f1 100644 --- a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java +++ b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java @@ -113,7 +113,7 @@ public class ExceptionRetryIT extends ESIntegTestCase { } refresh(); - SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addField("_id").get(); + SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get(); Set uniqueIds = new HashSet(); long dupCounter = 0; diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index c72a6ecef27..c9b3daa806a 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.translog; import com.carrotsearch.randomizedtesting.generators.RandomPicks; - import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.Term; import org.apache.lucene.mockfile.FilterFileChannel; @@ -60,9 +59,7 @@ import java.nio.charset.Charset; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.InvalidPathException; -import java.nio.file.OpenOption; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collection; @@ -217,24 +214,24 @@ public class TranslogTests extends ESTestCase { Translog.Location loc2 = translog.add(new Translog.Index("test", "2", new byte[]{2})); assertThat(loc2, greaterThan(loc1)); assertThat(translog.getLastWriteLocation(), greaterThan(loc2)); - assertThat(translog.read(loc1).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{1}))); - assertThat(translog.read(loc2).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{2}))); + assertThat(translog.read(loc1).getSource().source, equalTo(new BytesArray(new byte[]{1}))); + assertThat(translog.read(loc2).getSource().source, equalTo(new BytesArray(new byte[]{2}))); Translog.Location lastLocBeforeSync = translog.getLastWriteLocation(); translog.sync(); assertEquals(lastLocBeforeSync, translog.getLastWriteLocation()); - assertThat(translog.read(loc1).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{1}))); - assertThat(translog.read(loc2).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{2}))); + assertThat(translog.read(loc1).getSource().source, equalTo(new BytesArray(new byte[]{1}))); + assertThat(translog.read(loc2).getSource().source, equalTo(new BytesArray(new byte[]{2}))); Translog.Location loc3 = translog.add(new Translog.Index("test", "2", new byte[]{3})); assertThat(loc3, greaterThan(loc2)); assertThat(translog.getLastWriteLocation(), greaterThan(loc3)); - assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3}))); + assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3}))); lastLocBeforeSync = translog.getLastWriteLocation(); translog.sync(); assertEquals(lastLocBeforeSync, translog.getLastWriteLocation()); - assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3}))); + assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3}))); translog.prepareCommit(); /* * The commit adds to the lastWriteLocation even though is isn't really a write. This is just an implementation artifact but it can @@ -242,7 +239,7 @@ public class TranslogTests extends ESTestCase { * and less than the location of the next write operation. */ assertThat(translog.getLastWriteLocation(), greaterThan(lastLocBeforeSync)); - assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3}))); + assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3}))); translog.commit(); assertNull(translog.read(loc1)); assertNull(translog.read(loc2)); @@ -274,7 +271,7 @@ public class TranslogTests extends ESTestCase { Translog.Index index = (Translog.Index) snapshot.next(); assertThat(index != null, equalTo(true)); - assertThat(index.source().toBytes(), equalTo(new byte[]{1})); + assertThat(BytesReference.toBytes(index.source()), equalTo(new byte[]{1})); Translog.Delete delete = (Translog.Delete) snapshot.next(); assertThat(delete != null, equalTo(true)); @@ -303,7 +300,7 @@ public class TranslogTests extends ESTestCase { if (randomBoolean()) { BytesStreamOutput out = new BytesStreamOutput(); stats.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); stats = new TranslogStats(); stats.readFrom(in); } @@ -350,7 +347,7 @@ public class TranslogTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); total.writeTo(out); TranslogStats copy = new TranslogStats(); - copy.readFrom(StreamInput.wrap(out.bytes())); + copy.readFrom(out.bytes().streamInput()); assertEquals(6, copy.estimatedNumberOfOperations()); assertEquals(431, copy.getTranslogSizeInBytes()); @@ -458,7 +455,7 @@ public class TranslogTests extends ESTestCase { final BlockingQueue writtenOperations = new ArrayBlockingQueue<>(threadCount * opsPerThread); Thread[] threads = new Thread[threadCount]; - final Throwable[] threadExceptions = new Throwable[threadCount]; + final Exception[] threadExceptions = new Exception[threadCount]; final CountDownLatch downLatch = new CountDownLatch(1); for (int i = 0; i < threadCount; i++) { final int threadId = i; @@ -626,7 +623,7 @@ public class TranslogTests extends ESTestCase { final AtomicBoolean run = new AtomicBoolean(true); // any errors on threads - final List errors = new CopyOnWriteArrayList<>(); + final List errors = new CopyOnWriteArrayList<>(); logger.debug("using [{}] readers. [{}] writers. flushing every ~[{}] ops.", readers.length, writers.length, flushEveryOps); for (int i = 0; i < writers.length; i++) { final String threadName = "writer_" + i; @@ -665,9 +662,9 @@ public class TranslogTests extends ESTestCase { } @Override - public void onFailure(Throwable t) { - logger.error("--> writer [{}] had an error", t, threadName); - errors.add(t); + public void onFailure(Exception e) { + logger.error("--> writer [{}] had an error", e, threadName); + errors.add(e); } }, threadName); writers[i].start(); @@ -680,14 +677,14 @@ public class TranslogTests extends ESTestCase { Set writtenOpsAtView; @Override - public void onFailure(Throwable t) { - logger.error("--> reader [{}] had an error", t, threadId); - errors.add(t); + public void onFailure(Exception e) { + logger.error("--> reader [{}] had an error", e, threadId); + errors.add(e); try { closeView(); - } catch (IOException e) { - logger.error("unexpected error while closing view, after failure"); - t.addSuppressed(e); + } catch (IOException inner) { + inner.addSuppressed(e); + logger.error("unexpected error while closing view, after failure", inner); } } @@ -827,7 +824,7 @@ public class TranslogTests extends ESTestCase { assertEquals(max.generation, translog.currentFileGeneration()); final Translog.Operation read = translog.read(max); - assertEquals(read.getSource().source.toUtf8(), Integer.toString(count)); + assertEquals(read.getSource().source.utf8ToString(), Integer.toString(count)); } public static Translog.Location max(Translog.Location a, Translog.Location b) { @@ -859,7 +856,7 @@ public class TranslogTests extends ESTestCase { Translog.Location location = locations.get(op); if (op <= lastSynced) { final Translog.Operation read = reader.read(location); - assertEquals(Integer.toString(op), read.getSource().source.toUtf8()); + assertEquals(Integer.toString(op), read.getSource().source.utf8ToString()); } else { try { reader.read(location); @@ -995,7 +992,7 @@ public class TranslogTests extends ESTestCase { assertEquals("expected operation" + i + " to be in the previous translog but wasn't", translog.currentFileGeneration() - 1, locations.get(i).generation); Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); - assertEquals(i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals(i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1030,7 +1027,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } if (randomBoolean()) { // recover twice @@ -1043,7 +1040,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1084,7 +1081,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } @@ -1099,7 +1096,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1143,7 +1140,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1157,7 +1154,7 @@ public class TranslogTests extends ESTestCase { ops.add(test); } Translog.writeOperations(out, ops); - final List readOperations = Translog.readOperations(StreamInput.wrap(out.bytes())); + final List readOperations = Translog.readOperations(out.bytes().streamInput()); assertEquals(ops.size(), readOperations.size()); assertEquals(ops, readOperations); } @@ -1218,7 +1215,7 @@ public class TranslogTests extends ESTestCase { for (int i = firstUncommitted; i < translogOperations; i++) { Translog.Operation next = snapshot.next(); assertNotNull("" + i, next); - assertEquals(Integer.parseInt(next.getSource().source.toUtf8()), i); + assertEquals(Integer.parseInt(next.getSource().source.utf8ToString()), i); } assertNull(snapshot.next()); } @@ -1242,7 +1239,7 @@ public class TranslogTests extends ESTestCase { final BlockingQueue writtenOperations = new ArrayBlockingQueue<>(threadCount * opsPerThread); Thread[] threads = new Thread[threadCount]; - final Throwable[] threadExceptions = new Throwable[threadCount]; + final Exception[] threadExceptions = new Exception[threadCount]; final CountDownLatch downLatch = new CountDownLatch(1); for (int i = 0; i < threadCount; i++) { final int threadId = i; @@ -1269,10 +1266,10 @@ public class TranslogTests extends ESTestCase { private final int opsPerThread; private final int threadId; private final Collection writtenOperations; - private final Throwable[] threadExceptions; + private final Exception[] threadExceptions; private final Translog translog; - public TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, Collection writtenOperations, Throwable[] threadExceptions) { + public TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, Collection writtenOperations, Exception[] threadExceptions) { this.translog = translog; this.downLatch = downLatch; this.opsPerThread = opsPerThread; @@ -1306,7 +1303,7 @@ public class TranslogTests extends ESTestCase { writtenOperations.add(new LocationOperation(op, loc)); afterAdd(); } - } catch (Throwable t) { + } catch (Exception t) { threadExceptions[threadId] = t; } } @@ -1392,7 +1389,7 @@ public class TranslogTests extends ESTestCase { assertEquals("expected operation" + i + " to be in the previous translog but wasn't", tlog.currentFileGeneration() - 1, locations.get(i).generation); Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); - assertEquals(i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals(i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1448,7 +1445,7 @@ public class TranslogTests extends ESTestCase { final int threadCount = randomIntBetween(1, 5); Thread[] threads = new Thread[threadCount]; - final Throwable[] threadExceptions = new Throwable[threadCount]; + final Exception[] threadExceptions = new Exception[threadCount]; final CountDownLatch downLatch = new CountDownLatch(1); final CountDownLatch added = new CountDownLatch(randomIntBetween(10, 100)); List writtenOperations = Collections.synchronizedList(new ArrayList<>()); @@ -1716,7 +1713,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < 1; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); - assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); } tlog.add(new Translog.Index("test", "" + 1, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); } @@ -1727,7 +1724,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < 2; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); - assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1771,7 +1768,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < 1; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); - assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); } tlog.add(new Translog.Index("test", "" + 1, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); } @@ -1870,7 +1867,7 @@ public class TranslogTests extends ESTestCase { assertEquals(syncedDocs.size(), snapshot.totalOperations()); for (int i = 0; i < syncedDocs.size(); i++) { Translog.Operation next = snapshot.next(); - assertEquals(syncedDocs.get(i), next.getSource().source.toUtf8()); + assertEquals(syncedDocs.get(i), next.getSource().source.utf8ToString()); assertNotNull("operation " + i + " must be non-null", next); } } diff --git a/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java b/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java index 23925f574ff..60e062c0d1c 100644 --- a/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java +++ b/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java @@ -50,7 +50,7 @@ public class IndexActionIT extends ESIntegTestCase { public void testAutoGenerateIdNoDuplicates() throws Exception { int numberOfIterations = scaledRandomIntBetween(10, 50); for (int i = 0; i < numberOfIterations; i++) { - Throwable firstError = null; + Exception firstError = null; createIndex("test"); int numOfDocs = randomIntBetween(10, 100); logger.info("indexing [{}] docs", numOfDocs); @@ -66,19 +66,19 @@ public class IndexActionIT extends ESIntegTestCase { try { logger.debug("running search with all types"); assertHitCount(client().prepareSearch("test").get(), numOfDocs); - } catch (Throwable t) { - logger.error("search for all docs types failed", t); + } catch (Exception e) { + logger.error("search for all docs types failed", e); if (firstError == null) { - firstError = t; + firstError = e; } } try { logger.debug("running search with a specific type"); assertHitCount(client().prepareSearch("test").setTypes("type").get(), numOfDocs); - } catch (Throwable t) { - logger.error("search for all docs of a specific type failed", t); + } catch (Exception e) { + logger.error("search for all docs of a specific type failed", e); if (firstError == null) { - firstError = t; + firstError = e; } } } diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java index 1f1b758f349..7558fbd66fe 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java @@ -22,9 +22,8 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingHelper; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.IndexService; @@ -70,8 +69,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase { super(Settings.builder() .put("indices.memory.interval", "200h") // disable it .put(settings) - .build(), - null, null, 100 * 1024 * 1024); // fix jvm mem size to 100mb + .build(), null, null); } public void deleteShard(IndexShard shard) { @@ -449,7 +447,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase { try { assertEquals(0, imc.availableShards().size()); ShardRouting routing = newShard.routingEntry(); - DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); assertEquals(1, imc.availableShards().size()); diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java index 92a411a95de..17a4b93c240 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java @@ -25,7 +25,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingHelper; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.NodeServicesProvider; @@ -103,7 +103,7 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas newRouting = ShardRoutingHelper.initialize(newRouting, nodeId); IndexShard shard = index.createShard(newRouting); shard.updateRoutingEntry(newRouting); - final DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, + final DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); shard.markAsRecovering("store", new RecoveryState(shard.shardId(), newRouting.primary(), RecoveryState.Type.SNAPSHOT, newRouting.restoreSource(), localNode)); shard.recoverFromStore(); diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java index 1cca3bb7215..d43217d9785 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java @@ -62,7 +62,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // initial cache TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0); BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value).readString()); + assertEquals("foo", value.streamInput().readString()); assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -72,7 +72,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // cache hit entity = new TestEntity(requestCacheStats, reader, indexShard, 0); value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value).readString()); + assertEquals("foo", value.streamInput().readString()); assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -117,7 +117,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // initial cache TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0); BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value).readString()); + assertEquals("foo", value.streamInput().readString()); assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -130,7 +130,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // cache the second TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0); value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", StreamInput.wrap(value).readString()); + assertEquals("bar", value.streamInput().readString()); assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -141,7 +141,7 @@ public class IndicesRequestCacheTests extends ESTestCase { secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0); value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", StreamInput.wrap(value).readString()); + assertEquals("bar", value.streamInput().readString()); assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -150,7 +150,7 @@ public class IndicesRequestCacheTests extends ESTestCase { entity = new TestEntity(requestCacheStats, reader, indexShard, 0); value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value).readString()); + assertEquals("foo", value.streamInput().readString()); assertEquals(2, requestCacheStats.stats().getHitCount()); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -209,9 +209,9 @@ public class IndicesRequestCacheTests extends ESTestCase { TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0); BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value1).readString()); + assertEquals("foo", value1.streamInput().readString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", StreamInput.wrap(value2).readString()); + assertEquals("bar", value2.streamInput().readString()); size = requestCacheStats.stats().getMemorySize(); IOUtils.close(reader, secondReader, writer, dir, cache); } @@ -240,12 +240,12 @@ public class IndicesRequestCacheTests extends ESTestCase { TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, indexShard, 0); BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value1).readString()); + assertEquals("foo", value1.streamInput().readString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", StreamInput.wrap(value2).readString()); + assertEquals("bar", value2.streamInput().readString()); logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize()); BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes()); - assertEquals("baz", StreamInput.wrap(value3).readString()); + assertEquals("baz", value3.streamInput().readString()); assertEquals(2, cache.count()); assertEquals(1, requestCacheStats.stats().getEvictions()); IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache); @@ -277,12 +277,12 @@ public class IndicesRequestCacheTests extends ESTestCase { TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, differentIdentity, 0); BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value1).readString()); + assertEquals("foo", value1.streamInput().readString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", StreamInput.wrap(value2).readString()); + assertEquals("bar", value2.streamInput().readString()); logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize()); BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes()); - assertEquals("baz", StreamInput.wrap(value3).readString()); + assertEquals("baz", value3.streamInput().readString()); assertEquals(3, cache.count()); final long hitCount = requestCacheStats.stats().getHitCount(); // clear all for the indexShard Idendity even though is't still open @@ -292,7 +292,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // third has not been validated since it's a different identity value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes()); assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount()); - assertEquals("baz", StreamInput.wrap(value3).readString()); + assertEquals("baz", value3.streamInput().readString()); IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache); diff --git a/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java b/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java index 59d86ddce67..fea69133377 100644 --- a/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java +++ b/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java @@ -70,7 +70,7 @@ public class TermsLookupTests extends ESTestCase { TermsLookup termsLookup = randomTermsLookup(); try (BytesStreamOutput output = new BytesStreamOutput()) { termsLookup.writeTo(output); - try (StreamInput in = StreamInput.wrap(output.bytes())) { + try (StreamInput in = output.bytes().streamInput()) { TermsLookup deserializedLookup = new TermsLookup(in); assertEquals(deserializedLookup, termsLookup); assertEquals(deserializedLookup.hashCode(), termsLookup.hashCode()); diff --git a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java index 4312dd6105e..5e636bed939 100644 --- a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java +++ b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java @@ -456,9 +456,9 @@ public class AnalyzeActionIT extends ESIntegTestCase { .setAnalyzer("not_exist_analyzer") .get(); fail("shouldn't get here"); - } catch (Throwable t) { - assertThat(t, instanceOf(IllegalArgumentException.class)); - assertThat(t.getMessage(), startsWith("failed to find global analyzer")); + } catch (Exception e) { + assertThat(e, instanceOf(IllegalArgumentException.class)); + assertThat(e.getMessage(), startsWith("failed to find global analyzer")); } diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java index 69bee510710..4bf12bd9138 100644 --- a/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java +++ b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java @@ -181,7 +181,8 @@ public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestC } @Override - public @Nullable MockIndexService indexService(Index index) { + @Nullable + public MockIndexService indexService(Index index) { return indices.get(index.getUUID()); } diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index 78ef13dde56..ad20cb577bb 100644 --- a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -37,7 +37,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation.FailedShard; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.indices.recovery.RecoveryTargetService; import org.elasticsearch.repositories.RepositoriesService; @@ -123,7 +123,7 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice for (Iterator> it = clusterStateServiceMap.entrySet().iterator(); it.hasNext(); ) { DiscoveryNode node = it.next().getKey(); - if (state.nodes().nodeExists(node.getId()) == false) { + if (state.nodes().nodeExists(node) == false) { it.remove(); } } @@ -254,7 +254,7 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) { roles.add(mustHaveRole); } - return new DiscoveryNode("node_" + randomAsciiOfLength(8), DummyTransportAddress.INSTANCE, Collections.emptyMap(), roles, + return new DiscoveryNode("node_" + randomAsciiOfLength(8), LocalTransportAddress.buildUnique(), Collections.emptyMap(), roles, Version.CURRENT); } diff --git a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java index 8c724efdfc7..5f7f26cd38c 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java @@ -68,14 +68,14 @@ public class FlushIT extends ESIntegTestCase { // don't use assertAllSuccessful it uses a randomized context that belongs to a different thread assertThat("Unexpected ShardFailures: " + Arrays.toString(flushResponse.getShardFailures()), flushResponse.getFailedShards(), equalTo(0)); latch.countDown(); - } catch (Throwable ex) { + } catch (Exception ex) { onFailure(ex); } } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { errors.add(e); latch.countDown(); } diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java index 5932434438c..ea2a80bada5 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.indices.flush; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -31,9 +32,11 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardNotFoundException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.threadpool.ThreadPool; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; /** */ @@ -103,7 +106,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase { assertTrue(response.success()); } - public void testSyncFailsIfOperationIsInFlight() throws InterruptedException { + public void testSyncFailsIfOperationIsInFlight() throws InterruptedException, ExecutionException { createIndex("test"); client().prepareIndex("test", "test", "1").setSource("{}").get(); IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test")); @@ -111,7 +114,9 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase { SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class); final ShardId shardId = shard.shardId(); - try (Releasable operationLock = shard.acquirePrimaryOperationLock()) { + PlainActionFuture fut = new PlainActionFuture<>(); + shard.acquirePrimaryOperationLock(fut, ThreadPool.Names.INDEX); + try (Releasable operationLock = fut.get()) { SyncedFlushUtil.LatchedListener listener = new SyncedFlushUtil.LatchedListener<>(); flushService.attemptSyncedFlush(shardId, listener); listener.latch.await(); diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUtil.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUtil.java index 485ec020c3f..b71ba63a157 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUtil.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUtil.java @@ -57,7 +57,7 @@ public class SyncedFlushUtil { public static final class LatchedListener implements ActionListener { public volatile T result; - public volatile Throwable error; + public volatile Exception error; public final CountDownLatch latch = new CountDownLatch(1); @Override @@ -67,7 +67,7 @@ public class SyncedFlushUtil { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { error = e; latch.countDown(); } diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java index 2981f2d110c..eeaeb84d9a9 100644 --- a/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java @@ -69,7 +69,7 @@ public class ConcurrentDynamicTemplateIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { throwable.add(e); latch.countDown(); } @@ -83,4 +83,4 @@ public class ConcurrentDynamicTemplateIT extends ESIntegTestCase { } } -} \ No newline at end of file +} diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java index 68a176e22c3..91fd7bb972b 100644 --- a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; @@ -258,7 +259,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { // not all shards are allocated with the initial create index. Wait for it.. ensureYellow(); - final Throwable[] threadException = new Throwable[1]; + final AtomicReference threadException = new AtomicReference<>(); final AtomicBoolean stop = new AtomicBoolean(false); Thread[] threads = new Thread[3]; final CyclicBarrier barrier = new CyclicBarrier(threads.length); @@ -298,8 +299,8 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { assertThat(mappings.containsKey(typeName), equalTo(true)); assertThat(((Map) mappings.get(typeName).getSourceAsMap().get("properties")).keySet(), Matchers.hasItem(fieldName)); } - } catch (Throwable t) { - threadException[0] = t; + } catch (Exception e) { + threadException.set(e); stop.set(true); } } @@ -311,8 +312,8 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { for (Thread t : threads) t.join(); - if (threadException[0] != null) { - throw threadException[0]; + if (threadException.get() != null) { + throw threadException.get(); } } diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java index 0ae8f71c742..b448f35c21b 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java @@ -28,6 +28,8 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; @@ -62,7 +64,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.startsWith; @@ -70,7 +71,7 @@ import static org.hamcrest.Matchers.startsWith; /** * Integration tests for InternalCircuitBreakerService */ -@ClusterScope(scope = TEST, randomDynamicTemplates = false) +@ClusterScope(scope = TEST, randomDynamicTemplates = false, numClientNodes = 0, maxNumDataNodes = 1) public class CircuitBreakerServiceIT extends ESIntegTestCase { /** Reset all breaker settings back to their defaults */ private void reset() { @@ -266,17 +267,26 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase { // Perform a search to load field data for the "test" field try { - client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get(); - fail("should have thrown an exception"); + SearchResponse searchResponse = client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get(); + if (searchResponse.getShardFailures().length > 0) { + // each shard must have failed with CircuitBreakingException + for (ShardSearchFailure shardSearchFailure : searchResponse.getShardFailures()) { + Throwable cause = ExceptionsHelper.unwrap(shardSearchFailure.getCause(), CircuitBreakingException.class); + assertThat(cause, instanceOf(CircuitBreakingException.class)); + assertEquals(((CircuitBreakingException) cause).getByteLimit(), 500L); + } + } else { + fail("should have thrown a CircuitBreakingException"); + } } catch (Exception e) { - final Throwable cause = ExceptionsHelper.unwrap(e, CircuitBreakingException.class); - assertNotNull("CircuitBreakingException is not the cause of " + e, cause); - String errMsg = "would be larger than limit of [500/500b]]"; - assertThat("Exception: [" + cause.toString() + "] should contain a CircuitBreakingException", + Throwable cause = ExceptionsHelper.unwrap(e, CircuitBreakingException.class); + assertThat(cause, instanceOf(CircuitBreakingException.class)); + assertEquals(((CircuitBreakingException) cause).getByteLimit(), 500L); + assertThat("Exception: [" + cause.toString() + "] should be caused by the parent circuit breaker", cause.toString(), startsWith("CircuitBreakingException[[parent] Data too large")); - assertThat("Exception: [" + cause.toString() + "] should contain a CircuitBreakingException", - cause.toString(), endsWith(errMsg)); } + + reset(); } public void testRequestBreaker() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 0d6d5122006..2c52cd33015 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -38,7 +38,7 @@ import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.lucene.store.IndexOutputOutputStream; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.DirectoryService; @@ -69,8 +69,8 @@ public class RecoverySourceHandlerTests extends ESTestCase { put("indices.recovery.concurrent_small_file_streams", 1).build(); final RecoverySettings recoverySettings = new RecoverySettings(settings, service); StartRecoveryRequest request = new StartRecoveryRequest(shardId, - new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), - new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), + new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), + new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), null, RecoveryState.Type.STORE, randomLong()); Store store = newStore(createTempDir()); RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, recoverySettings.getChunkSize().bytesAsInt(), @@ -119,8 +119,8 @@ public class RecoverySourceHandlerTests extends ESTestCase { put("indices.recovery.concurrent_small_file_streams", 1).build(); final RecoverySettings recoverySettings = new RecoverySettings(settings, service); StartRecoveryRequest request = new StartRecoveryRequest(shardId, - new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), - new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), + new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), + new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), null, RecoveryState.Type.STORE, randomLong()); Path tempDir = createTempDir(); Store store = newStore(tempDir, false); @@ -182,8 +182,8 @@ public class RecoverySourceHandlerTests extends ESTestCase { put("indices.recovery.concurrent_small_file_streams", 1).build(); final RecoverySettings recoverySettings = new RecoverySettings(settings, service); StartRecoveryRequest request = new StartRecoveryRequest(shardId, - new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), - new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), + new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), + new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), null, RecoveryState.Type.STORE, randomLong()); Path tempDir = createTempDir(); Store store = newStore(tempDir, false); diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java index 8552db2d376..d0401196b95 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java @@ -24,7 +24,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Streamable; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.recovery.RecoveryState.File; import org.elasticsearch.indices.recovery.RecoveryState.Index; @@ -57,9 +57,9 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; public class RecoveryTargetTests extends ESTestCase { abstract class Streamer extends Thread { private T lastRead; - final private AtomicBoolean shouldStop; - final private T source; - final AtomicReference error = new AtomicReference<>(); + private final AtomicBoolean shouldStop; + private final T source; + final AtomicReference error = new AtomicReference<>(); final Version streamVersion; Streamer(AtomicBoolean shouldStop, T source) { @@ -73,7 +73,7 @@ public class RecoveryTargetTests extends ESTestCase { } public T lastRead() throws Throwable { - Throwable t = error.get(); + Exception t = error.get(); if (t != null) { throw t; } @@ -84,7 +84,7 @@ public class RecoveryTargetTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); source.writeTo(out); out.close(); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); T obj = deserialize(in); lastRead = obj; return obj; @@ -105,8 +105,8 @@ public class RecoveryTargetTests extends ESTestCase { serializeDeserialize(); } serializeDeserialize(); - } catch (Throwable t) { - error.set(t); + } catch (Exception e) { + error.set(e); } } } @@ -339,7 +339,8 @@ public class RecoveryTargetTests extends ESTestCase { } public void testStageSequenceEnforcement() { - final DiscoveryNode discoveryNode = new DiscoveryNode("1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + final DiscoveryNode discoveryNode = new DiscoveryNode("1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), + Version.CURRENT); Stage[] stages = Stage.values(); int i = randomIntBetween(0, stages.length - 1); int j; diff --git a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java index 81c50cc4f9c..2ad8ebb52f9 100644 --- a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java @@ -42,7 +42,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.DiscoverySettings; @@ -126,7 +126,7 @@ public class RareClusterStateIT extends ESIntegTestCase { // inject a node ClusterState.Builder builder = ClusterState.builder(currentState); builder.nodes(DiscoveryNodes.builder(currentState.nodes()).put(new DiscoveryNode("_non_existent", - DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT))); + LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT))); // open index final IndexMetaData indexMetaData = IndexMetaData.builder(currentState.metaData().index(index)).state(IndexMetaData.State.OPEN).build(); @@ -145,7 +145,7 @@ public class RareClusterStateIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { } }); @@ -165,7 +165,7 @@ public class RareClusterStateIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { } }); @@ -260,7 +260,7 @@ public class RareClusterStateIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { putMappingResponse.set(e); } }); @@ -292,7 +292,7 @@ public class RareClusterStateIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { docIndexResponse.set(e); } }); @@ -376,7 +376,7 @@ public class RareClusterStateIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { putMappingResponse.set(e); } }); @@ -403,7 +403,7 @@ public class RareClusterStateIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { docIndexResponse.set(e); } }); diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 6bce95af184..8e88aff523c 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -647,7 +647,7 @@ public class IndexStatsIT extends ESIntegTestCase { flags.writeTo(out); out.close(); BytesReference bytes = out.bytes(); - CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(StreamInput.wrap(bytes)); + CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(bytes.streamInput()); for (Flag flag : values) { assertThat(flags.isSet(flag), equalTo(readStats.isSet(flag))); } @@ -661,7 +661,7 @@ public class IndexStatsIT extends ESIntegTestCase { flags.writeTo(out); out.close(); BytesReference bytes = out.bytes(); - CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(StreamInput.wrap(bytes)); + CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(bytes.streamInput()); for (Flag flag : values) { assertThat(flags.isSet(flag), equalTo(readStats.isSet(flag))); } diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java index b4f66c2e17b..ad26ec71226 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java @@ -426,7 +426,7 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { } }); waitNoPendingTasksOnAll(); diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java index e558f0f2a12..96af4ef3671 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java @@ -57,7 +57,7 @@ import static org.elasticsearch.test.VersionUtils.randomVersion; /** */ public class IndicesStoreTests extends ESTestCase { - private final static ShardRoutingState[] NOT_STARTED_STATES; + private static final ShardRoutingState[] NOT_STARTED_STATES; static { Set set = new HashSet<>(); diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index 493f8b74e04..a5ec8e4ecd7 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.IndexTemplateAlreadyExistsException; import org.elasticsearch.indices.InvalidAliasNameException; -import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; @@ -116,7 +115,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { ensureGreen(); SearchResponse searchResponse = client().prepareSearch("test_index") .setQuery(termQuery("field1", "value1")) - .addField("field1").addField("field2") + .addStoredField("field1").addStoredField("field2") .execute().actionGet(); assertHitCount(searchResponse, 1); @@ -130,7 +129,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase { // now only match on one template (template_1) searchResponse = client().prepareSearch("text_index") .setQuery(termQuery("field1", "value1")) - .addField("field1").addField("field2") + .addStoredField("field1").addStoredField("field2") .execute().actionGet(); if (searchResponse.getFailedShards() > 0) { logger.warn("failed search {}", Arrays.toString(searchResponse.getShardFailures())); diff --git a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java index 82886628c85..99c407c18b7 100644 --- a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java @@ -21,10 +21,8 @@ package org.elasticsearch.ingest; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.ingest.ProcessorsRegistry; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.TestThreadPool; import org.junit.Before; import java.util.ArrayList; diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java index 119e94580ad..9974dd568a8 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java @@ -62,7 +62,7 @@ public class IngestStatsTests extends ESTestCase { private IngestStats serialize(IngestStats stats) throws IOException { BytesStreamOutput out = new BytesStreamOutput(); stats.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); return new IngestStats(in); } } diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java index 8bf6f77a026..53964132abe 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java @@ -74,7 +74,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { public void testExecuteIndexPipelineDoesNotExist() { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id"); @SuppressWarnings("unchecked") - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); try { @@ -83,7 +83,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("pipeline with id [_id] does not exist")); } - verify(failureHandler, never()).accept(any(Throwable.class)); + verify(failureHandler, never()).accept(any(Exception.class)); verify(completionHandler, never()).accept(anyBoolean()); } @@ -98,9 +98,9 @@ public class PipelineExecutionServiceTests extends ESTestCase { new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("does_not_exist"); bulkRequest.add(indexRequest2); @SuppressWarnings("unchecked") - BiConsumer failureHandler = mock(BiConsumer.class); + BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") - Consumer completionHandler = mock(Consumer.class); + Consumer completionHandler = mock(Consumer.class); executionService.executeBulkRequest(bulkRequest.requests(), failureHandler, completionHandler); verify(failureHandler, times(1)).accept( argThat(new CustomTypeSafeMatcher("failure handler was not called with the expected arguments") { @@ -126,7 +126,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id"); @SuppressWarnings("unchecked") - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); @@ -134,8 +134,25 @@ public class PipelineExecutionServiceTests extends ESTestCase { verify(completionHandler, times(1)).accept(true); } + public void testExecuteEmptyPipeline() throws Exception { + CompoundProcessor processor = mock(CompoundProcessor.class); + when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", processor)); + when(processor.getProcessors()).thenReturn(Collections.emptyList()); + + IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id"); + @SuppressWarnings("unchecked") + Consumer failureHandler = mock(Consumer.class); + @SuppressWarnings("unchecked") + Consumer completionHandler = mock(Consumer.class); + executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); + verify(processor, never()).execute(any()); + verify(failureHandler, never()).accept(any()); + verify(completionHandler, times(1)).accept(true); + } + public void testExecutePropagateAllMetaDataUpdates() throws Exception { CompoundProcessor processor = mock(CompoundProcessor.class); + when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class))); doAnswer((InvocationOnMock invocationOnMock) -> { IngestDocument ingestDocument = (IngestDocument) invocationOnMock.getArguments()[0]; for (IngestDocument.MetaData metaData : IngestDocument.MetaData.values()) { @@ -152,7 +169,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id"); @SuppressWarnings("unchecked") - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); @@ -171,11 +188,12 @@ public class PipelineExecutionServiceTests extends ESTestCase { public void testExecuteFailure() throws Exception { CompoundProcessor processor = mock(CompoundProcessor.class); + when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class))); when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", processor)); IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id"); doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap())); @SuppressWarnings("unchecked") - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); @@ -195,7 +213,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id"); doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap())); @SuppressWarnings("unchecked") - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); @@ -213,7 +231,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap())); doThrow(new RuntimeException()).when(onFailureProcessor).execute(eqID("_index", "_type", "_id", Collections.emptyMap())); @SuppressWarnings("unchecked") - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); @@ -235,7 +253,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { doThrow(new RuntimeException()).when(onFailureProcessor).execute(eqID("_index", "_type", "_id", Collections.emptyMap())); doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap())); @SuppressWarnings("unchecked") - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); @@ -250,7 +268,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id"); @SuppressWarnings("unchecked") - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); @@ -266,7 +284,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id"); @SuppressWarnings("unchecked") - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); @@ -280,7 +298,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline("_id") .source(Collections.emptyMap()) .ttl(1000L); - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); Consumer completionHandler = mock(Consumer.class); executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); @@ -313,12 +331,13 @@ public class PipelineExecutionServiceTests extends ESTestCase { } CompoundProcessor processor = mock(CompoundProcessor.class); + when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class))); Exception error = new RuntimeException(); doThrow(error).when(processor).execute(any()); when(store.get(pipelineId)).thenReturn(new Pipeline(pipelineId, null, processor)); - BiConsumer requestItemErrorHandler = mock(BiConsumer.class); - Consumer completionHandler = mock(Consumer.class); + BiConsumer requestItemErrorHandler = mock(BiConsumer.class); + Consumer completionHandler = mock(Consumer.class); executionService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler); verify(requestItemErrorHandler, times(numIndexRequests)).accept(any(IndexRequest.class), eq(error)); @@ -339,9 +358,9 @@ public class PipelineExecutionServiceTests extends ESTestCase { when(store.get(pipelineId)).thenReturn(new Pipeline(pipelineId, null, new CompoundProcessor())); @SuppressWarnings("unchecked") - BiConsumer requestItemErrorHandler = mock(BiConsumer.class); + BiConsumer requestItemErrorHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") - Consumer completionHandler = mock(Consumer.class); + Consumer completionHandler = mock(Consumer.class); executionService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler); verify(requestItemErrorHandler, never()).accept(any(), any()); @@ -356,15 +375,15 @@ public class PipelineExecutionServiceTests extends ESTestCase { assertThat(ingestStats.getTotalStats().getIngestFailedCount(), equalTo(0L)); assertThat(ingestStats.getTotalStats().getIngestTimeInMillis(), equalTo(0L)); - when(store.get("_id1")).thenReturn(new Pipeline("_id1", null, new CompoundProcessor())); - when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, new CompoundProcessor())); + when(store.get("_id1")).thenReturn(new Pipeline("_id1", null, new CompoundProcessor(mock(Processor.class)))); + when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, new CompoundProcessor(mock(Processor.class)))); Map configurationMap = new HashMap<>(); configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"))); configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"))); executionService.updatePipelineStats(new IngestMetadata(configurationMap)); - Consumer failureHandler = mock(Consumer.class); + Consumer failureHandler = mock(Consumer.class); Consumer completionHandler = mock(Consumer.class); IndexRequest indexRequest = new IndexRequest("_index"); diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java index 8fe7ddd84cb..ced25419d92 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; @@ -69,6 +70,17 @@ public class PipelineFactoryTests extends ESTestCase { } } + public void testCreateWithEmptyProcessorsField() throws Exception { + Map pipelineConfig = new HashMap<>(); + pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description"); + pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.emptyList()); + Pipeline.Factory factory = new Pipeline.Factory(); + Pipeline pipeline = factory.create("_id", pipelineConfig, null); + assertThat(pipeline.getId(), equalTo("_id")); + assertThat(pipeline.getDescription(), equalTo("_description")); + assertThat(pipeline.getProcessors(), is(empty())); + } + public void testCreateWithPipelineOnFailure() throws Exception { Map processorConfig = new HashMap<>(); Map pipelineConfig = new HashMap<>(); diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java index ec41eda47b0..600e6dbb51e 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java @@ -259,7 +259,7 @@ public class PipelineStoreTests extends ESTestCase { store.validatePipeline(ingestInfos, putRequest); fail("exception expected"); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), equalTo("Processor type [remove] is not installed on node [{_node_id2}{local}{local[_id]}]")); + assertThat(e.getMessage(), equalTo("Processor type [remove] is not installed on node [" + node2 + "]")); } ingestInfos.put(node2, new IngestInfo(Arrays.asList(new ProcessorInfo("set"), new ProcessorInfo("remove")))); diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java index a1f4d381911..94a56d11933 100644 --- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java @@ -54,9 +54,9 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase { public void testNegativeSetting() throws InterruptedException { String collector = randomAsciiOfLength(5); Settings settings = Settings.builder().put("monitor.jvm.gc.collector." + collector + ".warn", "-" + randomTimeValue()).build(); - execute(settings, (command, interval) -> null, t -> { - assertThat(t, instanceOf(IllegalArgumentException.class)); - assertThat(t.getMessage(), allOf(containsString("invalid gc_threshold"), containsString("for [monitor.jvm.gc.collector." + collector + "."))); + execute(settings, (command, interval) -> null, e -> { + assertThat(e, instanceOf(IllegalArgumentException.class)); + assertThat(e.getMessage(), allOf(containsString("invalid gc_threshold"), containsString("for [monitor.jvm.gc.collector." + collector + "."))); }, true, null); } @@ -74,9 +74,9 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase { } // we should get an exception that a setting is missing - execute(builder.build(), (command, interval) -> null, t -> { - assertThat(t, instanceOf(IllegalArgumentException.class)); - assertThat(t.getMessage(), containsString("missing gc_threshold for [monitor.jvm.gc.collector." + collector + ".")); + execute(builder.build(), (command, interval) -> null, e -> { + assertThat(e, instanceOf(IllegalArgumentException.class)); + assertThat(e.getMessage(), containsString("missing gc_threshold for [monitor.jvm.gc.collector." + collector + ".")); }, true, null); } @@ -84,18 +84,18 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase { for (final String threshold : new String[] { "warn", "info", "debug" }) { final Settings.Builder builder = Settings.builder(); builder.put("monitor.jvm.gc.overhead." + threshold, randomIntBetween(Integer.MIN_VALUE, -1)); - execute(builder.build(), (command, interval) -> null, t -> { - assertThat(t, instanceOf(IllegalArgumentException.class)); - assertThat(t.getMessage(), containsString("setting [monitor.jvm.gc.overhead." + threshold + "] must be >= 0")); + execute(builder.build(), (command, interval) -> null, e -> { + assertThat(e, instanceOf(IllegalArgumentException.class)); + assertThat(e.getMessage(), containsString("setting [monitor.jvm.gc.overhead." + threshold + "] must be >= 0")); }, true, null); } for (final String threshold : new String[] { "warn", "info", "debug" }) { final Settings.Builder builder = Settings.builder(); builder.put("monitor.jvm.gc.overhead." + threshold, randomIntBetween(100 + 1, Integer.MAX_VALUE)); - execute(builder.build(), (command, interval) -> null, t -> { - assertThat(t, instanceOf(IllegalArgumentException.class)); - assertThat(t.getMessage(), containsString("setting [monitor.jvm.gc.overhead." + threshold + "] must be <= 100")); + execute(builder.build(), (command, interval) -> null, e -> { + assertThat(e, instanceOf(IllegalArgumentException.class)); + assertThat(e.getMessage(), containsString("setting [monitor.jvm.gc.overhead." + threshold + "] must be <= 100")); }, true, null); } @@ -104,9 +104,9 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase { infoWarnOutOfOrderBuilder.put("monitor.jvm.gc.overhead.info", info); final int warn = randomIntBetween(1, info - 1); infoWarnOutOfOrderBuilder.put("monitor.jvm.gc.overhead.warn", warn); - execute(infoWarnOutOfOrderBuilder.build(), (command, interval) -> null, t -> { - assertThat(t, instanceOf(IllegalArgumentException.class)); - assertThat(t.getMessage(), containsString("[monitor.jvm.gc.overhead.warn] must be greater than [monitor.jvm.gc.overhead.info] [" + info + "] but was [" + warn + "]")); + execute(infoWarnOutOfOrderBuilder.build(), (command, interval) -> null, e -> { + assertThat(e, instanceOf(IllegalArgumentException.class)); + assertThat(e.getMessage(), containsString("[monitor.jvm.gc.overhead.warn] must be greater than [monitor.jvm.gc.overhead.info] [" + info + "] but was [" + warn + "]")); }, true, null); final Settings.Builder debugInfoOutOfOrderBuilder = Settings.builder(); @@ -114,9 +114,9 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase { final int debug = randomIntBetween(info + 1, 99); debugInfoOutOfOrderBuilder.put("monitor.jvm.gc.overhead.debug", debug); debugInfoOutOfOrderBuilder.put("monitor.jvm.gc.overhead.warn", randomIntBetween(debug + 1, 100)); // or the test will fail for the wrong reason - execute(debugInfoOutOfOrderBuilder.build(), (command, interval) -> null, t -> { - assertThat(t, instanceOf(IllegalArgumentException.class)); - assertThat(t.getMessage(), containsString("[monitor.jvm.gc.overhead.info] must be greater than [monitor.jvm.gc.overhead.debug] [" + debug + "] but was [" + info + "]")); + execute(debugInfoOutOfOrderBuilder.build(), (command, interval) -> null, e -> { + assertThat(e, instanceOf(IllegalArgumentException.class)); + assertThat(e.getMessage(), containsString("[monitor.jvm.gc.overhead.info] must be greater than [monitor.jvm.gc.overhead.debug] [" + debug + "] but was [" + info + "]")); }, true, null); } @@ -124,7 +124,7 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase { execute(settings, scheduler, null, false, asserts); } - private static void execute(Settings settings, BiFunction> scheduler, Consumer consumer, boolean constructionShouldFail, Runnable asserts) throws InterruptedException { + private static void execute(Settings settings, BiFunction> scheduler, Consumer consumer, boolean constructionShouldFail, Runnable asserts) throws InterruptedException { assert constructionShouldFail == (consumer != null); assert constructionShouldFail == (asserts == null); ThreadPool threadPool = null; @@ -143,7 +143,7 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase { service.doStart(); asserts.run(); service.doStop(); - } catch (Throwable t) { + } catch (Exception t) { consumer.accept(t); } } finally { diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java index 91862e9cd18..278a47ed21f 100644 --- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java @@ -48,10 +48,10 @@ public class JvmMonitorTests extends ESTestCase { AtomicBoolean invoked = new AtomicBoolean(); JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap(), IGNORE) { @Override - void onMonitorFailure(Throwable t) { + void onMonitorFailure(Exception e) { invoked.set(true); - assertThat(t, instanceOf(RuntimeException.class)); - assertThat(t, hasToString(containsString("simulated"))); + assertThat(e, instanceOf(RuntimeException.class)); + assertThat(e, hasToString(containsString("simulated"))); } @Override @@ -174,7 +174,7 @@ public class JvmMonitorTests extends ESTestCase { JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(gcThresholds, IGNORE) { @Override - void onMonitorFailure(Throwable t) { + void onMonitorFailure(Exception e) { } @Override @@ -284,7 +284,7 @@ public class JvmMonitorTests extends ESTestCase { final JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap(), IGNORE) { @Override - void onMonitorFailure(Throwable t) { + void onMonitorFailure(Exception e) { } @Override @@ -358,7 +358,7 @@ public class JvmMonitorTests extends ESTestCase { final JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap(), gcOverheadThreshold) { @Override - void onMonitorFailure(final Throwable t) { + void onMonitorFailure(final Exception e) { } @Override diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java index 0d07bcf0981..090517adfcd 100644 --- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java +++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java @@ -28,7 +28,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ToXContent; @@ -70,7 +70,7 @@ public class NodeInfoStreamingTests extends ESTestCase { out.setVersion(version); nodeInfo.writeTo(out); out.close(); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); in.setVersion(version); NodeInfo readNodeInfo = NodeInfo.readNodeInfo(in); assertExpectedUnchanged(nodeInfo, readNodeInfo); @@ -81,11 +81,6 @@ public class NodeInfoStreamingTests extends ESTestCase { assertThat(nodeInfo.getBuild().toString(), equalTo(readNodeInfo.getBuild().toString())); assertThat(nodeInfo.getHostname(), equalTo(readNodeInfo.getHostname())); assertThat(nodeInfo.getVersion(), equalTo(readNodeInfo.getVersion())); - assertThat(nodeInfo.getServiceAttributes().size(), equalTo(readNodeInfo.getServiceAttributes().size())); - for (Map.Entry entry : nodeInfo.getServiceAttributes().entrySet()) { - assertNotNull(readNodeInfo.getServiceAttributes().get(entry.getKey())); - assertThat(readNodeInfo.getServiceAttributes().get(entry.getKey()), equalTo(entry.getValue())); - } compareJsonOutput(nodeInfo.getHttp(), readNodeInfo.getHttp()); compareJsonOutput(nodeInfo.getJvm(), readNodeInfo.getJvm()); compareJsonOutput(nodeInfo.getProcess(), readNodeInfo.getProcess()); @@ -122,7 +117,7 @@ public class NodeInfoStreamingTests extends ESTestCase { private NodeInfo createNodeInfo() { Build build = Build.CURRENT; - DiscoveryNode node = new DiscoveryNode("test_node", DummyTransportAddress.INSTANCE, + DiscoveryNode node = new DiscoveryNode("test_node", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), VersionUtils.randomVersion(random())); Map serviceAttributes = new HashMap<>(); serviceAttributes.put("test", "attribute"); @@ -134,7 +129,7 @@ public class NodeInfoStreamingTests extends ESTestCase { threadPoolInfos.add(new ThreadPool.Info("test_threadpool", ThreadPool.ThreadPoolType.FIXED, 5)); ThreadPoolInfo threadPoolInfo = new ThreadPoolInfo(threadPoolInfos); Map profileAddresses = new HashMap<>(); - BoundTransportAddress dummyBoundTransportAddress = new BoundTransportAddress(new TransportAddress[]{DummyTransportAddress.INSTANCE}, DummyTransportAddress.INSTANCE); + BoundTransportAddress dummyBoundTransportAddress = new BoundTransportAddress(new TransportAddress[]{LocalTransportAddress.buildUnique()}, LocalTransportAddress.buildUnique()); profileAddresses.put("test_address", dummyBoundTransportAddress); TransportInfo transport = new TransportInfo(dummyBoundTransportAddress, profileAddresses); HttpInfo htttpInfo = new HttpInfo(dummyBoundTransportAddress, randomLong()); @@ -149,6 +144,7 @@ public class NodeInfoStreamingTests extends ESTestCase { // pick a random long that sometimes exceeds an int: indexingBuffer = new ByteSizeValue(random().nextLong() & ((1L<<40)-1)); } - return new NodeInfo(VersionUtils.randomVersion(random()), build, node, serviceAttributes, settings, osInfo, process, jvm, threadPoolInfo, transport, htttpInfo, plugins, ingestInfo, indexingBuffer); + return new NodeInfo(VersionUtils.randomVersion(random()), build, node, settings, osInfo, process, jvm, + threadPoolInfo, transport, htttpInfo, plugins, ingestInfo, indexingBuffer); } } diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java index 0cc30f8d569..0916cad60d5 100644 --- a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java +++ b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java @@ -24,9 +24,9 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.ESIntegTestCase; import java.util.List; diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java index 2564b31488b..d56e1341165 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java +++ b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; @@ -30,16 +30,13 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoveriesCollection; import org.elasticsearch.indices.recovery.RecoveryFailedException; import org.elasticsearch.indices.recovery.RecoveryState; -import org.elasticsearch.indices.recovery.RecoveryTarget; import org.elasticsearch.indices.recovery.RecoveryTargetService; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; -import java.util.ArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Predicate; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; @@ -47,7 +44,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; public class RecoveriesCollectionTests extends ESSingleNodeTestCase { - final static RecoveryTargetService.RecoveryListener listener = new RecoveryTargetService.RecoveryListener() { + static final RecoveryTargetService.RecoveryListener listener = new RecoveryTargetService.RecoveryListener() { @Override public void onRecoveryDone(RecoveryState state) { @@ -135,7 +132,8 @@ public class RecoveriesCollectionTests extends ESSingleNodeTestCase { long startRecovery(RecoveriesCollection collection, RecoveryTargetService.RecoveryListener listener, TimeValue timeValue) { IndicesService indexServices = getInstanceFromNode(IndicesService.class); IndexShard indexShard = indexServices.indexServiceSafe(resolveIndex("test")).getShardOrNull(0); - final DiscoveryNode sourceNode = new DiscoveryNode("id", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); + final DiscoveryNode sourceNode = new DiscoveryNode("id", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), + Version.CURRENT); return collection.startRecovery(indexShard, sourceNode, listener, timeValue); } } diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 620dfeb94c2..2d778de37c9 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -22,6 +22,7 @@ package org.elasticsearch.recovery; import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.hppc.procedures.IntProcedure; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.English; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -93,8 +94,6 @@ import static org.hamcrest.Matchers.startsWith; public class RelocationIT extends ESIntegTestCase { private final TimeValue ACCEPTABLE_RELOCATION_TIME = new TimeValue(5, TimeUnit.MINUTES); - - @Override protected Collection> nodePlugins() { return pluginList(MockTransportService.TestPlugin.class, MockIndexEventListener.TestPlugin.class); @@ -219,7 +218,7 @@ public class RelocationIT extends ESIntegTestCase { for (int i = 0; i < 10; i++) { try { logger.info("--> START search test round {}", i + 1); - SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).setNoFields().execute().actionGet().getHits(); + SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).setNoStoredFields().execute().actionGet().getHits(); ranOnce = true; if (hits.totalHits() != indexer.totalIndexedDocs()) { int[] hitIds = new int[(int) indexer.totalIndexedDocs()]; @@ -433,22 +432,23 @@ public class RelocationIT extends ESIntegTestCase { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18553") public void testIndexAndRelocateConcurrently() throws ExecutionException, InterruptedException { + int halfNodes = randomIntBetween(1, 3); Settings blueSetting = Settings.builder().put("node.attr.color", "blue").build(); - InternalTestCluster.Async> blueFuture = internalCluster().startNodesAsync(blueSetting, blueSetting); + InternalTestCluster.Async> blueFuture = internalCluster().startNodesAsync(halfNodes, blueSetting); Settings redSetting = Settings.builder().put("node.attr.color", "red").build(); - InternalTestCluster.Async> redFuture = internalCluster().startNodesAsync(redSetting, redSetting); + InternalTestCluster.Async> redFuture = internalCluster().startNodesAsync(halfNodes, redSetting); blueFuture.get(); redFuture.get(); logger.info("blue nodes: {}", blueFuture.get()); logger.info("red nodes: {}", redFuture.get()); - ensureStableCluster(4); + ensureStableCluster(halfNodes * 2); assertAcked(prepareCreate("test").setSettings(Settings.builder() .put("index.routing.allocation.exclude.color", "blue") - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .put(indexSettings()))); + .put(indexSettings()) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) // NORELEASE: set to randomInt(halfNodes - 1) once replica data loss is fixed + )); ensureYellow(); assertAllShardsOnNodes("test", redFuture.get().toArray(new String[2])); int numDocs = randomIntBetween(100, 150); @@ -480,9 +480,11 @@ public class RelocationIT extends ESIntegTestCase { numDocs *= 2; logger.info(" --> waiting for relocation to complete"); - ensureGreen("test");// move all shards to the new node (it waits on relocation) + ensureGreen("test"); // move all shards to the new nodes (it waits on relocation) + final int numIters = randomIntBetween(10, 20); for (int i = 0; i < numIters; i++) { + logger.info(" --> checking iteration {}", i); SearchResponse afterRelocation = client().prepareSearch().setSize(ids.size()).get(); assertNoFailures(afterRelocation); assertSearchHits(afterRelocation, ids.toArray(new String[ids.size()])); @@ -505,7 +507,8 @@ public class RelocationIT extends ESIntegTestCase { if (chunkRequest.name().startsWith(IndexFileNames.SEGMENTS)) { // corrupting the segments_N files in order to make sure future recovery re-send files logger.debug("corrupting [{}] to {}. file name: [{}]", action, node, chunkRequest.name()); - byte[] array = chunkRequest.content().array(); + assert chunkRequest.content().toBytesRef().bytes == chunkRequest.content().toBytesRef().bytes : "no internal reference!!"; + byte[] array = chunkRequest.content().toBytesRef().bytes; array[0] = (byte) ~array[0]; // flip one byte in the content corruptionCount.countDown(); } diff --git a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index 3d46c0bbacf..17e2481b7d0 100644 --- a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRes import org.elasticsearch.client.Client; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -46,7 +45,6 @@ import java.util.List; import java.util.stream.Collectors; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.blobId; -import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.parseNameUUIDFromBlobName; import static org.hamcrest.Matchers.equalTo; /** @@ -108,7 +106,90 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase { assertThat(snapshotIds, equalTo(originalSnapshots)); } - public void testSnapshotIndexFile() throws Exception { + public void testReadAndWriteSnapshotsThroughIndexFile() throws Exception { + final BlobStoreRepository repository = setupRepo(); + + // write to and read from a snapshot file with no entries + assertThat(repository.snapshots().size(), equalTo(0)); + repository.writeSnapshotsToIndexGen(Collections.emptyList()); + assertThat(repository.snapshots().size(), equalTo(0)); + + // write to and read from a snapshot file with a random number of entries + final int numSnapshots = randomIntBetween(1, 1000); + final List snapshotIds = new ArrayList<>(numSnapshots); + for (int i = 0; i < numSnapshots; i++) { + snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())); + } + repository.writeSnapshotsToIndexGen(snapshotIds); + assertThat(repository.snapshots(), equalTo(snapshotIds)); + } + + public void testIndexGenerationalFiles() throws Exception { + final BlobStoreRepository repository = setupRepo(); + + // write to index generational file + final int numSnapshots = randomIntBetween(1, 1000); + final List snapshotIds = new ArrayList<>(numSnapshots); + for (int i = 0; i < numSnapshots; i++) { + snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())); + } + repository.writeSnapshotsToIndexGen(snapshotIds); + assertThat(Sets.newHashSet(repository.readSnapshotsFromIndex()), equalTo(Sets.newHashSet(snapshotIds))); + assertThat(repository.latestIndexBlobId(), equalTo(0L)); + assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(0L)); + + // adding more and writing to a new index generational file + for (int i = 0; i < 10; i++) { + snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())); + } + repository.writeSnapshotsToIndexGen(snapshotIds); + assertThat(Sets.newHashSet(repository.readSnapshotsFromIndex()), equalTo(Sets.newHashSet(snapshotIds))); + assertThat(repository.latestIndexBlobId(), equalTo(1L)); + assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(1L)); + + // removing a snapshot adn writing to a new index generational file + snapshotIds.remove(0); + repository.writeSnapshotsToIndexGen(snapshotIds); + assertThat(Sets.newHashSet(repository.readSnapshotsFromIndex()), equalTo(Sets.newHashSet(snapshotIds))); + assertThat(repository.latestIndexBlobId(), equalTo(2L)); + assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(2L)); + } + + public void testOldIndexFileFormat() throws Exception { + final BlobStoreRepository repository = setupRepo(); + + // write old index file format + final int numOldSnapshots = randomIntBetween(1, 50); + final List snapshotIds = new ArrayList<>(); + for (int i = 0; i < numOldSnapshots; i++) { + snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), SnapshotId.UNASSIGNED_UUID)); + } + writeOldFormat(repository, snapshotIds.stream().map(SnapshotId::getName).collect(Collectors.toList())); + assertThat(Sets.newHashSet(repository.snapshots()), equalTo(Sets.newHashSet(snapshotIds))); + + // write to and read from a snapshot file with a random number of new entries added + final int numSnapshots = randomIntBetween(1, 1000); + for (int i = 0; i < numSnapshots; i++) { + snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())); + } + repository.writeSnapshotsToIndexGen(snapshotIds); + assertThat(Sets.newHashSet(repository.snapshots()), equalTo(Sets.newHashSet(snapshotIds))); + } + + public void testBlobId() { + SnapshotId snapshotId = new SnapshotId("abc123", SnapshotId.UNASSIGNED_UUID); + assertThat(blobId(snapshotId), equalTo("abc123")); // just the snapshot name + snapshotId = new SnapshotId("abc-123", SnapshotId.UNASSIGNED_UUID); + assertThat(blobId(snapshotId), equalTo("abc-123")); // just the snapshot name + String uuid = UUIDs.randomBase64UUID(); + snapshotId = new SnapshotId("abc123", uuid); + assertThat(blobId(snapshotId), equalTo("abc123-" + uuid)); // snapshot name + '-' + uuid + uuid = UUIDs.randomBase64UUID(); + snapshotId = new SnapshotId("abc-123", uuid); + assertThat(blobId(snapshotId), equalTo("abc-123-" + uuid)); // snapshot name + '-' + uuid + } + + private BlobStoreRepository setupRepo() { final Client client = client(); final Path location = ESIntegTestCase.randomRepoPath(node().settings()); final String repositoryName = "test-repo"; @@ -123,89 +204,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase { final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); @SuppressWarnings("unchecked") final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repositoryName); - - // write to and read from a snapshot file with no entries - repository.writeSnapshotList(Collections.emptyList()); - List readSnapshotIds = repository.readSnapshotList(); - assertThat(readSnapshotIds.size(), equalTo(0)); - - // write to and read from a snapshot file with a random number of entries - final int numSnapshots = randomIntBetween(1, 1000); - final List snapshotIds = new ArrayList<>(numSnapshots); - for (int i = 0; i < numSnapshots; i++) { - snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())); - } - repository.writeSnapshotList(snapshotIds); - readSnapshotIds = repository.readSnapshotList(); - assertThat(readSnapshotIds, equalTo(snapshotIds)); - } - - public void testOldIndexFileFormat() throws Exception { - final Client client = client(); - final Path location = ESIntegTestCase.randomRepoPath(node().settings()); - final String repositoryName = "test-repo"; - - PutRepositoryResponse putRepositoryResponse = - client.admin().cluster().preparePutRepository(repositoryName) - .setType("fs") - .setSettings(Settings.builder().put(node().settings()).put("location", location)) - .get(); - assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); - - final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); - @SuppressWarnings("unchecked") final BlobStoreRepository repository = - (BlobStoreRepository) repositoriesService.repository(repositoryName); - - // write old index file format - final int numOldSnapshots = randomIntBetween(1, 50); - final List snapshotIds = new ArrayList<>(); - for (int i = 0; i < numOldSnapshots; i++) { - snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), SnapshotId.UNASSIGNED_UUID)); - } - writeOldFormat(repository, snapshotIds.stream().map(SnapshotId::getName).collect(Collectors.toList())); - List readSnapshotIds = repository.readSnapshotList(); - assertThat(Sets.newHashSet(readSnapshotIds), equalTo(Sets.newHashSet(snapshotIds))); - - // write to and read from a snapshot file with a random number of new entries added - final int numSnapshots = randomIntBetween(1, 1000); - for (int i = 0; i < numSnapshots; i++) { - snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID())); - } - repository.writeSnapshotList(snapshotIds); - readSnapshotIds = repository.readSnapshotList(); - assertThat(Sets.newHashSet(readSnapshotIds), equalTo(Sets.newHashSet(snapshotIds))); - } - - public void testParseUUIDFromBlobName() { - String blobStr = "abc123"; - Tuple pair = parseNameUUIDFromBlobName(blobStr); - assertThat(pair.v1(), equalTo(blobStr)); // snapshot name - assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID)); // snapshot uuid - blobStr = "abcefghijklmnopqrstuvwxyz"; - pair = parseNameUUIDFromBlobName(blobStr); - assertThat(pair.v1(), equalTo(blobStr)); - assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID)); - blobStr = "abc123-xyz"; // not enough characters after '-' to have a uuid - pair = parseNameUUIDFromBlobName(blobStr); - assertThat(pair.v1(), equalTo(blobStr)); - assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID)); - blobStr = "abc123-a1b2c3d4e5f6g7h8i9j0k1"; - pair = parseNameUUIDFromBlobName(blobStr); - assertThat(pair.v1(), equalTo("abc123")); - assertThat(pair.v2(), equalTo("a1b2c3d4e5f6g7h8i9j0k1")); - } - - public void testBlobId() { - SnapshotId snapshotId = new SnapshotId("abc123", SnapshotId.UNASSIGNED_UUID); - assertThat(blobId(snapshotId), equalTo("abc123")); // just the snapshot name - snapshotId = new SnapshotId("abc-123", SnapshotId.UNASSIGNED_UUID); - assertThat(blobId(snapshotId), equalTo("abc-123")); // just the snapshot name - String uuid = UUIDs.randomBase64UUID(); - snapshotId = new SnapshotId("abc123", uuid); - assertThat(blobId(snapshotId), equalTo("abc123-" + uuid)); // snapshot name + '-' + uuid - uuid = UUIDs.randomBase64UUID(); - snapshotId = new SnapshotId("abc-123", uuid); - assertThat(blobId(snapshotId), equalTo("abc-123-" + uuid)); // snapshot name + '-' + uuid + return repository; } private void writeOldFormat(final BlobStoreRepository repository, final List snapshotNames) throws Exception { diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java index aa3b11e6250..051159b448b 100644 --- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java +++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java @@ -36,17 +36,21 @@ import java.io.IOException; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -/** - * - */ public class BytesRestResponseTests extends ESTestCase { + class UnknownException extends Exception { + + public UnknownException(final String message, final Throwable cause) { + super(message, cause); + } + + } + public void testWithHeaders() throws Exception { RestRequest request = new FakeRestRequest(); RestChannel channel = randomBoolean() ? new DetailedExceptionRestChannel(request) : new SimpleExceptionRestChannel(request); @@ -62,9 +66,9 @@ public class BytesRestResponseTests extends ESTestCase { RestRequest request = new FakeRestRequest(); RestChannel channel = new SimpleExceptionRestChannel(request); - Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar")); + Exception t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar")); BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); assertThat(text, containsString("ElasticsearchException[an error occurred reading data]")); assertThat(text, not(containsString("FileNotFoundException"))); assertThat(text, not(containsString("/foo/bar"))); @@ -75,9 +79,9 @@ public class BytesRestResponseTests extends ESTestCase { RestRequest request = new FakeRestRequest(); RestChannel channel = new DetailedExceptionRestChannel(request); - Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar")); + Exception t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar")); BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); assertThat(text, containsString("{\"type\":\"exception\",\"reason\":\"an error occurred reading data\"}")); assertThat(text, containsString("{\"type\":\"file_not_found_exception\",\"reason\":\"/foo/bar\"}")); } @@ -86,10 +90,10 @@ public class BytesRestResponseTests extends ESTestCase { RestRequest request = new FakeRestRequest(); RestChannel channel = new SimpleExceptionRestChannel(request); - Throwable t = new Throwable("an error occurred reading data", new FileNotFoundException("/foo/bar")); + Exception t = new UnknownException("an error occurred reading data", new FileNotFoundException("/foo/bar")); BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); - assertThat(text, not(containsString("Throwable[an error occurred reading data]"))); + String text = response.content().utf8ToString(); + assertThat(text, not(containsString("UnknownException[an error occurred reading data]"))); assertThat(text, not(containsString("FileNotFoundException[/foo/bar]"))); assertThat(text, not(containsString("error_trace"))); assertThat(text, containsString("\"error\":\"No ElasticsearchException found\"")); @@ -100,10 +104,10 @@ public class BytesRestResponseTests extends ESTestCase { request.params().put("error_trace", "true"); RestChannel channel = new DetailedExceptionRestChannel(request); - Throwable t = new Throwable("an error occurred reading data", new FileNotFoundException("/foo/bar")); + Exception t = new UnknownException("an error occurred reading data", new FileNotFoundException("/foo/bar")); BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); - assertThat(text, containsString("\"type\":\"throwable\",\"reason\":\"an error occurred reading data\"")); + String text = response.content().utf8ToString(); + assertThat(text, containsString("\"type\":\"unknown_exception\",\"reason\":\"an error occurred reading data\"")); assertThat(text, containsString("{\"type\":\"file_not_found_exception\"")); assertThat(text, containsString("\"stack_trace\":\"[an error occurred reading data]")); } @@ -112,15 +116,15 @@ public class BytesRestResponseTests extends ESTestCase { RestRequest request = new FakeRestRequest(); RestChannel channel = new DetailedExceptionRestChannel(request); { - Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar")); - BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); + Exception e = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar")); + BytesRestResponse response = new BytesRestResponse(channel, e); + String text = response.content().utf8ToString(); assertThat(text, containsString("{\"root_cause\":[{\"type\":\"exception\",\"reason\":\"an error occurred reading data\"}]")); } { - Throwable t = new FileNotFoundException("/foo/bar"); - BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); + Exception e = new FileNotFoundException("/foo/bar"); + BytesRestResponse response = new BytesRestResponse(channel, e); + String text = response.content().utf8ToString(); assertThat(text, containsString("{\"root_cause\":[{\"type\":\"file_not_found_exception\",\"reason\":\"/foo/bar\"}]")); } } @@ -130,7 +134,7 @@ public class BytesRestResponseTests extends ESTestCase { RestChannel channel = new SimpleExceptionRestChannel(request); BytesRestResponse response = new BytesRestResponse(channel, null); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); assertThat(text, containsString("\"error\":\"unknown\"")); assertThat(text, not(containsString("error_trace"))); } @@ -144,7 +148,7 @@ public class BytesRestResponseTests extends ESTestCase { new SearchShardTarget("node_1", new Index("foo", "_na_"), 2)); SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[] {failure, failure1}); BytesRestResponse response = new BytesRestResponse(channel, new RemoteTransportException("foo", ex)); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); String expected = "{\"error\":{\"root_cause\":[{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}],\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]},\"status\":400}"; assertEquals(expected.trim(), text.trim()); String stackTrace = ExceptionsHelper.stackTrace(ex); @@ -160,7 +164,7 @@ public class BytesRestResponseTests extends ESTestCase { // if we try to decode the path, this will throw an IllegalArgumentException again final BytesRestResponse response = new BytesRestResponse(channel, e); assertNotNull(response.content()); - final String content = response.content().toUtf8(); + final String content = response.content().utf8ToString(); assertThat(content, containsString("\"type\":\"illegal_argument_exception\"")); assertThat(content, containsString("\"reason\":\"partial escape sequence at end of string: %a\"")); assertThat(content, containsString("\"status\":" + 400)); @@ -171,7 +175,7 @@ public class BytesRestResponseTests extends ESTestCase { final RestChannel channel = new DetailedExceptionRestChannel(request); final BytesRestResponse response = new BytesRestResponse(channel, new ElasticsearchException("simulated")); assertNotNull(response.content()); - final String content = response.content().toUtf8(); + final String content = response.content().utf8ToString(); assertThat(content, containsString("\"type\":\"exception\"")); assertThat(content, containsString("\"reason\":\"simulated\"")); assertThat(content, containsString("\"status\":" + 500)); diff --git a/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java b/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java index dd6d1dac47b..19b9051dd7e 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java @@ -142,7 +142,7 @@ public class RestFilterChainTests extends ESTestCase { assertThat(fakeRestChannel.errors().get(), equalTo(additionalContinueCount)); } - private static enum Operation implements Callback { + private enum Operation implements Callback { CONTINUE_PROCESSING { @Override public void execute(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception { @@ -157,7 +157,7 @@ public class RestFilterChainTests extends ESTestCase { } } - private static interface Callback { + private interface Callback { void execute(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception; } diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java new file mode 100644 index 00000000000..db2148b9f61 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java @@ -0,0 +1,162 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.cat; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsTests; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.common.Table; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.cache.query.QueryCacheStats; +import org.elasticsearch.index.cache.request.RequestCacheStats; +import org.elasticsearch.index.engine.SegmentsStats; +import org.elasticsearch.index.fielddata.FieldDataStats; +import org.elasticsearch.index.flush.FlushStats; +import org.elasticsearch.index.get.GetStats; +import org.elasticsearch.index.merge.MergeStats; +import org.elasticsearch.index.refresh.RefreshStats; +import org.elasticsearch.index.search.stats.SearchStats; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.IndexingStats; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.index.warmer.WarmerStats; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.search.suggest.completion.CompletionStats; +import org.elasticsearch.test.ESTestCase; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.hamcrest.Matchers.equalTo; + +/** + * Tests for {@link RestIndicesAction} + */ +public class RestIndicesActionTests extends ESTestCase { + + public void testBuildTable() { + final Settings settings = Settings.EMPTY; + final RestController restController = new RestController(settings); + final RestIndicesAction action = new RestIndicesAction(settings, restController, new IndexNameExpressionResolver(settings)); + + // build a (semi-)random table + final int numIndices = randomIntBetween(0, 5); + Index[] indices = new Index[numIndices]; + for (int i = 0; i < numIndices; i++) { + indices[i] = new Index(randomAsciiOfLength(5), UUIDs.randomBase64UUID()); + } + + final MetaData.Builder metaDataBuilder = MetaData.builder(); + for (final Index index : indices) { + metaDataBuilder.put(IndexMetaData.builder(index.getName()) + .settings(Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())) + .creationDate(System.currentTimeMillis()) + .numberOfShards(1) + .numberOfReplicas(1) + .state(IndexMetaData.State.OPEN)); + } + final MetaData metaData = metaDataBuilder.build(); + + final ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metaData(metaData) + .build(); + final String[] indicesStr = new String[indices.length]; + for (int i = 0; i < indices.length; i++) { + indicesStr[i] = indices[i].getName(); + } + final ClusterHealthResponse clusterHealth = new ClusterHealthResponse( + clusterState.getClusterName().value(), indicesStr, clusterState, 0, 0, 0, TimeValue.timeValueMillis(1000L) + ); + + final Table table = action.buildTable(null, indices, clusterHealth, randomIndicesStatsResponse(indices), metaData); + + // now, verify the table is correct + int count = 0; + List headers = table.getHeaders(); + assertThat(headers.get(count++).value, equalTo("health")); + assertThat(headers.get(count++).value, equalTo("status")); + assertThat(headers.get(count++).value, equalTo("index")); + assertThat(headers.get(count++).value, equalTo("uuid")); + + List> rows = table.getRows(); + assertThat(rows.size(), equalTo(indices.length)); + // TODO: more to verify (e.g. randomize cluster health, num primaries, num replicas, etc) + for (int i = 0; i < rows.size(); i++) { + count = 0; + final List row = rows.get(i); + assertThat(row.get(count++).value, equalTo("red*")); // all are red because cluster state doesn't have routing entries + assertThat(row.get(count++).value, equalTo("open")); // all are OPEN for now + assertThat(row.get(count++).value, equalTo(indices[i].getName())); + assertThat(row.get(count++).value, equalTo(indices[i].getUUID())); + } + } + + private IndicesStatsResponse randomIndicesStatsResponse(final Index[] indices) { + List shardStats = new ArrayList<>(); + for (final Index index : indices) { + for (int i = 0; i < 2; i++) { + ShardId shardId = new ShardId(index, i); + Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i)); + ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, null, i == 0, + new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE); + shardRouting = shardRouting.moveToStarted(); + CommonStats stats = new CommonStats(); + stats.fieldData = new FieldDataStats(); + stats.queryCache = new QueryCacheStats(); + stats.docs = new DocsStats(); + stats.store = new StoreStats(); + stats.indexing = new IndexingStats(); + stats.search = new SearchStats(); + stats.segments = new SegmentsStats(); + stats.merge = new MergeStats(); + stats.refresh = new RefreshStats(); + stats.completion = new CompletionStats(); + stats.requestCache = new RequestCacheStats(); + stats.get = new GetStats(); + stats.flush = new FlushStats(); + stats.warmer = new WarmerStats(); + shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null)); + } + } + return IndicesStatsTests.newIndicesStatsResponse( + shardStats.toArray(new ShardStats[shardStats.size()]), shardStats.size(), shardStats.size(), 0, emptyList() + ); + } +} diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java index ebb7dd255aa..ffefa074df7 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java @@ -95,6 +95,6 @@ public class RestMainActionTests extends ESTestCase { } mainResponse.toXContent(responseBuilder, ToXContent.EMPTY_PARAMS); BytesReference xcontentBytes = responseBuilder.bytes(); - assertTrue(BytesReference.Helper.bytesEqual(xcontentBytes, response.content())); + assertEquals(xcontentBytes, response.content()); } } diff --git a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java index a7e17785d48..3dfae8cc4f8 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java @@ -169,7 +169,7 @@ public class RestTableTests extends ESTestCase { private void assertResponse(Map headers, String mediaType, String body) throws Exception { RestResponse response = assertResponseContentType(headers, mediaType); - assertThat(response.content().toUtf8(), equalTo(body)); + assertThat(response.content().utf8ToString(), equalTo(body)); } private List getHeaderNames(List headers) { diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java index d8d6b0f5409..658a3bf5658 100644 --- a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java +++ b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java @@ -29,8 +29,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke public class StoredScriptsIT extends ESIntegTestCase { - private final static int SCRIPT_MAX_SIZE_IN_BYTES = 64; - private final static String LANG = MockScriptEngine.NAME; + private static final int SCRIPT_MAX_SIZE_IN_BYTES = 64; + private static final String LANG = MockScriptEngine.NAME; @Override protected Settings nodeSettings(int nodeOrdinal) { diff --git a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index 2ca255ea1a3..192f40d4b2b 100644 --- a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -43,13 +43,13 @@ public class DocValueFormatTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); out.writeNamedWriteable(DocValueFormat.BOOLEAN); - StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); assertSame(DocValueFormat.BOOLEAN, in.readNamedWriteable(DocValueFormat.class)); DocValueFormat.Decimal decimalFormat = new DocValueFormat.Decimal("###.##"); out = new BytesStreamOutput(); out.writeNamedWriteable(decimalFormat); - in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); DocValueFormat vf = in.readNamedWriteable(DocValueFormat.class); assertEquals(DocValueFormat.Decimal.class, vf.getClass()); assertEquals("###.##", ((DocValueFormat.Decimal) vf).pattern); @@ -57,7 +57,7 @@ public class DocValueFormatTests extends ESTestCase { DocValueFormat.DateTime dateFormat = new DocValueFormat.DateTime(Joda.forPattern("epoch_second"), DateTimeZone.forOffsetHours(1)); out = new BytesStreamOutput(); out.writeNamedWriteable(dateFormat); - in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); vf = in.readNamedWriteable(DocValueFormat.class); assertEquals(DocValueFormat.DateTime.class, vf.getClass()); assertEquals("epoch_second", ((DocValueFormat.DateTime) vf).formatter.format()); @@ -65,17 +65,17 @@ public class DocValueFormatTests extends ESTestCase { out = new BytesStreamOutput(); out.writeNamedWriteable(DocValueFormat.GEOHASH); - in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); assertSame(DocValueFormat.GEOHASH, in.readNamedWriteable(DocValueFormat.class)); out = new BytesStreamOutput(); out.writeNamedWriteable(DocValueFormat.IP); - in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); assertSame(DocValueFormat.IP, in.readNamedWriteable(DocValueFormat.class)); out = new BytesStreamOutput(); out.writeNamedWriteable(DocValueFormat.RAW); - in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class)); } diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java index a4837e382ac..5caba0fb441 100644 --- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java +++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -753,35 +753,35 @@ public class MultiValueModeTests extends ESTestCase { public void testWriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { MultiValueMode.SUM.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { MultiValueMode.AVG.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { MultiValueMode.MEDIAN.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(2)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { MultiValueMode.MIN.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(3)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { MultiValueMode.MAX.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(4)); } } @@ -790,35 +790,35 @@ public class MultiValueModeTests extends ESTestCase { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.SUM)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.AVG)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(2); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.MEDIAN)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(3); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.MIN)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(4); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.MAX)); } } diff --git a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java index 2bb39ad10ea..6542bad5b8a 100644 --- a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java +++ b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java @@ -65,7 +65,7 @@ public class SearchWithRejectionsIT extends ESIntegTestCase { for (int i = 0; i < numSearches; i++) { try { responses[i].get(); - } catch (Throwable t) { + } catch (Exception t) { } } awaitBusy(() -> client().admin().indices().prepareStats().execute().actionGet().getTotal().getSearch().getOpenContexts() == 0, 1, TimeUnit.SECONDS); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java index a4103e7ee56..ddde1fd9eb6 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java @@ -215,7 +215,7 @@ public abstract class BaseAggregationTestCase categoryToControl = new HashMap<>(); + private static final Map categoryToControl = new HashMap<>(); @Override public void setupSuiteScopeCluster() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java index a48facc4d66..0c8b9a22c37 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java @@ -52,9 +52,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; -/** - * - */ @ESIntegTestCase.SuiteScopeTestCase public class FilterIT extends ESIntegTestCase { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java index a95df3ff5e6..592861ccce2 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java @@ -58,9 +58,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; -/** - * - */ @ESIntegTestCase.SuiteScopeTestCase public class FiltersIT extends ESIntegTestCase { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java index 49ef9e1b6b5..5cc6ec58630 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -181,7 +181,7 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase { // Added to debug a test failure where the terms aggregation seems to be reporting two documents with the same value for NUMBER_FIELD_NAME. This will check that after // random indexing each document only has 1 value for NUMBER_FIELD_NAME and it is the correct value. Following this initial change its seems that this call was getting // more that 2000 hits (actual value was 2059) so now it will also check to ensure all hits have the correct index and type - SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME).addField(NUMBER_FIELD_NAME).addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME) + SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME).addStoredField(NUMBER_FIELD_NAME).addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME) .order(SortOrder.ASC)).setSize(5000).get(); assertSearchResponse(response); long totalHits = response.getHits().totalHits(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java index 36c4caae12d..97d5cf1f9ee 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java @@ -38,14 +38,14 @@ public class PercentilesMethodTests extends ESTestCase { public void testwriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { PercentilesMethod.TDIGEST.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { PercentilesMethod.HDR.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } @@ -54,13 +54,13 @@ public class PercentilesMethodTests extends ESTestCase { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(PercentilesMethod.readFromStream(in), equalTo(PercentilesMethod.TDIGEST)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(PercentilesMethod.readFromStream(in), equalTo(PercentilesMethod.HDR)); } } @@ -69,7 +69,7 @@ public class PercentilesMethodTests extends ESTestCase { public void testInvalidReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE)); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { PercentilesMethod.readFromStream(in); fail("Expected IOException"); } catch(IOException e) { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java index e1441b0dc54..390501d2002 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder; import static org.hamcrest.Matchers.equalTo; @@ -51,7 +50,7 @@ public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase CONTEXT_FACTORY = new ContextFactory() { diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java index 14d0fc959c3..4ba1b902fef 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java @@ -355,7 +355,7 @@ public class QueryRescorerIT extends ESIntegTestCase { // and shard id are equal during merging shard results. // This comparator uses a custom tie in case the scores are equal, so that both regular hits and rescored hits // are sorted equally. This is fine since tests only care about the fact the scores should be equal, not ordering. - private final static Comparator searchHitsComparator = new Comparator() { + private static final Comparator searchHitsComparator = new Comparator() { @Override public int compare(SearchHit hit1, SearchHit hit2) { int cmp = Float.compare(hit2.getScore(), hit1.getScore()); diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java index 0debdb263af..985605c4e65 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java @@ -415,13 +415,13 @@ public class GeoFilterIT extends ESIntegTestCase { assertThat(hit.getId(), equalTo(key)); } - SearchResponse world = client().prepareSearch().addField("pin").setQuery( + SearchResponse world = client().prepareSearch().addStoredField("pin").setQuery( geoBoundingBoxQuery("pin").setCorners(90, -179.99999, -90, 179.99999) ).execute().actionGet(); assertHitCount(world, 53); - SearchResponse distance = client().prepareSearch().addField("pin").setQuery( + SearchResponse distance = client().prepareSearch().addStoredField("pin").setQuery( geoDistanceQuery("pin").distance("425km").point(51.11, 9.851) ).execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java index 930a7b220e0..1e3c5453fd7 100644 --- a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java @@ -484,14 +484,14 @@ public class HighlightBuilderTests extends ESTestCase { public void testOrderSerialization() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { Order.NONE.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { Order.SCORE.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } @@ -738,7 +738,7 @@ public class HighlightBuilderTests extends ESTestCase { private static HighlightBuilder serializedCopy(HighlightBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return new HighlightBuilder(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java index 01f98564814..56b33c6007e 100644 --- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java @@ -156,7 +156,7 @@ public class InnerHitsIT extends ESIntegTestCase { .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit( new InnerHitBuilder().setHighlightBuilder(new HighlightBuilder().field("comments.message")) .setExplain(true) - .addFieldDataField("comments.message") + .addDocValueField("comments.message") .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap())) .setSize(1) )).get(); @@ -287,7 +287,7 @@ public class InnerHitsIT extends ESIntegTestCase { .setQuery( hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit( new InnerHitBuilder() - .addFieldDataField("message") + .addDocValueField("message") .setHighlightBuilder(new HighlightBuilder().field("message")) .setExplain(true).setSize(1) .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, diff --git a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java index 77fc2f0e6a9..dedd47d3e43 100644 --- a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java +++ b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java @@ -67,7 +67,7 @@ public class InternalSearchHitTests extends ESTestCase { context.streamShardTarget(InternalSearchHits.StreamContext.ShardTargetType.STREAM); BytesStreamOutput output = new BytesStreamOutput(); hits.writeTo(output, context); - InputStream input = new ByteArrayInputStream(output.bytes().toBytes()); + InputStream input = output.bytes().streamInput(); context = new InternalSearchHits.StreamContext(); context.streamShardTarget(InternalSearchHits.StreamContext.ShardTargetType.STREAM); InternalSearchHits results = InternalSearchHits.readSearchHits(new InputStreamStreamInput(input), context); diff --git a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java index ef7a4ecc7ce..300c4f141b0 100644 --- a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java +++ b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java @@ -362,8 +362,8 @@ public class MatchedQueriesIT extends ESIntegTestCase { refresh(); QueryBuilder[] queries = new QueryBuilder[]{ - wrapperQuery(matchQuery("content", "amet").queryName("abc").buildAsBytes().toUtf8()), - constantScoreQuery(wrapperQuery(termQuery("content", "amet").queryName("abc").buildAsBytes().toUtf8())) + wrapperQuery(matchQuery("content", "amet").queryName("abc").buildAsBytes().utf8ToString()), + constantScoreQuery(wrapperQuery(termQuery("content", "amet").queryName("abc").buildAsBytes().utf8ToString())) }; for (QueryBuilder query : queries) { SearchResponse searchResponse = client().prepareSearch() diff --git a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java index 875256a0f92..766aff8d274 100644 --- a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java @@ -623,7 +623,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase { assertFirstHit(searchResponse, hasId("ultimate1")); } - private static final void assertEquivalent(String query, SearchResponse left, SearchResponse right) { + private static void assertEquivalent(String query, SearchResponse left, SearchResponse right) { assertNoFailures(left); assertNoFailures(right); SearchHits leftHits = left.getHits(); diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java index 7c3690dcf35..28eb56bdcaf 100644 --- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java @@ -340,7 +340,7 @@ public class QueryRescoreBuilderTests extends ESTestCase { private static RescoreBuilder serializedCopy(RescoreBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { output.writeNamedWriteable(original); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return in.readNamedWriteable(RescoreBuilder.class); } } diff --git a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java index 3c675926328..88af1ddf2b6 100644 --- a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java @@ -65,7 +65,7 @@ public class SearchAfterBuilderTests extends ESTestCase { indicesQueriesRegistry = null; } - private final SearchAfterBuilder randomSearchFromBuilder() throws IOException { + private SearchAfterBuilder randomSearchFromBuilder() throws IOException { int numSearchFrom = randomIntBetween(1, 10); SearchAfterBuilder searchAfterBuilder = new SearchAfterBuilder(); Object[] values = new Object[numSearchFrom]; @@ -112,7 +112,7 @@ public class SearchAfterBuilderTests extends ESTestCase { // ensure that every number type remain the same before/after xcontent (de)serialization. // This is not a problem because the final type of each field value is extracted from associated sort field. // This little trick ensure that equals and hashcode are the same when using the xcontent serialization. - private final SearchAfterBuilder randomJsonSearchFromBuilder() throws IOException { + private SearchAfterBuilder randomJsonSearchFromBuilder() throws IOException { int numSearchAfter = randomIntBetween(1, 10); XContentBuilder jsonBuilder = XContentFactory.jsonBuilder(); jsonBuilder.startObject(); @@ -164,7 +164,7 @@ public class SearchAfterBuilderTests extends ESTestCase { private static SearchAfterBuilder serializedCopy(SearchAfterBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return new SearchAfterBuilder(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index 217f97ace0a..668351c6db9 100644 --- a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -88,7 +88,7 @@ public class SliceBuilderTests extends ESTestCase { indicesQueriesRegistry = null; } - private final SliceBuilder randomSliceBuilder() throws IOException { + private SliceBuilder randomSliceBuilder() throws IOException { int max = randomIntBetween(2, MAX_SLICE); int id = randomInt(max - 1); String field = randomAsciiOfLengthBetween(5, 20); @@ -99,7 +99,7 @@ public class SliceBuilderTests extends ESTestCase { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); try (StreamInput in = - new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return new SliceBuilder(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index b494fa4d1e6..8d46372aa42 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -272,7 +272,7 @@ public abstract class AbstractSortTestCase> extends EST private T copyItem(T original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return (T) namedWriteableRegistry.getReader(SortBuilder.class, original.getWriteableName()).read(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java index 2de48decbd8..208b4ed1b53 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java @@ -37,7 +37,7 @@ public class SortOrderTests extends ESTestCase { for (SortOrder unit : SortOrder.values()) { try (BytesStreamOutput out = new BytesStreamOutput()) { unit.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat("Roundtrip serialisation failed.", SortOrder.readFromStream(in), equalTo(unit)); } } diff --git a/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java b/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java index 33fcb55cada..df147ce106f 100644 --- a/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java +++ b/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java @@ -37,10 +37,10 @@ public class SourceFetchingIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("test").get(); assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue()); - response = client().prepareSearch("test").addField("bla").get(); + response = client().prepareSearch("test").addStoredField("bla").get(); assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue()); - response = client().prepareSearch("test").addField("_source").get(); + response = client().prepareSearch("test").addStoredField("_source").get(); assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue()); } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index cd6c34497f7..b67036e1152 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -215,7 +215,7 @@ public abstract class AbstractSuggestionBuilderTestCase extends ESTestCase { private M copyModel(M original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), provideNamedWritableRegistry())) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), provideNamedWritableRegistry())) { return readFrom(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java index 3fd3850b98a..846d3193f6d 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java @@ -206,7 +206,7 @@ public class DirectCandidateGeneratorTests extends ESTestCase{ private static DirectCandidateGeneratorBuilder serializedCopy(DirectCandidateGeneratorBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = StreamInput.wrap(output.bytes())) { + try (StreamInput in = output.bytes().streamInput()) { return new DirectCandidateGeneratorBuilder(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java index f167eefa43d..c7b883b583b 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java @@ -180,7 +180,7 @@ public abstract class SmoothingModelTestCase extends ESTestCase { static SmoothingModel copyModel(SmoothingModel original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return namedWriteableRegistry.getReader(SmoothingModel.class, original.getWriteableName()).read(in); } } diff --git a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 50fb3f9074b..bbd1bec1d45 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -279,8 +279,8 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase { } @Override - public void onFailure(String source, Throwable t) { - logger.warn("failed to execute [{}]", t, source); + public void onFailure(String source, Exception e) { + logger.warn("failed to execute [{}]", e, source); } }); diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 19c508e2bb1..9d571c02c90 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -272,7 +272,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest } @Override - public void onFailure(String source, @Nullable Throwable t) { + public void onFailure(String source, @Nullable Exception e) { countDownLatch.countDown(); } @@ -284,8 +284,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest countDownLatch.await(); } - private static interface ClusterStateUpdater { - public ClusterState execute(ClusterState currentState) throws Exception; + private interface ClusterStateUpdater { + ClusterState execute(ClusterState currentState) throws Exception; } public void testSnapshotDuringNodeShutdown() throws Exception { @@ -392,8 +392,11 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest logger.info("--> making sure that snapshot no longer exists"); assertThrows(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute(), SnapshotMissingException.class); - // Subtract index file from the count - assertThat("not all files were deleted during snapshot cancellation", numberOfFilesBeforeSnapshot, equalTo(numberOfFiles(repo) - 1)); + // Subtract three files that will remain in the repository: + // (1) index-1 + // (2) index-0 (because we keep the previous version) and + // (3) index-latest + assertThat("not all files were deleted during snapshot cancellation", numberOfFilesBeforeSnapshot, equalTo(numberOfFiles(repo) - 3)); logger.info("--> done"); } @@ -646,8 +649,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest @Override public void sendResponse(RestResponse response) { try { - assertThat(response.content().toUtf8(), containsString("notsecretusername")); - assertThat(response.content().toUtf8(), not(containsString("verysecretpassword"))); + assertThat(response.content().utf8ToString(), containsString("notsecretusername")); + assertThat(response.content().utf8ToString(), not(containsString("verysecretpassword"))); } catch (AssertionError ex) { getRepoError.set(ex); } @@ -667,8 +670,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest @Override public void sendResponse(RestResponse response) { try { - assertThat(response.content().toUtf8(), containsString("notsecretusername")); - assertThat(response.content().toUtf8(), not(containsString("verysecretpassword"))); + assertThat(response.content().utf8ToString(), containsString("notsecretusername")); + assertThat(response.content().utf8ToString(), not(containsString("verysecretpassword"))); } catch (AssertionError ex) { clusterStateError.set(ex); } diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java index 79d1497912a..48a1cc6081e 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -40,6 +40,7 @@ import java.util.List; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -153,7 +154,9 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase { .get(); fail("Shouldn't be here"); } catch (RepositoryException ex) { - assertThat(ex.toString(), containsString("unsupported url protocol [netdoc]")); + assertThat(ex.toString(), + either(containsString("unsupported url protocol [netdoc]")) + .or(containsString("unknown protocol: netdoc"))); // newer versions of JDK 9 } logger.info("--> trying creating url repository with location that is not registered in path.repo setting"); diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 19b46710fea..f6049002852 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -818,8 +818,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas logger.info("--> delete the last snapshot"); client.admin().cluster().prepareDeleteSnapshot("test-repo", lastSnapshot).get(); - logger.info("--> make sure that number of files is back to what it was when the first snapshot was made"); - assertThat(numberOfFiles(repo), equalTo(numberOfFiles[0])); + logger.info("--> make sure that number of files is back to what it was when the first snapshot was made, " + + "plus one because one backup index-N file should remain"); + assertThat(numberOfFiles(repo), equalTo(numberOfFiles[0] + 1)); } public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exception { @@ -2041,7 +2042,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { fail(); } diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java index 38d858c49aa..c178b2a6f83 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.snapshots; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ESTestCase; @@ -75,7 +76,7 @@ public class SnapshotRequestsTests extends ESTestCase { builder.endArray(); } - byte[] bytes = builder.endObject().bytes().toBytes(); + byte[] bytes = BytesReference.toBytes(builder.endObject().bytes()); request.source(bytes); @@ -134,7 +135,7 @@ public class SnapshotRequestsTests extends ESTestCase { builder.endArray(); } - byte[] bytes = builder.endObject().bytes().toBytes(); + byte[] bytes = BytesReference.toBytes(builder.endObject().bytes()); request.source(bytes); diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java index cb297785e4b..41cfa3d4141 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java @@ -20,12 +20,10 @@ package org.elasticsearch.snapshots; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.nio.ByteBuffer; import static org.hamcrest.CoreMatchers.equalTo; @@ -50,8 +48,7 @@ public class SnapshotTests extends ESTestCase { final Snapshot original = new Snapshot(randomAsciiOfLength(randomIntBetween(2, 8)), snapshotId); final BytesStreamOutput out = new BytesStreamOutput(); original.writeTo(out); - final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes())); - assertThat(new Snapshot(in), equalTo(original)); + assertThat(new Snapshot(out.bytes().streamInput()), equalTo(original)); } } diff --git a/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java index bfbb2dff4c7..5b507436129 100644 --- a/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java +++ b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java @@ -37,7 +37,7 @@ import java.util.Map; import java.util.TreeMap; /** - * Round trip tests for {@link PersistedTaskInfo} and those classes that it includes like {@link TaskInfo} and {@link RawTaskStatus}. + * Round trip tests for {@link PersistedTaskInfo} and those classes that it includes like {@link TaskInfo} and {@link RawTaskStatus}. */ public class PersistedTaskInfoTests extends ESTestCase { public void testBinaryRoundTrip() throws IOException { @@ -47,7 +47,7 @@ public class PersistedTaskInfoTests extends ESTestCase { PersistedTaskInfo read; try (BytesStreamOutput out = new BytesStreamOutput()) { result.writeTo(out); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry)) { read = new PersistedTaskInfo(in); } } catch (IOException e) { diff --git a/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java b/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java index b13de26b976..f7990cfacb7 100644 --- a/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java +++ b/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java @@ -57,7 +57,7 @@ public class TaskIdTests extends ESTestCase { taskId.writeTo(out); BytesReference bytes = out.bytes(); assertEquals(expectedSize, bytes.length()); - try (StreamInput in = StreamInput.wrap(bytes)) { + try (StreamInput in = bytes.streamInput()) { return TaskId.readFromStream(in); } } diff --git a/core/src/test/java/org/elasticsearch/test/MockLogAppender.java b/core/src/test/java/org/elasticsearch/test/MockLogAppender.java index 9e4a881b25b..8f10ccd6537 100644 --- a/core/src/test/java/org/elasticsearch/test/MockLogAppender.java +++ b/core/src/test/java/org/elasticsearch/test/MockLogAppender.java @@ -34,7 +34,7 @@ import static org.hamcrest.MatcherAssert.assertThat; */ public class MockLogAppender extends AppenderSkeleton { - private final static String COMMON_PREFIX = System.getProperty("es.logger.prefix", "org.elasticsearch."); + private static final String COMMON_PREFIX = System.getProperty("es.logger.prefix", "org.elasticsearch."); private List expectations; @@ -75,7 +75,7 @@ public class MockLogAppender extends AppenderSkeleton { void assertMatched(); } - public static abstract class AbstractEventExpectation implements LoggingExpectation { + public abstract static class AbstractEventExpectation implements LoggingExpectation { protected final String name; protected final String logger; protected final Level level; diff --git a/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java b/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java index 3193aaf458e..c36082f1475 100644 --- a/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java +++ b/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java @@ -20,7 +20,7 @@ package org.elasticsearch.test; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.RoutingService; +import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.discovery.Discovery; @@ -41,7 +41,7 @@ public class NoopDiscovery implements Discovery { } @Override - public void setRoutingService(RoutingService routingService) { + public void setAllocationService(AllocationService allocationService) { } @@ -86,17 +86,11 @@ public class NoopDiscovery implements Discovery { } @Override - public Discovery start() { - return null; - } + public void start() {} @Override - public Discovery stop() { - return null; - } + public void stop() {} @Override - public void close() { - - } + public void close() {} } diff --git a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java index 897fa44b593..20c82e6f518 100644 --- a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java +++ b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java @@ -20,16 +20,9 @@ package org.elasticsearch.test.geo; import com.carrotsearch.randomizedtesting.generators.RandomInts; -import org.locationtech.spatial4j.context.jts.JtsSpatialContext; -import org.locationtech.spatial4j.distance.DistanceUtils; -import org.locationtech.spatial4j.exception.InvalidShapeException; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Rectangle; -import org.locationtech.spatial4j.shape.impl.Range; import com.vividsolutions.jts.algorithm.ConvexHull; import com.vividsolutions.jts.geom.Coordinate; import com.vividsolutions.jts.geom.Geometry; - import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.geo.builders.CoordinateCollection; import org.elasticsearch.common.geo.builders.CoordinatesBuilder; @@ -42,6 +35,12 @@ import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.search.geo.GeoShapeQueryTests; import org.junit.Assert; +import org.locationtech.spatial4j.context.jts.JtsSpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.exception.InvalidShapeException; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.Rectangle; +import org.locationtech.spatial4j.shape.impl.Range; import java.util.Random; @@ -230,14 +229,10 @@ public class RandomShapeGenerator extends RandomGeoGenerator { // The validate flag will check for these possibilities and bail if an incorrect geometry is created try { pgb.build(); - } catch (Throwable e) { + } catch (AssertionError | InvalidShapeException e) { // jts bug may occasionally misinterpret coordinate order causing an unhelpful ('geom' assertion) // or InvalidShapeException - if (e instanceof InvalidShapeException || e instanceof AssertionError) { - return null; - } - // throw any other exception - throw e; + return null; } } return pgb; diff --git a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java index 28267e9beb7..974929dddf2 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java +++ b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java @@ -19,8 +19,6 @@ package org.elasticsearch.threadpool; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; @@ -36,7 +34,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.hamcrest.RegexMatcher; -import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.tribe.TribeIT; import java.io.IOException; @@ -46,19 +43,11 @@ import java.lang.management.ThreadMXBean; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.BrokenBarrierException; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.sameInstance; /** */ @@ -136,9 +125,9 @@ public class SimpleThreadPoolIT extends ESIntegTestCase { try { new Node(settings); fail("The node startup is supposed to fail"); - } catch(Throwable t) { + } catch(Exception e) { //all good - assertThat(t.getMessage(), containsString("mandatory plugins [non_existing]")); + assertThat(e.getMessage(), containsString("mandatory plugins [non_existing]")); } } diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java index 486b0635c64..14cf10b8f31 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java @@ -58,7 +58,7 @@ public class ThreadPoolSerializationTests extends ESTestCase { output.setVersion(Version.CURRENT); info.writeTo(output); - StreamInput input = StreamInput.wrap(output.bytes()); + StreamInput input = output.bytes().streamInput(); ThreadPool.Info newInfo = new ThreadPool.Info(); newInfo.readFrom(input); @@ -70,7 +70,7 @@ public class ThreadPoolSerializationTests extends ESTestCase { output.setVersion(Version.CURRENT); info.writeTo(output); - StreamInput input = StreamInput.wrap(output.bytes()); + StreamInput input = output.bytes().streamInput(); ThreadPool.Info newInfo = new ThreadPool.Info(); newInfo.readFrom(input); @@ -125,7 +125,7 @@ public class ThreadPoolSerializationTests extends ESTestCase { output.setVersion(Version.CURRENT); info.writeTo(output); - StreamInput input = StreamInput.wrap(output.bytes()); + StreamInput input = output.bytes().streamInput(); ThreadPool.Info newInfo = new ThreadPool.Info(); newInfo.readFrom(input); diff --git a/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java index a284f6ea911..bc10458378f 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java @@ -18,11 +18,10 @@ */ package org.elasticsearch.transport.netty; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; -import org.elasticsearch.transport.netty.NettyUtils; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; @@ -38,17 +37,16 @@ public class ChannelBufferBytesReferenceTests extends AbstractBytesReferenceTest assertEquals(out.size(), length); BytesReference ref = out.bytes(); assertEquals(ref.length(), length); - BytesArray bytesArray = ref.toBytesArray(); - return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(), - bytesArray.length())); + BytesRef bytesRef = ref.toBytesRef(); + final ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset, bytesRef.length); + return NettyUtils.toBytesReference(channelBuffer); } public void testSliceOnAdvancedBuffer() throws IOException { BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE)); - BytesArray bytesArray = bytesReference.toBytesArray(); - - ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(), - bytesArray.length()); + BytesRef bytesRef = bytesReference.toBytesRef(); + ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length); int numBytesToRead = randomIntBetween(1, 5); for (int i = 0; i < numBytesToRead; i++) { channelBuffer.readByte(); @@ -56,7 +54,25 @@ public class ChannelBufferBytesReferenceTests extends AbstractBytesReferenceTest BytesReference other = NettyUtils.toBytesReference(channelBuffer); BytesReference slice = bytesReference.slice(numBytesToRead, bytesReference.length() - numBytesToRead); assertEquals(other, slice); - assertEquals(other.slice(3, 1), slice.slice(3, 1)); } + + public void testImmutable() throws IOException { + BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE)); + BytesRef bytesRef = BytesRef.deepCopyOf(bytesReference.toBytesRef()); + ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length); + ChannelBufferBytesReference channelBufferBytesReference = new ChannelBufferBytesReference(channelBuffer, bytesRef.length); + assertEquals(channelBufferBytesReference, bytesReference); + channelBuffer.readInt(); // this advances the index of the channel buffer + assertEquals(channelBufferBytesReference, bytesReference); + assertEquals(bytesRef, channelBufferBytesReference.toBytesRef()); + + BytesRef unicodeBytes = new BytesRef(randomUnicodeOfCodepointLength(100)); + channelBuffer = ChannelBuffers.wrappedBuffer(unicodeBytes.bytes, unicodeBytes.offset, unicodeBytes.length); + channelBufferBytesReference = new ChannelBufferBytesReference(channelBuffer, unicodeBytes.length); + String utf8ToString = channelBufferBytesReference.utf8ToString(); + channelBuffer.readInt(); // this advances the index of the channel buffer + assertEquals(utf8ToString, channelBufferBytesReference.utf8ToString()); + } } diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java index fa8f30249bb..aa1db4a44c7 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.transport.netty; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; @@ -47,7 +49,7 @@ public class NettyUtilsTests extends ESTestCase { BytesReference slice = ref.slice(sliceOffset, sliceLength); ChannelBuffer channelBuffer = NettyUtils.toChannelBuffer(slice); BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer); - assertArrayEquals(slice.toBytes(), bytesReference.toBytes()); + assertArrayEquals(BytesReference.toBytes(slice), BytesReference.toBytes(bytesReference)); } public void testToChannelBufferWithSliceAfter() throws IOException { @@ -56,7 +58,8 @@ public class NettyUtilsTests extends ESTestCase { int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset); ChannelBuffer channelBuffer = NettyUtils.toChannelBuffer(ref); BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer); - assertArrayEquals(ref.slice(sliceOffset, sliceLength).toBytes(), bytesReference.slice(sliceOffset, sliceLength).toBytes()); + assertArrayEquals(BytesReference.toBytes(ref.slice(sliceOffset, sliceLength)), + BytesReference.toBytes(bytesReference.slice(sliceOffset, sliceLength))); } public void testToChannelBuffer() throws IOException { @@ -65,10 +68,10 @@ public class NettyUtilsTests extends ESTestCase { BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer); if (ref instanceof ChannelBufferBytesReference) { assertEquals(channelBuffer, ((ChannelBufferBytesReference) ref).toChannelBuffer()); - } else if (ref.hasArray() == false) { // we gather the buffers into a channel buffer + } else if (AbstractBytesReferenceTestCase.getNumPages(ref) > 1) { // we gather the buffers into a channel buffer assertTrue(channelBuffer instanceof CompositeChannelBuffer); } - assertArrayEquals(ref.toBytes(), bytesReference.toBytes()); + assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference)); } private BytesReference getRandomizedBytesReference(int length) throws IOException { @@ -81,13 +84,14 @@ public class NettyUtilsTests extends ESTestCase { BytesReference ref = out.bytes(); assertEquals(ref.length(), length); if (randomBoolean()) { - return ref.toBytesArray(); + return new BytesArray(ref.toBytesRef()); } else if (randomBoolean()) { - BytesArray bytesArray = ref.toBytesArray(); - return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(), - bytesArray.length())); + BytesRef bytesRef = ref.toBytesRef(); + return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length)); } else { return ref; } } + } diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java index 2bbedd8784b..f361496c537 100644 --- a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java +++ b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java @@ -27,7 +27,7 @@ public class TribeServiceTests extends ESTestCase { Settings globalSettings = Settings.builder() .put("node.name", "nodename") .put("path.home", "some/path").build(); - Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY); + Settings clientSettings = TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, Settings.EMPTY); assertEquals("some/path", clientSettings.get("path.home")); assertEquals("nodename/tribe1", clientSettings.get("node.name")); assertEquals("tribe1", clientSettings.get("tribe.name")); @@ -35,7 +35,9 @@ public class TribeServiceTests extends ESTestCase { assertEquals("false", clientSettings.get("node.master")); assertEquals("false", clientSettings.get("node.data")); assertEquals("false", clientSettings.get("node.ingest")); - assertEquals(7, clientSettings.getAsMap().size()); + assertEquals("false", clientSettings.get("node.local_storage")); + assertEquals("3707202549613653169", clientSettings.get("node.id.seed")); // should be fixed by the parent id and tribe name + assertEquals(9, clientSettings.getAsMap().size()); } public void testEnvironmentSettings() { @@ -45,7 +47,7 @@ public class TribeServiceTests extends ESTestCase { .put("path.conf", "conf/path") .put("path.scripts", "scripts/path") .put("path.logs", "logs/path").build(); - Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY); + Settings clientSettings = TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, Settings.EMPTY); assertEquals("some/path", clientSettings.get("path.home")); assertEquals("conf/path", clientSettings.get("path.conf")); assertEquals("scripts/path", clientSettings.get("path.scripts")); @@ -54,7 +56,7 @@ public class TribeServiceTests extends ESTestCase { Settings tribeSettings = Settings.builder() .put("path.home", "alternate/path").build(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { - TribeService.buildClientSettings("tribe1", globalSettings, tribeSettings); + TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, tribeSettings); }); assertTrue(e.getMessage(), e.getMessage().contains("Setting [path.home] not allowed in tribe client")); } @@ -69,7 +71,7 @@ public class TribeServiceTests extends ESTestCase { .put("transport.host", "3.3.3.3") .put("transport.bind_host", "4.4.4.4") .put("transport.publish_host", "5.5.5.5").build(); - Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY); + Settings clientSettings = TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, Settings.EMPTY); assertEquals("0.0.0.0", clientSettings.get("network.host")); assertEquals("1.1.1.1", clientSettings.get("network.bind_host")); assertEquals("2.2.2.2", clientSettings.get("network.publish_host")); @@ -85,7 +87,7 @@ public class TribeServiceTests extends ESTestCase { .put("transport.host", "6.6.6.6") .put("transport.bind_host", "7.7.7.7") .put("transport.publish_host", "8.8.8.8").build(); - clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, tribeSettings); + clientSettings = TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, tribeSettings); assertEquals("3.3.3.3", clientSettings.get("network.host")); assertEquals("4.4.4.4", clientSettings.get("network.bind_host")); assertEquals("5.5.5.5", clientSettings.get("network.publish_host")); diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java index 9e08ecde6fa..583d8a0288d 100644 --- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java +++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java @@ -59,7 +59,7 @@ import static org.hamcrest.Matchers.nullValue; @ClusterScope(scope= Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class SimpleTTLIT extends ESIntegTestCase { - static private final long PURGE_INTERVAL = 200; + private static final long PURGE_INTERVAL = 200; @Override protected int numberOfShards() { diff --git a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java index 3573089fcaa..e81b4decb2d 100644 --- a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java +++ b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java @@ -87,7 +87,7 @@ public class TimestampTTLBWIT extends ESIntegTestCase { .setQuery(matchAllQuery()) .setSize(randomIntBetween(1, numDocs + 5)) .addSort("_timestamp", order) - .addField("_timestamp") + .addStoredField("_timestamp") .execute().actionGet(); assertNoFailures(searchResponse); SearchHit[] hits = searchResponse.getHits().hits(); diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java index 55834c181b0..c8cacbc36c3 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java @@ -65,11 +65,8 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -800,7 +797,7 @@ public class UpdateIT extends ESIntegTestCase { final CountDownLatch latch = new CountDownLatch(numberOfThreads); final CountDownLatch startLatch = new CountDownLatch(1); final int numberOfUpdatesPerThread = scaledRandomIntBetween(100, 500); - final List failures = new CopyOnWriteArrayList<>(); + final List failures = new CopyOnWriteArrayList<>(); for (int i = 0; i < numberOfThreads; i++) { Runnable r = new Runnable() { @@ -832,7 +829,7 @@ public class UpdateIT extends ESIntegTestCase { logger.warn("Test was forcefully stopped. Client [{}] may still have outstanding requests.", Thread.currentThread().getName()); failures.add(e); Thread.currentThread().interrupt(); - } catch (Throwable e) { + } catch (Exception e) { failures.add(e); } finally { latch.countDown(); @@ -900,7 +897,7 @@ public class UpdateIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { synchronized (failedMap) { incrementMapValue(id, failedMap); } @@ -922,7 +919,7 @@ public class UpdateIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { synchronized (failedMap) { incrementMapValue(id, failedMap); } @@ -976,7 +973,7 @@ public class UpdateIT extends ESIntegTestCase { } } } - } catch (Throwable e) { + } catch (Exception e) { logger.error("Something went wrong", e); failures.add(e); } finally { diff --git a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java index b4b5eefc832..f14d91465f6 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java @@ -240,12 +240,12 @@ public class UpdateNoopIT extends ESIntegTestCase { private void updateAndCheckSource(long expectedVersion, Boolean detectNoop, XContentBuilder xContentBuilder) { UpdateResponse updateResponse = update(detectNoop, expectedVersion, xContentBuilder); - assertEquals(updateResponse.getGetResult().sourceRef().toUtf8(), xContentBuilder.bytes().toUtf8()); + assertEquals(updateResponse.getGetResult().sourceRef().utf8ToString(), xContentBuilder.bytes().utf8ToString()); } private UpdateResponse update(Boolean detectNoop, long expectedVersion, XContentBuilder xContentBuilder) { UpdateRequestBuilder updateRequest = client().prepareUpdate("test", "type1", "1") - .setDoc(xContentBuilder.bytes().toUtf8()) + .setDoc(xContentBuilder.bytes().utf8ToString()) .setDocAsUpsert(true) .setFields("_source"); if (detectNoop != null) { diff --git a/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java b/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java index b2cc794ac6c..e2c572f783a 100644 --- a/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java +++ b/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java @@ -53,7 +53,7 @@ public class ConcurrentDocumentOperationIT extends ESIntegTestCase { } @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { logger.error("Unexpected exception while indexing", e); failure.set(e); latch.countDown(); diff --git a/docs/plugins/ingest-useragent.asciidoc b/docs/plugins/ingest-useragent.asciidoc new file mode 100644 index 00000000000..8868a4cf711 --- /dev/null +++ b/docs/plugins/ingest-useragent.asciidoc @@ -0,0 +1,74 @@ +[[ingest-useragent]] +=== Ingest Useragent Processor Plugin + +The Useragent processor extracts details from the user agent string a browser sends with its web requests. +This processor adds this information by default under the `useragent` field. + +The ingest-useragent plugin ships by default with the regexes.yaml made available by uap-java with an Apache 2.0 license. For more details see https://github.com/ua-parser/uap-core. + +[[ingest-useragent-install]] +[float] +==== Installation + +This plugin can be installed using the plugin manager: + +[source,sh] +---------------------------------------------------------------- +sudo bin/elasticsearch-plugin install ingest-useragent +---------------------------------------------------------------- + +The plugin must be installed on every node in the cluster, and each node must +be restarted after installation. + +[[ingest-useragent-remove]] +[float] +==== Removal + +The plugin can be removed with the following command: + +[source,sh] +---------------------------------------------------------------- +sudo bin/elasticsearch-plugin remove ingest-useragent +---------------------------------------------------------------- + +The node must be stopped before removing the plugin. + +[[using-ingest-useragent]] +==== Using the Useragent Processor in a Pipeline + +[[ingest-useragent-options]] +.Useragent options +[options="header"] +|====== +| Name | Required | Default | Description +| `field` | yes | - | The field containing the user agent string. +| `target_field` | no | useragent | The field that will be filled with the user agent details. +| `regex_file` | no | - | The name of the file in the `config/ingest-useragent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-useragent will use the regexes.yaml from uap-core it ships with (see below). +| `properties` | no | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`. +|====== + +Here is an example that adds the user agent details to the `useragent` field based on the `agent` field: + +[source,js] +-------------------------------------------------- +{ + "description" : "...", + "processors" : [ + { + "useragent" : { + "field" : "agent" + } + } + ] +} +-------------------------------------------------- + +===== Using a custom regex file +To use a custom regex file for parsing the user agents, that file has to be put into the `config/ingest-useragent` directory and +has to have a `.yaml` filename extension. The file has to be present at node startup, any changes to it or any new files added +while the node is running will not have any effect. + +In practice, it will make most sense for any custom regex file to be a variant of the default file, either a more recent version +or a customised version. + +The default file included in `ingest-useragent` is the `regexes.yaml` from uap-core: https://github.com/ua-parser/uap-core/blob/master/regexes.yaml diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc index b287decd4cf..f2c034a317e 100644 --- a/docs/plugins/mapper-attachments.asciidoc +++ b/docs/plugins/mapper-attachments.asciidoc @@ -246,7 +246,7 @@ PUT /test/person/1?refresh=true } GET /test/person/_search { - "fields": [ "file.content_type" ], + "stored_fields": [ "file.content_type" ], "query": { "match": { "file.content_type": "text plain" @@ -367,7 +367,7 @@ PUT /test/person/1?refresh=true } GET /test/person/_search { - "fields": [], + "stored_fields": [], "query": { "match": { "file.content": "king queen" diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc index 3c1f4ae860a..959b93611d8 100644 --- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc @@ -9,8 +9,8 @@ Example: -------------------------------------------------- { "aggs" : { - "genders" : { - "terms" : { "field" : "gender" } + "genres" : { + "terms" : { "field" : "genre" } } } } @@ -24,16 +24,20 @@ Response: ... "aggregations" : { - "genders" : { + "genres" : { "doc_count_error_upper_bound": 0, <1> "sum_other_doc_count": 0, <2> "buckets" : [ <3> { - "key" : "male", + "key" : "jazz", "doc_count" : 10 }, { - "key" : "female", + "key" : "rock", + "doc_count" : 10 + }, + { + "key" : "electronic", "doc_count" : 10 }, ] @@ -247,9 +251,9 @@ Ordering the buckets by their `doc_count` in an ascending manner: -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { - "field" : "gender", + "field" : "genre", "order" : { "_count" : "asc" } } } @@ -263,9 +267,9 @@ Ordering the buckets alphabetically by their terms in an ascending manner: -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { - "field" : "gender", + "field" : "genre", "order" : { "_term" : "asc" } } } @@ -280,13 +284,13 @@ Ordering the buckets by single value metrics sub-aggregation (identified by the -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { - "field" : "gender", - "order" : { "avg_height" : "desc" } + "field" : "genre", + "order" : { "avg_play_count" : "desc" } }, "aggs" : { - "avg_height" : { "avg" : { "field" : "height" } } + "avg_play_count" : { "avg" : { "field" : "play_count" } } } } } @@ -299,13 +303,13 @@ Ordering the buckets by multi value metrics sub-aggregation (identified by the a -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { - "field" : "gender", - "order" : { "height_stats.avg" : "desc" } + "field" : "genre", + "order" : { "playback_stats.avg" : "desc" } }, "aggs" : { - "height_stats" : { "stats" : { "field" : "height" } } + "playback_stats" : { "stats" : { "field" : "play_count" } } } } } @@ -343,14 +347,14 @@ PATH := []*[height_stats.avg" : "desc" } + "field" : "artist.country", + "order" : { "rock>playback_stats.avg" : "desc" } }, "aggs" : { - "females" : { - "filter" : { "term" : { "gender" : "female" }}, + "rock" : { + "filter" : { "term" : { "genre" : "rock" }}, "aggs" : { - "height_stats" : { "stats" : { "field" : "height" }} + "playback_stats" : { "stats" : { "field" : "play_count" }} } } } @@ -359,7 +363,7 @@ PATH := []*[height_stats.avg" : "desc" }, { "_count" : "desc" } ] + "field" : "artist.country", + "order" : [ { "rock>playback_stats.avg" : "desc" }, { "_count" : "desc" } ] }, "aggs" : { - "females" : { - "filter" : { "term" : { "gender" : { "female" }}}, + "rock" : { + "filter" : { "term" : { "genre" : { "rock" }}}, "aggs" : { - "height_stats" : { "stats" : { "field" : "height" }} + "playback_stats" : { "stats" : { "field" : "play_count" }} } } } @@ -385,7 +389,7 @@ Multiple criteria can be used to order the buckets by providing an array of orde } -------------------------------------------------- -The above will sort the countries buckets based on the average height among the female population and then by +The above will sort the artist's countries buckets based on the average play count among the rock songs and then by their `doc_count` in descending order. NOTE: In the event that two buckets share the same values for all order criteria the bucket's term value is used as a @@ -439,10 +443,10 @@ Generating the terms using a script: -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { "script" : { - "inline": "doc['gender'].value" + "inline": "doc['genre'].value" "lang": "painless" } } @@ -457,12 +461,12 @@ This will interpret the `script` parameter as an `inline` script with the defaul -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { "script" : { "file": "my_script", "params": { - "field": "gender" + "field": "genre" } } } @@ -480,11 +484,11 @@ TIP: for indexed scripts replace the `file` parameter with an `id` parameter. -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { - "field" : "gender", + "field" : "gendre", "script" : { - "inline" : "'Gender: ' +_value" + "inline" : "'Genre: ' +_value" "lang" : "painless" } } diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc index 83855a8aae5..1b955d2a898 100644 --- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc @@ -22,7 +22,7 @@ The top_hits aggregation returns regular search hits, because of this many per h * <> * <> * <> -* <> +* <> * <> ==== Example diff --git a/docs/reference/indices/shadow-replicas.asciidoc b/docs/reference/indices/shadow-replicas.asciidoc index 60360c147b5..3a0b23852b0 100644 --- a/docs/reference/indices/shadow-replicas.asciidoc +++ b/docs/reference/indices/shadow-replicas.asciidoc @@ -10,12 +10,12 @@ index. In order to fully utilize the `index.data_path` and `index.shadow_replicas` settings, you need to allow Elasticsearch to use the same data directory for -multiple instances by setting `node.add_id_to_custom_path` to false in +multiple instances by setting `node.add_lock_id_to_custom_path` to false in elasticsearch.yml: [source,yaml] -------------------------------------------------- -node.add_id_to_custom_path: false +node.add_lock_id_to_custom_path: false -------------------------------------------------- You will also need to indicate to the security manager where the custom indices @@ -114,7 +114,7 @@ settings API: These are non-dynamic settings that need to be configured in `elasticsearch.yml` -`node.add_id_to_custom_path`:: +`node.add_lock_id_to_custom_path`:: Boolean setting indicating whether Elasticsearch should append the node's ordinal to the custom data path. For example, if this is enabled and a path of "/tmp/foo" is used, the first locally-running node will use "/tmp/foo/0", diff --git a/docs/reference/mapping/params/store.asciidoc b/docs/reference/mapping/params/store.asciidoc index d34d1016546..53cac7493ff 100644 --- a/docs/reference/mapping/params/store.asciidoc +++ b/docs/reference/mapping/params/store.asciidoc @@ -48,7 +48,7 @@ PUT my_index/my_type/1 GET my_index/_search { - "fields": [ "title", "date" ] <2> + "stored_fields": [ "title", "date" ] <2> } -------------------------------------------------- // CONSOLE diff --git a/docs/reference/migration/migrate_5_0/fs.asciidoc b/docs/reference/migration/migrate_5_0/fs.asciidoc index 859f3092823..42c8b4ddcea 100644 --- a/docs/reference/migration/migrate_5_0/fs.asciidoc +++ b/docs/reference/migration/migrate_5_0/fs.asciidoc @@ -23,3 +23,9 @@ behavior will be removed. If you are using a multi-cluster setup with both instances of Elasticsearch pointing to the same data path, you will need to add the cluster name to the data path so that different clusters do not overwrite data. + +==== Local files + +Prior to 5.0, nodes that were marked with both `node.data: false` and `node.master: false` (or the now removed `node.client: true`) +didn't write any files or folder to disk. 5.x added persistent node ids, requiring nodes to store that information. As such, all +node types will write a small state file to their data folders. \ No newline at end of file diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc index 72b29e8a9e7..09478ee3251 100644 --- a/docs/reference/migration/migrate_5_0/search.asciidoc +++ b/docs/reference/migration/migrate_5_0/search.asciidoc @@ -64,11 +64,15 @@ characteristics as the former `scan` search type. ==== `fields` parameter -The `fields` parameter used to try to retrieve field values from stored -fields, and fall back to extracting from the `_source` if a field is not -marked as stored. Now, the `fields` parameter will only return stored fields +The `fields` parameter has been replaced by `stored_fields`. +The `stored_fields` parameter will only return stored fields -- it will no longer extract values from the `_source`. +==== `fielddata_fields` parameter + +The `fielddata_fields` has been deprecated, use parameter `docvalue_fields` instead. + + ==== search-exists API removed The search exists api has been removed in favour of using the search api with diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc index ffe69aa3cfb..7bfa9dc875c 100644 --- a/docs/reference/migration/migrate_5_0/settings.asciidoc +++ b/docs/reference/migration/migrate_5_0/settings.asciidoc @@ -26,6 +26,8 @@ should be used instead. The `name` setting has been removed and is replaced by `node.name`. Usage of `-Dname=some_node_name` is not supported anymore. +The `node.add_id_to_custom_path` was renamed to `add_lock_id_to_custom_path`. + ==== Node attribute settings Node level attributes used for allocation filtering, forced awareness or other node identification / grouping diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc index 93fb136913b..ad36cdd6df4 100644 --- a/docs/reference/modules/scripting/painless.asciidoc +++ b/docs/reference/modules/scripting/painless.asciidoc @@ -143,7 +143,7 @@ First, let's look at the source data for a player by submitting the following re ---------------------------------------------------------------- GET hockey/_search { - "fields": [ + "stored_fields": [ "_id", "_source" ], diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index 958320ea110..a9adc157bd3 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -143,11 +143,11 @@ include::request/sort.asciidoc[] include::request/source-filtering.asciidoc[] -include::request/fields.asciidoc[] +include::request/stored-fields.asciidoc[] include::request/script-fields.asciidoc[] -include::request/fielddata-fields.asciidoc[] +include::request/docvalue-fields.asciidoc[] include::request/post-filter.asciidoc[] diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc new file mode 100644 index 00000000000..b4d2493d853 --- /dev/null +++ b/docs/reference/search/request/docvalue-fields.asciidoc @@ -0,0 +1,23 @@ +[[search-request-docvalue-fields]] +=== Doc value Fields + +Allows to return the <> representation of a field for each hit, for +example: + +[source,js] +-------------------------------------------------- +GET /_search +{ + "query" : { + "match_all": {} + }, + "docvalue_fields" : ["test1", "test2"] +} +-------------------------------------------------- +// CONSOLE + +Doc value fields can work on fields that are not stored. + +Note that if the fields parameter specifies fields without docvalues it will try to load the value from the fielddata cache +causing the terms for that field to be loaded to memory (cached), which will result in more memory consumption. + diff --git a/docs/reference/search/request/fielddata-fields.asciidoc b/docs/reference/search/request/fielddata-fields.asciidoc deleted file mode 100644 index f3a3508b144..00000000000 --- a/docs/reference/search/request/fielddata-fields.asciidoc +++ /dev/null @@ -1,23 +0,0 @@ -[[search-request-fielddata-fields]] -=== Field Data Fields - -Allows to return the <> representation of a field for each hit, for -example: - -[source,js] --------------------------------------------------- -GET /_search -{ - "query" : { - "match_all": {} - }, - "fielddata_fields" : ["test1", "test2"] -} --------------------------------------------------- -// CONSOLE - -Field data fields can work on fields that are not stored. - -It's important to understand that using the `fielddata_fields` parameter will -cause the terms for that field to be loaded to memory (cached), which will -result in more memory consumption. diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc index c235c37b338..efb7053c179 100644 --- a/docs/reference/search/request/highlighting.asciidoc +++ b/docs/reference/search/request/highlighting.asciidoc @@ -372,7 +372,7 @@ query and the rescore query in `highlight_query`. -------------------------------------------------- GET /_search { - "fields": [ "_id" ], + "stored_fields": [ "_id" ], "query" : { "match": { "content": { diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc index 345bc9abde2..3c8e0e9f00e 100644 --- a/docs/reference/search/request/inner-hits.asciidoc +++ b/docs/reference/search/request/inner-hits.asciidoc @@ -72,7 +72,7 @@ Inner hits also supports the following per document features: * <> * <> * <> -* <> +* <> * <> [[nested-inner-hits]] diff --git a/docs/reference/search/request/fields.asciidoc b/docs/reference/search/request/stored-fields.asciidoc similarity index 91% rename from docs/reference/search/request/fields.asciidoc rename to docs/reference/search/request/stored-fields.asciidoc index 3483d470ee2..3d5b8c01b47 100644 --- a/docs/reference/search/request/fields.asciidoc +++ b/docs/reference/search/request/stored-fields.asciidoc @@ -1,7 +1,7 @@ [[search-request-fields]] === Fields -WARNING: The `fields` parameter is about fields that are explicitly marked as +WARNING: The `stored_fields` parameter is about fields that are explicitly marked as stored in the mapping, which is off by default and generally not recommended. Use <> instead to select subsets of the original source document to be returned. @@ -13,7 +13,7 @@ by a search hit. -------------------------------------------------- GET /_search { - "fields" : ["user", "postDate"], + "stored_fields" : ["user", "postDate"], "query" : { "term" : { "user" : "kimchy" } } @@ -30,7 +30,7 @@ returned, for example: -------------------------------------------------- GET /_search { - "fields" : [], + "stored_fields" : [], "query" : { "term" : { "user" : "kimchy" } } diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc index 496f04ea4af..ba36992f6fb 100644 --- a/docs/reference/search/uri-request.asciidoc +++ b/docs/reference/search/uri-request.asciidoc @@ -83,7 +83,7 @@ hits was computed. part of the document by using `_source_include` & `_source_exclude` (see the <> documentation for more details) -|`fields` |The selective stored fields of the document to return for each hit, +|`stored_fields` |The selective stored fields of the document to return for each hit, comma delimited. Not specifying any value will cause no fields to return. |`sort` |Sorting to perform. Can either be in the form of `fieldName`, or diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java index edef75389c8..20be7e72888 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java @@ -36,8 +36,8 @@ import java.util.Map; */ public class InternalMatrixStats extends InternalMetricsAggregation implements MatrixStats { - public final static Type TYPE = new Type("matrix_stats"); - public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() { + public static final Type TYPE = new Type("matrix_stats"); + public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() { @Override public InternalMatrixStats readResult(StreamInput in) throws IOException { InternalMatrixStats result = new InternalMatrixStats(); diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java index 96b7b74ab4e..1ae29e65761 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java @@ -34,9 +34,9 @@ import java.util.Map; */ class MatrixStatsResults implements Writeable { /** object holding results - computes results in place */ - final protected RunningStats results; + protected final RunningStats results; /** pearson product correlation coefficients */ - final protected Map> correlation; + protected final Map> correlation; /** Base ctor */ public MatrixStatsResults() { diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java index 51e5ce1cf27..e3aa171fe3d 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java @@ -52,7 +52,7 @@ public abstract class MultiValuesSourceAggregationBuilder> + public abstract static class LeafOnly> extends MultiValuesSourceAggregationBuilder { protected LeafOnly(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) { diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java index dd2b69696f3..0de5e13c058 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java @@ -165,8 +165,8 @@ public abstract class MultiValuesSourceParser implement return factory; } - private final void parseMissingAndAdd(final String aggregationName, final String currentFieldName, - XContentParser parser, final Map missing) throws IOException { + private void parseMissingAndAdd(final String aggregationName, final String currentFieldName, + XContentParser parser, final Map missing) throws IOException { XContentParser.Token token = parser.currentToken(); if (token == null) { token = parser.nextToken(); diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java index b1296bb1146..2e4fa4313bd 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java @@ -34,8 +34,8 @@ public abstract class BaseMatrixStatsTestCase extends ESTestCase { protected final ArrayList fieldA = new ArrayList<>(numObs); protected final ArrayList fieldB = new ArrayList<>(numObs); protected final MultiPassStats actualStats = new MultiPassStats(); - protected final static String fieldAKey = "fieldA"; - protected final static String fieldBKey = "fieldB"; + protected static final String fieldAKey = "fieldA"; + protected static final String fieldBKey = "fieldB"; @Before public void setup() { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java index f35f3fec534..a0fcdb1e0d1 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java @@ -53,7 +53,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor { protected abstract String process(String value); - static abstract class Factory implements Processor.Factory { + abstract static class Factory implements Processor.Factory { protected final String processorType; protected Factory(String processorType) { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index 558bdde96a2..9b087fc208f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -93,7 +93,7 @@ public final class ConvertProcessor extends AbstractProcessor { }; @Override - public final String toString() { + public String toString() { return name().toLowerCase(Locale.ROOT); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java index 32edc665d58..39d17ec09bf 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java @@ -114,7 +114,7 @@ public final class GrokProcessor extends AbstractProcessor { return combinedPattern; } - public final static class Factory implements Processor.Factory { + public static final class Factory implements Processor.Factory { private final Map builtinPatterns; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java index f8bc2afffdd..da0011d20c8 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java @@ -70,7 +70,7 @@ public final class JoinProcessor extends AbstractProcessor { return TYPE; } - public final static class Factory implements Processor.Factory { + public static final class Factory implements Processor.Factory { @Override public JoinProcessor create(String processorTag, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java index e7a8f3f3e6a..a0ae8e13158 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java @@ -44,7 +44,7 @@ public final class LowercaseProcessor extends AbstractStringProcessor { return TYPE; } - public final static class Factory extends AbstractStringProcessor.Factory { + public static final class Factory extends AbstractStringProcessor.Factory { public Factory() { super(TYPE); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java index 5ab4016921d..d5994d411c9 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java @@ -111,7 +111,7 @@ public final class SortProcessor extends AbstractProcessor { return TYPE; } - public final static class Factory implements Processor.Factory { + public static final class Factory implements Processor.Factory { @Override public SortProcessor create(String processorTag, Map config) throws Exception { diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java index 105d42c8c86..a83dd93a17e 100644 --- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java +++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java @@ -155,7 +155,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri GroovyClassLoader groovyClassLoader = new GroovyClassLoader(loader, configuration); return groovyClassLoader.parseClass(codeSource); - } catch (Throwable e) { + } catch (Exception e) { if (logger.isTraceEnabled()) { logger.trace("Exception compiling Groovy script:", e); } @@ -293,7 +293,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri // NOTE: we truncate the stack because IndyInterface has security issue (needs getClassLoader) // we don't do a security check just as a tradeoff, it cannot really escalate to anything. return AccessController.doPrivileged((PrivilegedAction) script::run); - } catch (Throwable e) { + } catch (Exception e) { if (logger.isTraceEnabled()) { logger.trace("failed to run {}", e, compiledScript); } diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java index e9c4bf6e359..0d7dd4e12e2 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java @@ -72,28 +72,28 @@ public class GeoDistanceTests extends ESIntegTestCase { refresh(); - SearchResponse searchResponse1 = client().prepareSearch().addField("_source") + SearchResponse searchResponse1 = client().prepareSearch().addStoredField("_source") .addScriptField("distance", new Script("doc['location'].arcDistance(" + target_lat + "," + target_long + ")")).execute() .actionGet(); Double resultDistance1 = searchResponse1.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance1, closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d)); - SearchResponse searchResponse2 = client().prepareSearch().addField("_source") + SearchResponse searchResponse2 = client().prepareSearch().addStoredField("_source") .addScriptField("distance", new Script("doc['location'].distance(" + target_lat + "," + target_long + ")")).execute() .actionGet(); Double resultDistance2 = searchResponse2.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance2, closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d)); - SearchResponse searchResponse3 = client().prepareSearch().addField("_source") + SearchResponse searchResponse3 = client().prepareSearch().addStoredField("_source") .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + target_lat + "," + target_long + ")")) .execute().actionGet(); Double resultArcDistance3 = searchResponse3.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance3, closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d)); - SearchResponse searchResponse4 = client().prepareSearch().addField("_source") + SearchResponse searchResponse4 = client().prepareSearch().addStoredField("_source") .addScriptField("distance", new Script("doc['location'].distanceInKm(" + target_lat + "," + target_long + ")")).execute() .actionGet(); Double resultDistance4 = searchResponse4.getHits().getHits()[0].getFields().get("distance").getValue(); @@ -102,7 +102,7 @@ public class GeoDistanceTests extends ESIntegTestCase { SearchResponse searchResponse5 = client() .prepareSearch() - .addField("_source") + .addStoredField("_source") .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + (target_lat) + "," + (target_long + 360) + ")")) .execute().actionGet(); Double resultArcDistance5 = searchResponse5.getHits().getHits()[0].getFields().get("distance").getValue(); @@ -111,21 +111,21 @@ public class GeoDistanceTests extends ESIntegTestCase { SearchResponse searchResponse6 = client() .prepareSearch() - .addField("_source") + .addStoredField("_source") .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + (target_lat + 360) + "," + (target_long) + ")")) .execute().actionGet(); Double resultArcDistance6 = searchResponse6.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance6, closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d)); - SearchResponse searchResponse7 = client().prepareSearch().addField("_source") + SearchResponse searchResponse7 = client().prepareSearch().addStoredField("_source") .addScriptField("distance", new Script("doc['location'].arcDistanceInMiles(" + target_lat + "," + target_long + ")")) .execute().actionGet(); Double resultDistance7 = searchResponse7.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance7, closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES), 0.01d)); - SearchResponse searchResponse8 = client().prepareSearch().addField("_source") + SearchResponse searchResponse8 = client().prepareSearch().addStoredField("_source") .addScriptField("distance", new Script("doc['location'].distanceInMiles(" + target_lat + "," + target_long + ")")) .execute().actionGet(); Double resultDistance8 = searchResponse8.getHits().getHits()[0].getFields().get("distance").getValue(); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java index a6349db0d07..b4260cf5530 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java @@ -858,7 +858,7 @@ public class HistogramTests extends ESIntegTestCase { // constructing the newly expected bucket list int bucketsCount = (int) ((boundsMaxKey - boundsMinKey) / interval) + 1; - long[] extendedValueCounts = new long[bucketsCount]; + long[] extendedValueCounts = new long[valueCounts.length + addedBucketsLeft + addedBucketsRight]; System.arraycopy(valueCounts, 0, extendedValueCounts, addedBucketsLeft, valueCounts.length); SearchResponse response = null; diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java index 662d4d2f30c..640c00b291d 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.messy.tests; import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongSet; import com.carrotsearch.randomizedtesting.generators.RandomStrings; - import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -316,7 +315,8 @@ public class MinDocCountTests extends AbstractTermsTestCase { Thread.sleep(60000); logger.debug("1m passed. retrying."); testMinDocCountOnTerms(field, script, order, include, false); - } catch (Throwable secondFailure) { + } catch (Exception secondFailure) { + secondFailure.addSuppressed(ae); logger.error("exception on retry (will re-throw the original in a sec)", secondFailure); } throw ae; diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java index 28894c5c1cc..b7be9693210 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java @@ -102,33 +102,33 @@ public class SearchFieldsTests extends ESIntegTestCase { client().admin().indices().prepareRefresh().execute().actionGet(); - SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field1").execute().actionGet(); + SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field1").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); // field2 is not stored, check that it is not extracted from source. - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field2").execute().actionGet(); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field2").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(0)); assertThat(searchResponse.getHits().getAt(0).fields().get("field2"), nullValue()); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field3").execute().actionGet(); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field3").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*3").execute().actionGet(); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*3").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*3").addField("field1").addField("field2").execute().actionGet(); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*3").addStoredField("field1").addStoredField("field2").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2)); @@ -136,20 +136,20 @@ public class SearchFieldsTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field*").execute().actionGet(); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field*").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2)); assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("f*3").execute().actionGet(); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("f*3").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1)); assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*").execute().actionGet(); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).source(), nullValue()); @@ -157,7 +157,7 @@ public class SearchFieldsTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1")); assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3")); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*").addField("_source").execute().actionGet(); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*").addStoredField("_source").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); assertThat(searchResponse.getHits().hits().length, equalTo(1)); assertThat(searchResponse.getHits().getAt(0).source(), notNullValue()); @@ -437,15 +437,15 @@ public class SearchFieldsTests extends ESIntegTestCase { client().admin().indices().prepareRefresh().execute().actionGet(); SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()) - .addField("byte_field") - .addField("short_field") - .addField("integer_field") - .addField("long_field") - .addField("float_field") - .addField("double_field") - .addField("date_field") - .addField("boolean_field") - .addField("binary_field") + .addStoredField("byte_field") + .addStoredField("short_field") + .addStoredField("integer_field") + .addStoredField("long_field") + .addStoredField("float_field") + .addStoredField("double_field") + .addStoredField("date_field") + .addStoredField("boolean_field") + .addStoredField("binary_field") .execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); @@ -465,7 +465,7 @@ public class SearchFieldsTests extends ESIntegTestCase { String dateTime = Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC)); assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo((Object) dateTime)); assertThat(searchResponse.getHits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) Boolean.TRUE)); - assertThat(((BytesReference) searchResponse.getHits().getAt(0).fields().get("binary_field").value()).toBytesArray(), equalTo((BytesReference) new BytesArray("testing text".getBytes("UTF8")))); + assertThat(((BytesReference) searchResponse.getHits().getAt(0).fields().get("binary_field").value()), equalTo((BytesReference) new BytesArray("testing text".getBytes("UTF8")))); } @@ -478,7 +478,7 @@ public class SearchFieldsTests extends ESIntegTestCase { SearchResponse searchResponse = client().prepareSearch("my-index") .setTypes("my-type1") - .addField("field1").addField("_routing") + .addStoredField("field1").addStoredField("_routing") .get(); assertThat(searchResponse.getHits().totalHits(), equalTo(1L)); @@ -493,7 +493,7 @@ public class SearchFieldsTests extends ESIntegTestCase { .setRefreshPolicy(IMMEDIATE) .get(); - assertFailures(client().prepareSearch("my-index").setTypes("my-type1").addField("field1"), + assertFailures(client().prepareSearch("my-index").setTypes("my-type1").addStoredField("field1"), RestStatus.BAD_REQUEST, containsString("field [field1] isn't a leaf field")); } @@ -557,14 +557,14 @@ public class SearchFieldsTests extends ESIntegTestCase { String field = "field1.field2.field3.field4"; - SearchResponse searchResponse = client().prepareSearch("my-index").setTypes("my-type1").addField(field).get(); + SearchResponse searchResponse = client().prepareSearch("my-index").setTypes("my-type1").addStoredField(field).get(); assertThat(searchResponse.getHits().totalHits(), equalTo(1L)); assertThat(searchResponse.getHits().getAt(0).field(field).isMetadataField(), equalTo(false)); assertThat(searchResponse.getHits().getAt(0).field(field).getValues().size(), equalTo(2)); assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(0).toString(), equalTo("value1")); assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(1).toString(), equalTo("value2")); - searchResponse = client().prepareSearch("my-index").setTypes("my-type2").addField(field).get(); + searchResponse = client().prepareSearch("my-index").setTypes("my-type2").addStoredField(field).get(); assertThat(searchResponse.getHits().totalHits(), equalTo(1L)); assertThat(searchResponse.getHits().getAt(0).field(field).isMetadataField(), equalTo(false)); assertThat(searchResponse.getHits().getAt(0).field(field).getValues().size(), equalTo(2)); @@ -621,16 +621,16 @@ public class SearchFieldsTests extends ESIntegTestCase { client().admin().indices().prepareRefresh().execute().actionGet(); SearchRequestBuilder builder = client().prepareSearch().setQuery(matchAllQuery()) - .addFieldDataField("text_field") - .addFieldDataField("keyword_field") - .addFieldDataField("byte_field") - .addFieldDataField("short_field") - .addFieldDataField("integer_field") - .addFieldDataField("long_field") - .addFieldDataField("float_field") - .addFieldDataField("double_field") - .addFieldDataField("date_field") - .addFieldDataField("boolean_field"); + .addDocValueField("text_field") + .addDocValueField("keyword_field") + .addDocValueField("byte_field") + .addDocValueField("short_field") + .addDocValueField("integer_field") + .addDocValueField("long_field") + .addDocValueField("float_field") + .addDocValueField("double_field") + .addDocValueField("date_field") + .addDocValueField("boolean_field"); SearchResponse searchResponse = builder.execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); @@ -704,7 +704,7 @@ public class SearchFieldsTests extends ESIntegTestCase { .setParent("parent_1") .setSource(jsonBuilder().startObject().field("field1", "value").endObject())); - SearchResponse response = client().prepareSearch("test").addField("field1").get(); + SearchResponse response = client().prepareSearch("test").addStoredField("field1").get(); assertSearchResponse(response); assertHitCount(response, 1); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java index 4a7b4350d23..f2eee2bb408 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java @@ -99,15 +99,15 @@ public class GroovyScriptTests extends ESIntegTestCase { try { client().prepareSearch("test") - .setQuery(constantScoreQuery(scriptQuery(new Script("assert false", ScriptType.INLINE, "groovy", null)))).get(); + .setQuery(constantScoreQuery(scriptQuery(new Script("null.foo", ScriptType.INLINE, "groovy", null)))).get(); fail("should have thrown an exception"); } catch (SearchPhaseExecutionException e) { assertThat(e.toString() + "should not contained NotSerializableTransportException", e.toString().contains("NotSerializableTransportException"), equalTo(false)); assertThat(e.toString() + "should have contained ScriptException", e.toString().contains("ScriptException"), equalTo(true)); - assertThat(e.toString()+ "should have contained an assert error", - e.toString().contains("AssertionError[assert false"), equalTo(true)); + assertThat(e.toString()+ "should have contained a NullPointerException", + e.toString().contains("NullPointerException[Cannot get property 'foo' on null object]"), equalTo(true)); } } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java index c779757f61b..5ca5078661c 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java @@ -40,21 +40,21 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera */ public static class Item implements Streamable { private SearchTemplateResponse response; - private Throwable throwable; + private Exception exception; Item() { } - public Item(SearchTemplateResponse response, Throwable throwable) { + public Item(SearchTemplateResponse response, Exception exception) { this.response = response; - this.throwable = throwable; + this.exception = exception; } /** * Is it a failed search? */ public boolean isFailure() { - return throwable != null; + return exception != null; } /** @@ -62,7 +62,7 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera */ @Nullable public String getFailureMessage() { - return throwable == null ? null : throwable.getMessage(); + return exception == null ? null : exception.getMessage(); } /** @@ -85,7 +85,7 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera this.response = new SearchTemplateResponse(); response.readFrom(in); } else { - throwable = in.readThrowable(); + exception = in.readException(); } } @@ -96,12 +96,12 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera response.writeTo(out); } else { out.writeBoolean(false); - out.writeThrowable(throwable); + out.writeThrowable(exception); } } - public Throwable getFailure() { - return throwable; + public Exception getFailure() { + return exception; } } @@ -150,7 +150,7 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera for (Item item : items) { builder.startObject(); if (item.isFailure()) { - ElasticsearchException.renderThrowable(builder, params, item.getFailure()); + ElasticsearchException.renderException(builder, params, item.getFailure()); } else { item.getResponse().toXContent(builder, params); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java index 1ffb19b5fc4..642fe7648da 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java @@ -61,7 +61,7 @@ public class TransportMultiSearchTemplateAction extends HandledTransportAction implements Iterable { + static final class ArrayMap extends AbstractMap implements Iterable { private final Object array; private final int length; @@ -109,7 +109,7 @@ final class CustomReflectionObjectHandler extends ReflectionObjectHandler { } - final static class CollectionMap extends AbstractMap implements Iterable { + static final class CollectionMap extends AbstractMap implements Iterable { private final Collection col; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java index 9c28225c081..03d04e518dc 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java @@ -146,14 +146,14 @@ public class MultiSearchTemplateIT extends ESIntegTestCase { SearchTemplateResponse searchTemplateResponse1 = response1.getResponse(); assertThat(searchTemplateResponse1.hasResponse(), is(true)); assertHitCount(searchTemplateResponse1.getResponse(), (numDocs / 2) + (numDocs % 2)); - assertThat(searchTemplateResponse1.getSource().toUtf8(), + assertThat(searchTemplateResponse1.getSource().utf8ToString(), equalTo("{\"query\":{\"match\":{\"odd\":\"true\"}}}")); MultiSearchTemplateResponse.Item response2 = response.getResponses()[1]; assertThat(response2.isFailure(), is(false)); SearchTemplateResponse searchTemplateResponse2 = response2.getResponse(); assertThat(searchTemplateResponse2.hasResponse(), is(false)); - assertThat(searchTemplateResponse2.getSource().toUtf8(), + assertThat(searchTemplateResponse2.getSource().utf8ToString(), equalTo("{\"query\":{\"match_phrase_prefix\":{\"message\":\"quick brown f\"}}}")); MultiSearchTemplateResponse.Item response3 = response.getResponses()[2]; @@ -161,7 +161,7 @@ public class MultiSearchTemplateIT extends ESIntegTestCase { SearchTemplateResponse searchTemplateResponse3 = response3.getResponse(); assertThat(searchTemplateResponse3.hasResponse(), is(true)); assertHitCount(searchTemplateResponse3.getResponse(), (numDocs / 2)); - assertThat(searchTemplateResponse3.getSource().toUtf8(), + assertThat(searchTemplateResponse3.getSource().utf8ToString(), equalTo("{\"query\":{\"term\":{\"odd\":\"false\"}}}")); MultiSearchTemplateResponse.Item response4 = response.getResponses()[3]; @@ -173,7 +173,7 @@ public class MultiSearchTemplateIT extends ESIntegTestCase { assertThat(response5.isFailure(), is(false)); SearchTemplateResponse searchTemplateResponse5 = response5.getResponse(); assertThat(searchTemplateResponse5.hasResponse(), is(false)); - assertThat(searchTemplateResponse5.getSource().toUtf8(), + assertThat(searchTemplateResponse5.getSource().utf8ToString(), equalTo("{\"query\":{\"terms\":{\"group\":[1,2,3,]}}}")); } } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java index 054268ef681..bdba0346a85 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java @@ -28,7 +28,6 @@ import org.junit.Before; import java.io.IOException; import java.io.StringWriter; -import java.nio.charset.Charset; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -58,7 +57,7 @@ public class MustacheScriptEngineTests extends ESTestCase { BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(null, template, compileParams)), vars).run(); assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.3 } }}", - new String(o.toBytes(), Charset.forName("UTF-8"))); + o.utf8ToString()); } { String template = "GET _search {\"query\": " + "{\"boosting\": {" + "\"positive\": {\"match\": {\"body\": \"gift\"}}," @@ -69,7 +68,7 @@ public class MustacheScriptEngineTests extends ESTestCase { BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(null, template, compileParams)), vars).run(); assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," + "\"negative\": {\"term\": {\"body\": {\"value\": \"\\\"quick brown\\\"\"}}}, \"negative_boost\": 0.3 } }}", - new String(o.toBytes(), Charset.forName("UTF-8"))); + o.utf8ToString()); } } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java index 8b6d0e69573..91098eb1c88 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java @@ -72,7 +72,7 @@ public class MustacheTests extends ESTestCase { "Mustache templating broken", "GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.2 } }}", - ((BytesReference) result.run()).toUtf8() + ((BytesReference) result.run()).utf8ToString() ); } @@ -88,7 +88,7 @@ public class MustacheTests extends ESTestCase { assertThat(output, notNullValue()); assertThat(output, instanceOf(BytesReference.class)); BytesReference bytes = (BytesReference) output; - assertThat(bytes.toUtf8(), equalTo("foo bar")); + assertThat(bytes.utf8ToString(), equalTo("foo bar")); // Sets can come out in any order Set setData = new HashSet<>(); @@ -99,7 +99,7 @@ public class MustacheTests extends ESTestCase { assertThat(output, notNullValue()); assertThat(output, instanceOf(BytesReference.class)); bytes = (BytesReference) output; - assertThat(bytes.toUtf8(), both(containsString("foo")).and(containsString("bar"))); + assertThat(bytes.utf8ToString(), both(containsString("foo")).and(containsString("bar"))); } public void testArrayInArrayAccess() throws Exception { @@ -116,7 +116,7 @@ public class MustacheTests extends ESTestCase { assertThat(output, notNullValue()); assertThat(output, instanceOf(BytesReference.class)); BytesReference bytes = (BytesReference) output; - assertThat(bytes.toUtf8(), equalTo("foo bar")); + assertThat(bytes.utf8ToString(), equalTo("foo bar")); } public void testMapInArrayAccess() throws Exception { @@ -131,7 +131,7 @@ public class MustacheTests extends ESTestCase { assertThat(output, notNullValue()); assertThat(output, instanceOf(BytesReference.class)); BytesReference bytes = (BytesReference) output; - assertThat(bytes.toUtf8(), equalTo("foo bar")); + assertThat(bytes.utf8ToString(), equalTo("foo bar")); // HashSet iteration order isn't fixed Set setData = new HashSet<>(); @@ -142,7 +142,7 @@ public class MustacheTests extends ESTestCase { assertThat(output, notNullValue()); assertThat(output, instanceOf(BytesReference.class)); bytes = (BytesReference) output; - assertThat(bytes.toUtf8(), both(containsString("foo")).and(containsString("bar"))); + assertThat(bytes.utf8ToString(), both(containsString("foo")).and(containsString("bar"))); } public void testEscaping() { @@ -152,7 +152,7 @@ public class MustacheTests extends ESTestCase { CompiledScript compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache); ExecutableScript executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\"")); BytesReference rawResult = (BytesReference) executableScript.run(); - String result = rawResult.toUtf8(); + String result = rawResult.utf8ToString(); assertThat(result, equalTo("{ \"field1\": \"a \\\"value\\\"\"}")); // json string escaping disabled: @@ -160,7 +160,7 @@ public class MustacheTests extends ESTestCase { compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache); executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\"")); rawResult = (BytesReference) executableScript.run(); - result = rawResult.toUtf8(); + result = rawResult.utf8ToString(); assertThat(result, equalTo("{ \"field1\": \"a \"value\"\"}")); } @@ -182,7 +182,7 @@ public class MustacheTests extends ESTestCase { BytesReference bytes = (BytesReference) output; String expectedString = String.format(Locale.ROOT, "%s %s", randomArrayValues.length, randomList.size()); - assertThat(bytes.toUtf8(), equalTo(expectedString)); + assertThat(bytes.utf8ToString(), equalTo(expectedString)); } public void testPrimitiveToJSON() throws Exception { @@ -378,7 +378,7 @@ public class MustacheTests extends ESTestCase { Object result = engine.executable(new CompiledScript(INLINE, "inline", "mustache", compile(script)), vars).run(); assertThat(result, notNullValue()); assertThat(result, instanceOf(BytesReference.class)); - assertThat(((BytesReference) result).toUtf8(), matcher); + assertThat(((BytesReference) result).utf8ToString(), matcher); } private Object compile(String script) { diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/10_basic.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/10_basic.yaml similarity index 97% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/10_basic.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/10_basic.yaml index 2a32e75331e..ab42133a2b2 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/10_basic.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/10_basic.yaml @@ -36,7 +36,7 @@ setup: "Basic multi-search template": - do: - template.msearch: + msearch_template: body: - index: index_* - inline: '{"query": {"match": {"foo": "{{value}}"} } }' @@ -71,7 +71,7 @@ setup: "Multi-search template with errors": - do: - template.msearch: + msearch_template: body: # Search 0 is OK - index: index_* @@ -109,7 +109,7 @@ setup: - do: catch: /(.)*action_request_validation_exception(.)*template.is.missing(.)*/ - template.msearch: + msearch_template: body: # Search 0 is OK - index: index_* diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/20_stored_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/20_stored_template.yaml similarity index 98% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/20_stored_template.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/20_stored_template.yaml index 5d04449e541..ffbe9200375 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/20_stored_template.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/20_stored_template.yaml @@ -42,7 +42,7 @@ setup: - match: { acknowledged: true } - do: - template.msearch: + msearch_template: body: - index: index_* - id: stored_template_1 diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/30_file_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/30_file_template.yaml similarity index 98% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/30_file_template.yaml rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/30_file_template.yaml index 0c6ad8021e1..a5d11898baf 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/30_file_template.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/30_file_template.yaml @@ -44,7 +44,7 @@ setup: - match: { template_output.size: 20 } - do: - template.msearch: + msearch_template: body: - index: index_* - file: template_1 diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java index 94d7a1305d6..f0e1bde74d0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java @@ -60,7 +60,7 @@ public final class CompilerSettings { * in a script before an exception is thrown. This attempts to prevent infinite loops. Note if * the counter is set to 0, no loop counter will be written. */ - public final int getMaxLoopCounter() { + public int getMaxLoopCounter() { return maxLoopCounter; } @@ -68,7 +68,7 @@ public final class CompilerSettings { * Set the cumulative total number of statements that can be made in all loops. * @see #getMaxLoopCounter */ - public final void setMaxLoopCounter(int max) { + public void setMaxLoopCounter(int max) { this.maxLoopCounter = max; } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java index e57aad862aa..69a74301bc8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java @@ -646,7 +646,7 @@ public final class Definition { } } - private final void addStruct(final String name, final Class clazz) { + private void addStruct(final String name, final Class clazz) { if (!name.matches("^[_a-zA-Z][\\.,_a-zA-Z0-9]*$")) { throw new IllegalArgumentException("Invalid struct name [" + name + "]."); } @@ -661,7 +661,7 @@ public final class Definition { simpleTypesMap.put(name, getTypeInternal(name)); } - private final void addConstructorInternal(final String struct, final String name, final Type[] args) { + private void addConstructorInternal(final String struct, final String name, final Type[] args) { final Struct owner = structsMap.get(struct); if (owner == null) { @@ -734,7 +734,7 @@ public final class Definition { * * no spaces allowed. */ - private final void addSignature(String className, String signature) { + private void addSignature(String className, String signature) { String elements[] = signature.split("\u0020"); if (elements.length != 2) { throw new IllegalArgumentException("Malformed signature: " + signature); @@ -774,8 +774,8 @@ public final class Definition { } } - private final void addMethodInternal(String struct, String name, boolean augmentation, - Type rtn, Type[] args) { + private void addMethodInternal(String struct, String name, boolean augmentation, + Type rtn, Type[] args) { final Struct owner = structsMap.get(struct); if (owner == null) { @@ -858,7 +858,7 @@ public final class Definition { } } - private final void addFieldInternal(String struct, String name, Type type) { + private void addFieldInternal(String struct, String name, Type type) { final Struct owner = structsMap.get(struct); if (owner == null) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java index ca549522018..c546207b1ee 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java @@ -43,37 +43,37 @@ import java.util.regex.Pattern; */ public final class WriterConstants { - public final static int CLASS_VERSION = Opcodes.V1_8; - public final static int ASM_VERSION = Opcodes.ASM5; - public final static String BASE_CLASS_NAME = Executable.class.getName(); - public final static Type BASE_CLASS_TYPE = Type.getType(Executable.class); + public static final int CLASS_VERSION = Opcodes.V1_8; + public static final int ASM_VERSION = Opcodes.ASM5; + public static final String BASE_CLASS_NAME = Executable.class.getName(); + public static final Type BASE_CLASS_TYPE = Type.getType(Executable.class); - public final static String CLASS_NAME = BASE_CLASS_NAME + "$Script"; - public final static Type CLASS_TYPE = Type.getObjectType(CLASS_NAME.replace('.', '/')); + public static final String CLASS_NAME = BASE_CLASS_NAME + "$Script"; + public static final Type CLASS_TYPE = Type.getObjectType(CLASS_NAME.replace('.', '/')); - public final static Method CONSTRUCTOR = getAsmMethod(void.class, "", String.class, String.class, BitSet.class); - public final static Method CLINIT = getAsmMethod(void.class, ""); - public final static Method EXECUTE = + public static final Method CONSTRUCTOR = getAsmMethod(void.class, "", String.class, String.class, BitSet.class); + public static final Method CLINIT = getAsmMethod(void.class, ""); + public static final Method EXECUTE = getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class, Object.class); - public final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class); + public static final Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class); - public final static Type NEEDS_SCORE_TYPE = Type.getType(NeedsScore.class); - public final static Type SCORER_TYPE = Type.getType(Scorer.class); - public final static Method SCORER_SCORE = getAsmMethod(float.class, "score"); + public static final Type NEEDS_SCORE_TYPE = Type.getType(NeedsScore.class); + public static final Type SCORER_TYPE = Type.getType(Scorer.class); + public static final Method SCORER_SCORE = getAsmMethod(float.class, "score"); - public final static Type MAP_TYPE = Type.getType(Map.class); - public final static Method MAP_GET = getAsmMethod(Object.class, "get", Object.class); + public static final Type MAP_TYPE = Type.getType(Map.class); + public static final Method MAP_GET = getAsmMethod(Object.class, "get", Object.class); - public final static Type ITERATOR_TYPE = Type.getType(Iterator.class); - public final static Method ITERATOR_HASNEXT = getAsmMethod(boolean.class, "hasNext"); - public final static Method ITERATOR_NEXT = getAsmMethod(Object.class, "next"); + public static final Type ITERATOR_TYPE = Type.getType(Iterator.class); + public static final Method ITERATOR_HASNEXT = getAsmMethod(boolean.class, "hasNext"); + public static final Method ITERATOR_NEXT = getAsmMethod(Object.class, "next"); - public final static Type UTILITY_TYPE = Type.getType(Utility.class); - public final static Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class); - public final static Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class); + public static final Type UTILITY_TYPE = Type.getType(Utility.class); + public static final Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class); + public static final Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class); - public final static Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class); + public static final Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class); public static final Type AUGMENTATION_TYPE = Type.getType(Augmentation.class); @@ -83,46 +83,46 @@ public final class WriterConstants { * because it can do it statically. This is both faster and prevents the script from doing something super slow like building a regex * per time it is run. */ - public final static Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class, int.class); - public final static Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", CharSequence.class); - public final static Method MATCHER_MATCHES = getAsmMethod(boolean.class, "matches"); - public final static Method MATCHER_FIND = getAsmMethod(boolean.class, "find"); + public static final Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class, int.class); + public static final Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", CharSequence.class); + public static final Method MATCHER_MATCHES = getAsmMethod(boolean.class, "matches"); + public static final Method MATCHER_FIND = getAsmMethod(boolean.class, "find"); /** dynamic callsite bootstrap signature */ - final static MethodType DEF_BOOTSTRAP_TYPE = + static final MethodType DEF_BOOTSTRAP_TYPE = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, int.class, Object[].class); - final static Handle DEF_BOOTSTRAP_HANDLE = + static final Handle DEF_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class), "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString(), false); - public final static Type DEF_UTIL_TYPE = Type.getType(Def.class); - public final static Method DEF_TO_BOOLEAN = getAsmMethod(boolean.class, "DefToboolean" , Object.class); - public final static Method DEF_TO_BYTE_IMPLICIT = getAsmMethod(byte.class , "DefTobyteImplicit" , Object.class); - public final static Method DEF_TO_SHORT_IMPLICIT = getAsmMethod(short.class , "DefToshortImplicit" , Object.class); - public final static Method DEF_TO_CHAR_IMPLICIT = getAsmMethod(char.class , "DefTocharImplicit" , Object.class); - public final static Method DEF_TO_INT_IMPLICIT = getAsmMethod(int.class , "DefTointImplicit" , Object.class); - public final static Method DEF_TO_LONG_IMPLICIT = getAsmMethod(long.class , "DefTolongImplicit" , Object.class); - public final static Method DEF_TO_FLOAT_IMPLICIT = getAsmMethod(float.class , "DefTofloatImplicit" , Object.class); - public final static Method DEF_TO_DOUBLE_IMPLICIT = getAsmMethod(double.class , "DefTodoubleImplicit", Object.class); - public final static Method DEF_TO_BYTE_EXPLICIT = getAsmMethod(byte.class , "DefTobyteExplicit" , Object.class); - public final static Method DEF_TO_SHORT_EXPLICIT = getAsmMethod(short.class , "DefToshortExplicit" , Object.class); - public final static Method DEF_TO_CHAR_EXPLICIT = getAsmMethod(char.class , "DefTocharExplicit" , Object.class); - public final static Method DEF_TO_INT_EXPLICIT = getAsmMethod(int.class , "DefTointExplicit" , Object.class); - public final static Method DEF_TO_LONG_EXPLICIT = getAsmMethod(long.class , "DefTolongExplicit" , Object.class); - public final static Method DEF_TO_FLOAT_EXPLICIT = getAsmMethod(float.class , "DefTofloatExplicit" , Object.class); - public final static Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class); + public static final Type DEF_UTIL_TYPE = Type.getType(Def.class); + public static final Method DEF_TO_BOOLEAN = getAsmMethod(boolean.class, "DefToboolean" , Object.class); + public static final Method DEF_TO_BYTE_IMPLICIT = getAsmMethod(byte.class , "DefTobyteImplicit" , Object.class); + public static final Method DEF_TO_SHORT_IMPLICIT = getAsmMethod(short.class , "DefToshortImplicit" , Object.class); + public static final Method DEF_TO_CHAR_IMPLICIT = getAsmMethod(char.class , "DefTocharImplicit" , Object.class); + public static final Method DEF_TO_INT_IMPLICIT = getAsmMethod(int.class , "DefTointImplicit" , Object.class); + public static final Method DEF_TO_LONG_IMPLICIT = getAsmMethod(long.class , "DefTolongImplicit" , Object.class); + public static final Method DEF_TO_FLOAT_IMPLICIT = getAsmMethod(float.class , "DefTofloatImplicit" , Object.class); + public static final Method DEF_TO_DOUBLE_IMPLICIT = getAsmMethod(double.class , "DefTodoubleImplicit", Object.class); + public static final Method DEF_TO_BYTE_EXPLICIT = getAsmMethod(byte.class , "DefTobyteExplicit" , Object.class); + public static final Method DEF_TO_SHORT_EXPLICIT = getAsmMethod(short.class , "DefToshortExplicit" , Object.class); + public static final Method DEF_TO_CHAR_EXPLICIT = getAsmMethod(char.class , "DefTocharExplicit" , Object.class); + public static final Method DEF_TO_INT_EXPLICIT = getAsmMethod(int.class , "DefTointExplicit" , Object.class); + public static final Method DEF_TO_LONG_EXPLICIT = getAsmMethod(long.class , "DefTolongExplicit" , Object.class); + public static final Method DEF_TO_FLOAT_EXPLICIT = getAsmMethod(float.class , "DefTofloatExplicit" , Object.class); + public static final Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class); /** invokedynamic bootstrap for lambda expression/method references */ - public final static MethodType LAMBDA_BOOTSTRAP_TYPE = + public static final MethodType LAMBDA_BOOTSTRAP_TYPE = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, Object[].class); - public final static Handle LAMBDA_BOOTSTRAP_HANDLE = + public static final Handle LAMBDA_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaMetafactory.class), "altMetafactory", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString(), false); /** dynamic invokedynamic bootstrap for indy string concats (Java 9+) */ - public final static Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE; + public static final Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE; static { Handle bs; try { @@ -139,24 +139,24 @@ public final class WriterConstants { INDY_STRING_CONCAT_BOOTSTRAP_HANDLE = bs; } - public final static int MAX_INDY_STRING_CONCAT_ARGS = 200; + public static final int MAX_INDY_STRING_CONCAT_ARGS = 200; - public final static Type STRING_TYPE = Type.getType(String.class); - public final static Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class); + public static final Type STRING_TYPE = Type.getType(String.class); + public static final Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class); - public final static Method STRINGBUILDER_CONSTRUCTOR = getAsmMethod(void.class, ""); - public final static Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class); - public final static Method STRINGBUILDER_APPEND_CHAR = getAsmMethod(StringBuilder.class, "append", char.class); - public final static Method STRINGBUILDER_APPEND_INT = getAsmMethod(StringBuilder.class, "append", int.class); - public final static Method STRINGBUILDER_APPEND_LONG = getAsmMethod(StringBuilder.class, "append", long.class); - public final static Method STRINGBUILDER_APPEND_FLOAT = getAsmMethod(StringBuilder.class, "append", float.class); - public final static Method STRINGBUILDER_APPEND_DOUBLE = getAsmMethod(StringBuilder.class, "append", double.class); - public final static Method STRINGBUILDER_APPEND_STRING = getAsmMethod(StringBuilder.class, "append", String.class); - public final static Method STRINGBUILDER_APPEND_OBJECT = getAsmMethod(StringBuilder.class, "append", Object.class); - public final static Method STRINGBUILDER_TOSTRING = getAsmMethod(String.class, "toString"); + public static final Method STRINGBUILDER_CONSTRUCTOR = getAsmMethod(void.class, ""); + public static final Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class); + public static final Method STRINGBUILDER_APPEND_CHAR = getAsmMethod(StringBuilder.class, "append", char.class); + public static final Method STRINGBUILDER_APPEND_INT = getAsmMethod(StringBuilder.class, "append", int.class); + public static final Method STRINGBUILDER_APPEND_LONG = getAsmMethod(StringBuilder.class, "append", long.class); + public static final Method STRINGBUILDER_APPEND_FLOAT = getAsmMethod(StringBuilder.class, "append", float.class); + public static final Method STRINGBUILDER_APPEND_DOUBLE = getAsmMethod(StringBuilder.class, "append", double.class); + public static final Method STRINGBUILDER_APPEND_STRING = getAsmMethod(StringBuilder.class, "append", String.class); + public static final Method STRINGBUILDER_APPEND_OBJECT = getAsmMethod(StringBuilder.class, "append", Object.class); + public static final Method STRINGBUILDER_TOSTRING = getAsmMethod(String.class, "toString"); - public final static Type OBJECTS_TYPE = Type.getType(Objects.class); - public final static Method EQUALS = getAsmMethod(boolean.class, "equals", Object.class, Object.class); + public static final Type OBJECTS_TYPE = Type.getType(Objects.class); + public static final Method EQUALS = getAsmMethod(boolean.class, "equals", Object.class, Object.class); private static Method getAsmMethod(final Class rtype, final String name, final Class... ptypes) { return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString()); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java index c05c0097c90..66b61ff0350 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java @@ -77,7 +77,7 @@ public class MultiPercolateResponse extends ActionResponse implements Iterablefalse is returned. */ public boolean isFailure() { - return throwable != null; + return exception != null; } - public Throwable getFailure() { - return throwable; + public Exception getFailure() { + return exception; } @Override @@ -161,7 +161,7 @@ public class MultiPercolateResponse extends ActionResponse implements Iterable INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING = + public static final XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE; + public static final Setting INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING = Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope); public static final String CONTENT_TYPE = "percolator"; private static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType(); @@ -216,7 +217,7 @@ public class PercolatorFieldMapper extends FieldMapper { try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) { queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap())); builder.flush(); - byte[] queryBuilderAsBytes = builder.bytes().toBytes(); + byte[] queryBuilderAsBytes = BytesReference.toBytes(builder.bytes()); context.doc().add(new Field(queryBuilderField.name(), queryBuilderAsBytes, queryBuilderField.fieldType())); } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java index 2b9f77ab82a..9968035ec85 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java @@ -107,7 +107,7 @@ public class TransportMultiPercolateAction extends HandledTransportAction> nodePlugins() { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java index 79904f422bb..c2c2a641a71 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java @@ -82,9 +82,9 @@ import static org.hamcrest.Matchers.is; public class PercolateQueryTests extends ESTestCase { - public final static String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms"; - public final static String UNKNOWN_QUERY_FIELD_NAME = "unknown_query"; - public final static FieldType EXTRACTED_TERMS_FIELD_TYPE = new FieldType(); + public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms"; + public static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query"; + public static final FieldType EXTRACTED_TERMS_FIELD_TYPE = new FieldType(); static { EXTRACTED_TERMS_FIELD_TYPE.setTokenized(false); @@ -433,7 +433,7 @@ public class PercolateQueryTests extends ESTestCase { } } - private final static class CustomQuery extends Query { + private static final class CustomQuery extends Query { private final Term term; @@ -462,7 +462,7 @@ public class PercolateQueryTests extends ESTestCase { } } - private final static class ControlQuery extends Query { + private static final class ControlQuery extends Query { private final MemoryIndex memoryIndex; private final PercolateQuery.QueryStore queryStore; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java index 75d4f408774..f4a436e7c40 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java @@ -50,8 +50,8 @@ import static org.hamcrest.Matchers.notNullValue; public class PercolatorAggregationsIT extends ESIntegTestCase { - private final static String INDEX_NAME = "queries"; - private final static String TYPE_NAME = "query"; + private static final String INDEX_NAME = "queries"; + private static final String TYPE_NAME = "query"; @Override protected Collection> nodePlugins() { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java index 701fd1b660f..7a51d8a7ab2 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java @@ -48,7 +48,7 @@ import static org.hamcrest.Matchers.notNullValue; // Can'r run as IT as the test cluster is immutable and this test adds nodes during the test public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase { - private final static String INDEX_NAME = "percolator_index"; + private static final String INDEX_NAME = "percolator_index"; @Override protected Collection> nodePlugins() { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java index ff9bf6634cd..e4e379c8f60 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java @@ -91,8 +91,8 @@ import static org.hamcrest.Matchers.nullValue; public class PercolatorIT extends ESIntegTestCase { - private final static String INDEX_NAME = "queries"; - private final static String TYPE_NAME = "query"; + private static final String INDEX_NAME = "queries"; + private static final String TYPE_NAME = "query"; @Override protected Collection> nodePlugins() { diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index 3403a8077b9..c8a791a7b71 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -138,8 +138,8 @@ public abstract class AbstractAsyncBulkByScrollAction client.search(firstSearchRequest, listener), (SearchResponse response) -> { @@ -185,8 +185,8 @@ public abstract class AbstractAsyncBulkByScrollAction indexingFailures, List searchFailures, boolean timedOut) { + void finishHim(Exception failure, List indexingFailures, List searchFailures, boolean timedOut) { String scrollId = scroll.get(); if (Strings.hasLength(scrollId)) { /* @@ -403,7 +402,7 @@ public abstract class AbstractAsyncBulkByScrollAction value = new AtomicReference<>(randomSimpleString(random())); indexRandom(true, client().prepareIndex("test", "test", "test").setSource("test", value.get())); - AtomicReference failure = new AtomicReference<>(); + AtomicReference failure = new AtomicReference<>(); AtomicBoolean keepUpdating = new AtomicBoolean(true); Thread updater = new Thread(() -> { while (keepUpdating.get()) { @@ -52,8 +52,8 @@ public class UpdateByQueryWhileModifyingTests extends ReindexTestCase { BulkIndexByScrollResponse response = updateByQuery().source("test").refresh(true).abortOnVersionConflict(false).get(); assertThat(response, matcher().updated(either(equalTo(0L)).or(equalTo(1L))) .versionConflicts(either(equalTo(0L)).or(equalTo(1L)))); - } catch (Throwable t) { - failure.set(t); + } catch (Exception e) { + failure.set(e); } } }); diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml index 8648c9034ee..7f17befe909 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml @@ -59,11 +59,11 @@ --- "source fields may not be modified": - do: - catch: /fields is not supported in this context/ + catch: /stored_fields is not supported in this context/ delete_by_query: index: test body: - fields: [_id] + stored_fields: [_id] --- "requests_per_second cannot be an empty string": diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml index a7a5198e430..b3e35d2f49e 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml @@ -216,11 +216,11 @@ --- "source fields may not be modified": - do: - catch: /fields is not supported in this context/ + catch: /stored_fields is not supported in this context/ reindex: body: source: index: test - fields: [_id] + stored_fields: [_id] dest: index: dest diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml index ea487eb54e0..08ab3cbcc10 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml @@ -67,11 +67,11 @@ --- "source fields may not be modified": - do: - catch: /fields is not supported in this context/ + catch: /stored_fields is not supported in this context/ update_by_query: index: test body: - fields: [_id] + stored_fields: [_id] --- "requests_per_second cannot be an empty string": diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java index e04724ee370..eac3ceebc16 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java @@ -40,9 +40,6 @@ import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; - -/** - */ public class IcuTokenizerFactory extends AbstractTokenizerFactory { private final ICUTokenizerConfig config; @@ -101,8 +98,8 @@ public class IcuTokenizerFactory extends AbstractTokenizerFactory { }; return config; } - } catch (Throwable t) { - throw new ElasticsearchException("failed to load ICU rule files", t); + } catch (Exception e) { + throw new ElasticsearchException("failed to load ICU rule files", e); } } diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java index 728a9354d97..e8b49a50edb 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java @@ -37,9 +37,9 @@ package org.elasticsearch.index.analysis.phonetic; */ public class HaasePhonetik extends KoelnerPhonetik { - private final static String[] HAASE_VARIATIONS_PATTERNS = {"OWN", "RB", "WSK", "A$", "O$", "SCH", + private static final String[] HAASE_VARIATIONS_PATTERNS = {"OWN", "RB", "WSK", "A$", "O$", "SCH", "GLI", "EAU$", "^CH", "AUX", "EUX", "ILLE"}; - private final static String[] HAASE_VARIATIONS_REPLACEMENTS = {"AUN", "RW", "RSK", "AR", "OW", "CH", + private static final String[] HAASE_VARIATIONS_REPLACEMENTS = {"AUN", "RW", "RSK", "AR", "OW", "CH", "LI", "O", "SCH", "O", "O", "I"}; @Override diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java index 7d6f81ad0b6..2375db2502b 100644 --- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java +++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java @@ -39,7 +39,7 @@ import java.util.ServiceLoader; /** * */ -public class AzureComputeServiceImpl extends AbstractLifecycleComponent +public class AzureComputeServiceImpl extends AbstractLifecycleComponent implements AzureComputeService { private final ComputeManagementClient client; diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java index 33f40a9159a..02de9db6d69 100644 --- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java +++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java @@ -26,7 +26,7 @@ import org.elasticsearch.common.settings.Settings; /** * */ -public abstract class AzureComputeServiceAbstractMock extends AbstractLifecycleComponent +public abstract class AzureComputeServiceAbstractMock extends AbstractLifecycleComponent implements AzureComputeService { protected AzureComputeServiceAbstractMock(Settings settings) { diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java index 5ce110487c4..2fc82a01f09 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java @@ -49,7 +49,7 @@ import java.util.Random; /** * */ -public class AwsEc2ServiceImpl extends AbstractLifecycleComponent implements AwsEc2Service { +public class AwsEc2ServiceImpl extends AbstractLifecycleComponent implements AwsEc2Service { public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/"; diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java index f14a80f01cc..19a5c8d3b96 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java @@ -27,7 +27,7 @@ import org.elasticsearch.common.settings.Settings; import java.util.List; -public class AwsEc2ServiceMock extends AbstractLifecycleComponent implements AwsEc2Service { +public class AwsEc2ServiceMock extends AbstractLifecycleComponent implements AwsEc2Service { private int nodes; private List> tagsList; diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java index a6faa390e5d..00a44cd56af 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -31,7 +31,7 @@ import java.util.Collections; import java.util.List; import java.util.function.Function; -public interface GceComputeService extends LifecycleComponent { +public interface GceComputeService extends LifecycleComponent { /** * GCE API Version: Elasticsearch/GceCloud/1.0 diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 63a19919a7b..8d1090dd82c 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -56,7 +56,7 @@ import java.util.Collections; import java.util.List; import java.util.function.Function; -public class GceComputeServiceImpl extends AbstractLifecycleComponent +public class GceComputeServiceImpl extends AbstractLifecycleComponent implements GceComputeService { // all settings just used for testing - not registered by default diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index c9dd2263245..be3d737b919 100644 --- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -249,8 +249,8 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas } } - } catch (Throwable e) { - logger.warn("Exception caught during discovery: {}", e, e.getMessage()); + } catch (Exception e) { + logger.warn("exception caught during discovery", e); } logger.debug("{} node(s) added", cachedDiscoNodes.size()); diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java index aa6e91fabdf..a49124749fe 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java @@ -44,7 +44,7 @@ import static org.hamcrest.Matchers.lessThan; public class RetryHttpInitializerWrapperTests extends ESTestCase { - static private class FailThenSuccessBackoffTransport extends MockHttpTransport { + private static class FailThenSuccessBackoffTransport extends MockHttpTransport { public int lowLevelExecCalls; int errorStatusCode; diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java index fb4dc37d641..b06232e1c41 100644 --- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java +++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java @@ -122,7 +122,7 @@ public final class AttachmentProcessor extends AbstractProcessor { String length = Strings.hasLength(contentLength) ? contentLength : String.valueOf(parsedContent.length()); additionalFields.put(Property.CONTENT_LENGTH.toLowerCase(), length); } - } catch (Throwable e) { + } catch (Exception e) { throw new ElasticsearchParseException("Error parsing document in field [{}]", e, field); } diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java index 0c63f65c247..4b9a40dd8a9 100644 --- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java +++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java @@ -58,7 +58,7 @@ public class TikaDocTests extends ESTestCase { assertNotNull(parsedContent); assertFalse(parsedContent.isEmpty()); logger.debug("extracted content: {}", parsedContent); - } catch (Throwable e) { + } catch (Exception e) { throw new RuntimeException("parsing of filename: " + fileName.getFileName() + " failed", e); } } diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 92c4785dbaf..64f0994b7d9 100644 --- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -272,7 +272,7 @@ public final class GeoIpProcessor extends AbstractProcessor { // Geoip2's AddressNotFoundException is checked and due to the fact that we need run their code // inside a PrivilegedAction code block, we are forced to catch any checked exception and rethrow // it with an unchecked exception. - private final static class AddressNotFoundRuntimeException extends RuntimeException { + private static final class AddressNotFoundRuntimeException extends RuntimeException { public AddressNotFoundRuntimeException(Throwable cause) { super(cause); diff --git a/plugins/ingest-useragent/build.gradle b/plugins/ingest-useragent/build.gradle new file mode 100644 index 00000000000..86fbf243497 --- /dev/null +++ b/plugins/ingest-useragent/build.gradle @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'Ingest processor that extracts information from a user agent' + classname 'org.elasticsearch.ingest.useragent.IngestUserAgentPlugin' +} + +integTest { + cluster { + extraConfigFile 'ingest-useragent/test-regexes.yaml', 'test/test-regexes.yaml' + } +} \ No newline at end of file diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java new file mode 100644 index 00000000000..e86a8443d7e --- /dev/null +++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.node.NodeModule; +import org.elasticsearch.plugins.Plugin; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +public class IngestUserAgentPlugin extends Plugin { + + private final Setting CACHE_SIZE_SETTING = Setting.longSetting("ingest.useragent.cache_size", 1000, 0, + Setting.Property.NodeScope); + + static final String DEFAULT_PARSER_NAME = "_default_"; + + public void onModule(NodeModule nodeModule) throws IOException { + Path userAgentConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-useragent"); + + if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) { + throw new IllegalStateException( + "the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist"); + } + + long cacheSize = CACHE_SIZE_SETTING.get(nodeModule.getNode().settings()); + + UserAgentCache cache = new UserAgentCache(cacheSize); + + Map userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, cache); + + nodeModule.registerProcessor(UserAgentProcessor.TYPE, (registry) -> new UserAgentProcessor.Factory(userAgentParsers)); + } + + static Map createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException { + Map userAgentParsers = new HashMap<>(); + + UserAgentParser defaultParser = new UserAgentParser(DEFAULT_PARSER_NAME, + IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yaml"), cache); + userAgentParsers.put(DEFAULT_PARSER_NAME, defaultParser); + + if (Files.exists(userAgentConfigDirectory) && Files.isDirectory(userAgentConfigDirectory)) { + PathMatcher pathMatcher = userAgentConfigDirectory.getFileSystem().getPathMatcher("glob:**.yaml"); + + try (Stream regexFiles = Files.find(userAgentConfigDirectory, 1, + (path, attr) -> attr.isRegularFile() && pathMatcher.matches(path))) { + Iterable iterable = regexFiles::iterator; + for (Path path : iterable) { + String parserName = path.getFileName().toString(); + try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ)) { + userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, cache)); + } + } + } + } + + return Collections.unmodifiableMap(userAgentParsers); + } + +} diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java new file mode 100644 index 00000000000..d1002f2df06 --- /dev/null +++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.common.cache.Cache; +import org.elasticsearch.common.cache.CacheBuilder; +import org.elasticsearch.ingest.useragent.UserAgentParser.Details; + +import java.util.Objects; + +class UserAgentCache { + private final Cache cache; + + UserAgentCache(long cacheSize) { + cache = CacheBuilder.builder().setMaximumWeight(cacheSize).build(); + } + + public Details get(String parserName, String userAgent) { + return cache.get(new CompositeCacheKey(parserName, userAgent)); + } + + public void put(String parserName, String userAgent, Details details) { + cache.put(new CompositeCacheKey(parserName, userAgent), details); + } + + private static final class CompositeCacheKey { + private final String parserName; + private final String userAgent; + + CompositeCacheKey(String parserName, String userAgent) { + this.parserName = parserName; + this.userAgent = userAgent; + } + + @Override + public boolean equals(Object obj) { + if(obj != null && obj instanceof CompositeCacheKey) { + CompositeCacheKey s = (CompositeCacheKey)obj; + return parserName.equals(s.parserName) && userAgent.equals(s.userAgent); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(parserName, userAgent); + } + } +} diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java new file mode 100644 index 00000000000..af764d5baf2 --- /dev/null +++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java @@ -0,0 +1,280 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +final class UserAgentParser { + + private final UserAgentCache cache; + private final List uaPatterns = new ArrayList<>(); + private final List osPatterns = new ArrayList<>(); + private final List devicePatterns = new ArrayList<>(); + private final String name; + + public UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) { + this.name = name; + this.cache = cache; + + try { + init(regexStream); + } catch (IOException e) { + throw new ElasticsearchParseException("error parsing regular expression file", e); + } + } + + private void init(InputStream regexStream) throws IOException { + XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(regexStream); + + XContentParser.Token token = yamlParser.nextToken(); + + if (token == XContentParser.Token.START_OBJECT) { + token = yamlParser.nextToken(); + + for (; token != null; token = yamlParser.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("user_agent_parsers")) { + List> parserConfigurations = readParserConfigurations(yamlParser); + + for (Map map : parserConfigurations) { + uaPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), + map.get("family_replacement"), map.get("v1_replacement"), map.get("v2_replacement"), + map.get("v3_replacement"), map.get("v4_replacement"))); + } + } + else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("os_parsers")) { + List> parserConfigurations = readParserConfigurations(yamlParser); + + for (Map map : parserConfigurations) { + osPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), + map.get("os_replacement"), map.get("os_v1_replacement"), map.get("os_v2_replacement"), + map.get("os_v3_replacement"), map.get("os_v4_replacement"))); + } + } + else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("device_parsers")) { + List> parserConfigurations = readParserConfigurations(yamlParser); + + for (Map map : parserConfigurations) { + devicePatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), + map.get("device_replacement"), null, null, null, null)); + } + } + } + } + + if (uaPatterns.isEmpty() && osPatterns.isEmpty() && devicePatterns.isEmpty()) { + throw new ElasticsearchParseException("not a valid regular expression file"); + } + } + + private Pattern compilePattern(String regex, String regex_flag) { + // Only flag present in the current default regexes.yaml + if (regex_flag != null && regex_flag.equals("i")) { + return Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + } else { + return Pattern.compile(regex); + } + } + + private List> readParserConfigurations(XContentParser yamlParser) throws IOException { + List > patternList = new ArrayList<>(); + + XContentParser.Token token = yamlParser.nextToken(); + if (token != XContentParser.Token.START_ARRAY) { + throw new ElasticsearchParseException("malformed regular expression file, should continue with 'array' after 'object'"); + } + + token = yamlParser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchParseException("malformed regular expression file, expecting 'object'"); + } + + while (token == XContentParser.Token.START_OBJECT) { + token = yamlParser.nextToken(); + + if (token != XContentParser.Token.FIELD_NAME) { + throw new ElasticsearchParseException("malformed regular expression file, should continue with 'field_name' after 'array'"); + } + + Map regexMap = new HashMap<>(); + for (; token == XContentParser.Token.FIELD_NAME; token = yamlParser.nextToken()) { + String fieldName = yamlParser.currentName(); + + token = yamlParser.nextToken(); + String fieldValue = yamlParser.text(); + regexMap.put(fieldName, fieldValue); + } + + patternList.add(regexMap); + + token = yamlParser.nextToken(); + } + + return patternList; + } + + List getUaPatterns() { + return uaPatterns; + } + + List getOsPatterns() { + return osPatterns; + } + + List getDevicePatterns() { + return devicePatterns; + } + + String getName() { + return name; + } + + public Details parse(String agentString) { + Details details = cache.get(name, agentString);; + + if (details == null) { + VersionedName userAgent = findMatch(uaPatterns, agentString); + VersionedName operatingSystem = findMatch(osPatterns, agentString); + VersionedName device = findMatch(devicePatterns, agentString); + + details = new Details(userAgent, operatingSystem, device); + + cache.put(name, agentString, details); + } + + return details; + } + + private VersionedName findMatch(List possiblePatterns, String agentString) { + VersionedName name; + for (UserAgentSubpattern pattern : possiblePatterns) { + name = pattern.match(agentString); + + if (name != null) { + return name; + } + } + + return null; + } + + static final class Details { + public final VersionedName userAgent; + public final VersionedName operatingSystem; + public final VersionedName device; + + public Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device) { + this.userAgent = userAgent; + this.operatingSystem = operatingSystem; + this.device = device; + } + } + + static final class VersionedName { + public final String name; + public final String major; + public final String minor; + public final String patch; + public final String build; + + public VersionedName(String name, String major, String minor, String patch, String build) { + this.name = name; + this.major = major; + this.minor = minor; + this.patch = patch; + this.build = build; + } + } + + /** + * One of: user agent, operating system, device + */ + static final class UserAgentSubpattern { + private final Pattern pattern; + private final String nameReplacement, v1Replacement, v2Replacement, v3Replacement, v4Replacement; + + public UserAgentSubpattern(Pattern pattern, String nameReplacement, + String v1Replacement, String v2Replacement, String v3Replacement, String v4Replacement) { + this.pattern = pattern; + this.nameReplacement = nameReplacement; + this.v1Replacement = v1Replacement; + this.v2Replacement = v2Replacement; + this.v3Replacement = v3Replacement; + this.v4Replacement = v4Replacement; + } + + public VersionedName match(String agentString) { + String name = null, major = null, minor = null, patch = null, build = null; + Matcher matcher = pattern.matcher(agentString); + + if (!matcher.find()) { + return null; + } + + int groupCount = matcher.groupCount(); + + if (nameReplacement != null) { + if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) { + name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1))); + } else { + name = nameReplacement; + } + } else if (groupCount >= 1) { + name = matcher.group(1); + } + + if (v1Replacement != null) { + major = v1Replacement; + } else if (groupCount >= 2) { + major = matcher.group(2); + } + + if (v2Replacement != null) { + minor = v2Replacement; + } else if (groupCount >= 3) { + minor = matcher.group(3); + } + + if (v3Replacement != null) { + patch = v3Replacement; + } else if (groupCount >= 4) { + patch = matcher.group(4); + } + + if (v4Replacement != null) { + build = v4Replacement; + } else if (groupCount >= 5) { + build = matcher.group(5); + } + + return name == null ? null : new VersionedName(name, major, minor, patch, build); + } + } +} diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java new file mode 100644 index 00000000000..7cb829e36e2 --- /dev/null +++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -0,0 +1,241 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.useragent.UserAgentParser.Details; +import org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; +import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList; +import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty; + +public class UserAgentProcessor extends AbstractProcessor { + + public static final String TYPE = "useragent"; + + private final String field; + private final String targetField; + private final Set properties; + + private final UserAgentParser parser; + + public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set properties) { + super(tag); + this.field = field; + this.targetField = targetField; + this.parser = parser; + this.properties = properties; + } + + @Override + public void execute(IngestDocument ingestDocument) throws Exception { + String userAgent = ingestDocument.getFieldValue(field, String.class); + + Details uaClient = parser.parse(userAgent); + + Map uaDetails = new HashMap<>(); + for (Property property : this.properties) { + switch (property) { + case NAME: + if (uaClient.userAgent != null && uaClient.userAgent.name != null) { + uaDetails.put("name", uaClient.userAgent.name); + } + else { + uaDetails.put("name", "Other"); + } + break; + case MAJOR: + if (uaClient.userAgent != null && uaClient.userAgent.major != null) { + uaDetails.put("major", uaClient.userAgent.major); + } + break; + case MINOR: + if (uaClient.userAgent != null && uaClient.userAgent.minor != null) { + uaDetails.put("minor", uaClient.userAgent.minor); + } + break; + case PATCH: + if (uaClient.userAgent != null && uaClient.userAgent.patch != null) { + uaDetails.put("patch", uaClient.userAgent.patch); + } + break; + case BUILD: + if (uaClient.userAgent != null && uaClient.userAgent.build != null) { + uaDetails.put("build", uaClient.userAgent.build); + } + break; + case OS: + if (uaClient.operatingSystem != null) { + uaDetails.put("os", buildFullOSName(uaClient.operatingSystem)); + } + else { + uaDetails.put("os", "Other"); + } + + break; + case OS_NAME: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) { + uaDetails.put("os_name", uaClient.operatingSystem.name); + } + else { + uaDetails.put("os_name", "Other"); + } + break; + case OS_MAJOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) { + uaDetails.put("os_major", uaClient.operatingSystem.major); + } + break; + case OS_MINOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) { + uaDetails.put("os_minor", uaClient.operatingSystem.minor); + } + break; + case DEVICE: + if (uaClient.device != null && uaClient.device.name != null) { + uaDetails.put("device", uaClient.device.name); + } + else { + uaDetails.put("device", "Other"); + } + break; + } + } + + ingestDocument.setFieldValue(targetField, uaDetails); + } + + /** To maintain compatibility with logstash-filter-useragent */ + private String buildFullOSName(VersionedName operatingSystem) { + if (operatingSystem == null || operatingSystem.name == null) { + return null; + } + + StringBuilder sb = new StringBuilder(operatingSystem.name); + + if (operatingSystem.major != null) { + sb.append(" "); + sb.append(operatingSystem.major); + + if (operatingSystem.minor != null) { + sb.append("."); + sb.append(operatingSystem.minor); + + if (operatingSystem.patch != null) { + sb.append("."); + sb.append(operatingSystem.patch); + + if (operatingSystem.build != null) { + sb.append("."); + sb.append(operatingSystem.build); + } + } + } + } + + return sb.toString(); + } + + @Override + public String getType() { + return TYPE; + } + + String getField() { + return field; + } + + String getTargetField() { + return targetField; + } + + Set getProperties() { + return properties; + } + + UserAgentParser getUaParser() { + return parser; + } + + public static final class Factory implements Processor.Factory { + + private final Map userAgentParsers; + + public Factory(Map userAgentParsers) { + this.userAgentParsers = userAgentParsers; + } + + @Override + public UserAgentProcessor create(String processorTag, Map config) throws Exception { + String field = readStringProperty(TYPE, processorTag, config, "field"); + String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "useragent"); + String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); + List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); + + UserAgentParser parser = userAgentParsers.get(regexFilename); + if (parser == null) { + throw newConfigurationException(TYPE, processorTag, + "regex_file", "regex file [" + regexFilename + "] doesn't exist (has to exist at node startup)"); + } + + final Set properties; + if (propertyNames != null) { + properties = EnumSet.noneOf(Property.class); + for (String fieldName : propertyNames) { + try { + properties.add(Property.parseProperty(fieldName)); + } catch (IllegalArgumentException e) { + throw newConfigurationException(TYPE, processorTag, "properties", e.getMessage()); + } + } + } else { + properties = EnumSet.allOf(Property.class); + } + + return new UserAgentProcessor(processorTag, field, targetField, parser, properties); + } + } + + enum Property { + + NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD; + + public static Property parseProperty(String propertyName) { + try { + return valueOf(propertyName.toUpperCase(Locale.ROOT)); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + + Arrays.toString(EnumSet.allOf(Property.class).toArray())); + } + } + } +} diff --git a/plugins/ingest-useragent/src/main/resources/regexes.yaml b/plugins/ingest-useragent/src/main/resources/regexes.yaml new file mode 100644 index 00000000000..cc28842308c --- /dev/null +++ b/plugins/ingest-useragent/src/main/resources/regexes.yaml @@ -0,0 +1,4815 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +user_agent_parsers: + #### SPECIAL CASES TOP #### + + # @note: iOS / OSX Applications + - regex: '(CFNetwork)(?:/(\d+)\.(\d+)\.?(\d+)?)?' + family_replacement: 'CFNetwork' + + # Pingdom + - regex: '(Pingdom.com_bot_version_)(\d+)\.(\d+)' + family_replacement: 'PingdomBot' + + # Facebook + - regex: '(facebookexternalhit)/(\d+)\.(\d+)' + family_replacement: 'FacebookBot' + + # Google Plus + - regex: 'Google.*/\+/web/snippet' + family_replacement: 'GooglePlusBot' + + # Twitter + - regex: '(Twitterbot)/(\d+)\.(\d+)' + family_replacement: 'TwitterBot' + + # Bots Pattern '/name-0.0' + - regex: '/((?:Ant-)?Nutch|[A-z]+[Bb]ot|[A-z]+[Ss]pider|Axtaris|fetchurl|Isara|ShopSalad|Tailsweep)[ \-](\d+)(?:\.(\d+)(?:\.(\d+))?)?' + # Bots Pattern 'name/0.0' + - regex: '(008|Altresium|Argus|BaiduMobaider|BoardReader|DNSGroup|DataparkSearch|EDI|Goodzer|Grub|INGRID|Infohelfer|LinkedInBot|LOOQ|Nutch|PathDefender|Peew|PostPost|Steeler|Twitterbot|VSE|WebCrunch|WebZIP|Y!J-BR[A-Z]|YahooSeeker|envolk|sproose|wminer)/(\d+)(?:\.(\d+)(?:\.(\d+))?)?' + + # MSIECrawler + - regex: '(MSIE) (\d+)\.(\d+)([a-z]\d?)?;.* MSIECrawler' + family_replacement: 'MSIECrawler' + + # Downloader ... + - regex: '(Google-HTTP-Java-Client|Apache-HttpClient|http%20client|Python-urllib|HttpMonitor|TLSProber|WinHTTP|JNLP)(?:[ /](\d+)(?:\.(\d+)(?:\.(\d+))?)?)?' + + # Bots + - regex: '(1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]+-Agent|AdsBot-Google(?:-[a-z]+)?|altavista|AppEngine-Google|archive.*?\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]+)*|bingbot|BingPreview|blitzbot|BlogBridge|BoardReader(?: [A-Za-z]+)*|boitho.com-dc|BotSeer|\b\w*favicon\w*\b|\bYeti(?:-[a-z]+)?|Catchpoint bot|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher)?|Feed Seeker Bot|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]+-)?Googlebot(?:-[a-zA-Z]+)?|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile)?|IconSurf|IlTrovatore(?:-Setaccio)?|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]+Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masidani_bot|Mediapartners-Google|Microsoft .*? Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media *)?|msrbot|netresearch|Netvibes|NewsGator[^/]*|^NING|Nutch[^/]*|Nymesis|ObjectsSearch|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PlantyNet_WebRobot|Pompos|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slurp|snappy|Speedy Spider|Squrl Java|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|TwitterBot|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]+|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s)? Link Sleuth|Xerka [A-z]+Bot|yacy(?:bot)?|Yahoo[a-z]*Seeker|Yahoo! Slurp|Yandex\w+|YodaoBot(?:-[A-z]+)?|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?' + + # Bots General matcher 'name/0.0' + - regex: '(?:\/[A-Za-z0-9\.]+)? *([A-Za-z0-9 \-_\!\[\]:]*(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]*))/(\d+)(?:\.(\d+)(?:\.(\d+))?)?' + # Bots General matcher 'name 0.0' + - regex: '(?:\/[A-Za-z0-9\.]+)? *([A-Za-z0-9 _\!\[\]:]*(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]*)) (\d+)(?:\.(\d+)(?:\.(\d+))?)?' + # Bots containing spider|scrape|bot(but not CUBOT)|Crawl + - regex: '((?:[A-z0-9]+|[A-z\-]+ ?)?(?: the )?(?:[Ss][Pp][Ii][Dd][Ee][Rr]|[Ss]crape|[A-Za-z0-9-]*(?:[^C][^Uu])[Bb]ot|[Cc][Rr][Aa][Ww][Ll])[A-z0-9]*)(?:(?:[ /]| v)(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?' + + # HbbTV standard defines what features the browser should understand. + # but it's like targeting "HTML5 browsers", effective browser support depends on the model + # See os_parsers if you want to target a specific TV + - regex: '(HbbTV)/(\d+)\.(\d+)\.(\d+) \(' + + # must go before Firefox to catch Chimera/SeaMonkey/Camino + - regex: '(Chimera|SeaMonkey|Camino)/(\d+)\.(\d+)\.?([ab]?\d+[a-z]*)?' + + # Social Networks + # Facebook + - regex: '\[FB.*;(FBAV)/(\d+)(?:\.(\d+)(?:\.(\d)+)?)?' + family_replacement: 'Facebook' + # Pinterest + - regex: '\[(Pinterest)/[^\]]+\]' + - regex: '(Pinterest)(?: for Android(?: Tablet)?)?/(\d+)(?:\.(\d+)(?:\.(\d)+)?)?' + + # Firefox + - regex: '(Pale[Mm]oon)/(\d+)\.(\d+)\.?(\d+)?' + family_replacement: 'Pale Moon (Firefox Variant)' + - regex: '(Fennec)/(\d+)\.(\d+)\.?([ab]?\d+[a-z]*)' + family_replacement: 'Firefox Mobile' + - regex: '(Fennec)/(\d+)\.(\d+)(pre)' + family_replacement: 'Firefox Mobile' + - regex: '(Fennec)/(\d+)\.(\d+)' + family_replacement: 'Firefox Mobile' + - regex: '(?:Mobile|Tablet);.*(Firefox)/(\d+)\.(\d+)' + family_replacement: 'Firefox Mobile' + - regex: '(Namoroka|Shiretoko|Minefield)/(\d+)\.(\d+)\.(\d+(?:pre)?)' + family_replacement: 'Firefox ($1)' + - regex: '(Firefox)/(\d+)\.(\d+)(a\d+[a-z]*)' + family_replacement: 'Firefox Alpha' + - regex: '(Firefox)/(\d+)\.(\d+)(b\d+[a-z]*)' + family_replacement: 'Firefox Beta' + - regex: '(Firefox)-(?:\d+\.\d+)?/(\d+)\.(\d+)(a\d+[a-z]*)' + family_replacement: 'Firefox Alpha' + - regex: '(Firefox)-(?:\d+\.\d+)?/(\d+)\.(\d+)(b\d+[a-z]*)' + family_replacement: 'Firefox Beta' + - regex: '(Namoroka|Shiretoko|Minefield)/(\d+)\.(\d+)([ab]\d+[a-z]*)?' + family_replacement: 'Firefox ($1)' + - regex: '(Firefox).*Tablet browser (\d+)\.(\d+)\.(\d+)' + family_replacement: 'MicroB' + - regex: '(MozillaDeveloperPreview)/(\d+)\.(\d+)([ab]\d+[a-z]*)?' + - regex: '(FxiOS)/(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?' + family_replacement: 'Firefox iOS' + + # e.g.: Flock/2.0b2 + - regex: '(Flock)/(\d+)\.(\d+)(b\d+?)' + + # RockMelt + - regex: '(RockMelt)/(\d+)\.(\d+)\.(\d+)' + + # e.g.: Fennec/0.9pre + - regex: '(Navigator)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Netscape' + + - regex: '(Navigator)/(\d+)\.(\d+)([ab]\d+)' + family_replacement: 'Netscape' + + - regex: '(Netscape6)/(\d+)\.(\d+)\.?([ab]?\d+)?' + family_replacement: 'Netscape' + + - regex: '(MyIBrow)/(\d+)\.(\d+)' + family_replacement: 'My Internet Browser' + + # Opera will stop at 9.80 and hide the real version in the Version string. + # see: http://dev.opera.com/articles/view/opera-ua-string-changes/ + - regex: '(Opera Tablet).*Version/(\d+)\.(\d+)(?:\.(\d+))?' + - regex: '(Opera Mini)(?:/att)?/?(\d+)?(?:\.(\d+))?(?:\.(\d+))?' + - regex: '(Opera)/.+Opera Mobi.+Version/(\d+)\.(\d+)' + family_replacement: 'Opera Mobile' + - regex: '(Opera)/(\d+)\.(\d+).+Opera Mobi' + family_replacement: 'Opera Mobile' + - regex: 'Opera Mobi.+(Opera)(?:/|\s+)(\d+)\.(\d+)' + family_replacement: 'Opera Mobile' + - regex: 'Opera Mobi' + family_replacement: 'Opera Mobile' + - regex: '(Opera)/9.80.*Version/(\d+)\.(\d+)(?:\.(\d+))?' + + # Opera 14 for Android uses a WebKit render engine. + - regex: '(?:Mobile Safari).*(OPR)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Opera Mobile' + + # Opera >=15 for Desktop is similar to Chrome but includes an "OPR" Version string. + - regex: '(?:Chrome).*(OPR)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Opera' + + # Opera Coast + - regex: '(Coast)/(\d+).(\d+).(\d+)' + family_replacement: 'Opera Coast' + + # Opera Mini for iOS (from version 8.0.0) + - regex: '(OPiOS)/(\d+).(\d+).(\d+)' + family_replacement: 'Opera Mini' + + # Palm WebOS looks a lot like Safari. + - regex: '(hpw|web)OS/(\d+)\.(\d+)(?:\.(\d+))?' + family_replacement: 'webOS Browser' + + # LuaKit has no version info. + # http://luakit.org/projects/luakit/ + - regex: '(luakit)' + family_replacement: 'LuaKit' + + # Snowshoe + - regex: '(Snowshoe)/(\d+)\.(\d+).(\d+)' + + # Lightning (for Thunderbird) + # http://www.mozilla.org/projects/calendar/lightning/ + - regex: '(Lightning)/(\d+)\.(\d+)\.?((?:[ab]?\d+[a-z]*)|(?:\d*))' + + # Swiftfox + - regex: '(Firefox)/(\d+)\.(\d+)\.(\d+(?:pre)?) \(Swiftfox\)' + family_replacement: 'Swiftfox' + - regex: '(Firefox)/(\d+)\.(\d+)([ab]\d+[a-z]*)? \(Swiftfox\)' + family_replacement: 'Swiftfox' + + # Rekonq + - regex: '(rekonq)/(\d+)\.(\d+)\.?(\d+)? Safari' + family_replacement: 'Rekonq' + - regex: 'rekonq' + family_replacement: 'Rekonq' + + # Conkeror lowercase/uppercase + # http://conkeror.org/ + - regex: '(conkeror|Conkeror)/(\d+)\.(\d+)\.?(\d+)?' + family_replacement: 'Conkeror' + + # catches lower case konqueror + - regex: '(konqueror)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Konqueror' + + - regex: '(WeTab)-Browser' + + - regex: '(Comodo_Dragon)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Comodo Dragon' + + - regex: '(Symphony) (\d+).(\d+)' + + - regex: '(Minimo)' + + - regex: 'PLAYSTATION 3.+WebKit' + family_replacement: 'NetFront NX' + - regex: 'PLAYSTATION 3' + family_replacement: 'NetFront' + - regex: '(PlayStation Portable)' + family_replacement: 'NetFront' + - regex: '(PlayStation Vita)' + family_replacement: 'NetFront NX' + + - regex: 'AppleWebKit.+ (NX)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'NetFront NX' + - regex: '(Nintendo 3DS)' + family_replacement: 'NetFront NX' + + # Amazon Silk, should go before Safari and Chrome Mobile + - regex: '(Silk)/(\d+)\.(\d+)(?:\.([0-9\-]+))?' + family_replacement: 'Amazon Silk' + + + # @ref: http://www.puffinbrowser.com + - regex: '(Puffin)/(\d+)\.(\d+)(?:\.(\d+))?' + + # Edge Mobile + - regex: 'Windows Phone .*(Edge)/(\d+)\.(\d+)' + family_replacement: 'Edge Mobile' + + # Samsung Internet (based on Chrome, but lacking some features) + - regex: '(SamsungBrowser)/(\d+)\.(\d+)' + family_replacement: 'Samsung Internet' + + # Chrome Mobile + - regex: '(CrMo)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Chrome Mobile' + - regex: '(CriOS)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Chrome Mobile iOS' + - regex: '(Chrome)/(\d+)\.(\d+)\.(\d+)\.(\d+) Mobile' + family_replacement: 'Chrome Mobile' + + # Chrome Frame must come before MSIE. + - regex: '(chromeframe)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Chrome Frame' + + # UC Browser + - regex: '(UCBrowser)[ /](\d+)\.(\d+)\.(\d+)' + family_replacement: 'UC Browser' + - regex: '(UC Browser)[ /](\d+)\.(\d+)\.(\d+)' + - regex: '(UC Browser|UCBrowser|UCWEB)(\d+)\.(\d+)\.(\d+)' + family_replacement: 'UC Browser' + + # Tizen Browser (second case included in browser/major.minor regex) + - regex: '(SLP Browser)/(\d+)\.(\d+)' + family_replacement: 'Tizen Browser' + + # Sogou Explorer 2.X + - regex: '(SE 2\.X) MetaSr (\d+)\.(\d+)' + family_replacement: 'Sogou Explorer' + + # Baidu Browsers (desktop spoofs chrome & IE, explorer is mobile) + - regex: '(baidubrowser)[/\s](\d+)' + family_replacement: 'Baidu Browser' + - regex: '(FlyFlow)/(\d+)\.(\d+)' + family_replacement: 'Baidu Explorer' + + # QQ Browsers + - regex: '(MQQBrowser/Mini)(?:(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?' + family_replacement: 'QQ Browser Mini' + - regex: '(MQQBrowser)(?:/(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?' + family_replacement: 'QQ Browser Mobile' + - regex: '(QQBrowser)(?:/(\d+)(?:\.(\d+)\.(\d+)(?:\.(\d+))?)?)?' + family_replacement: 'QQ Browser' + + # Rackspace Monitoring + - regex: '(Rackspace Monitoring)/(\d+)\.(\d+)' + family_replacement: 'RackspaceBot' + + # PyAMF + - regex: '(PyAMF)/(\d+)\.(\d+)\.(\d+)' + + # Yandex Browser + - regex: '(YaBrowser)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Yandex Browser' + + # Mail.ru Amigo/Internet Browser (Chromium-based) + - regex: '(Chrome)/(\d+)\.(\d+)\.(\d+).* MRCHROME' + family_replacement: 'Mail.ru Chromium Browser' + + # AOL Browser (IE-based) + - regex: '(AOL) (\d+)\.(\d+); AOLBuild (\d+)' + + #### END SPECIAL CASES TOP #### + + #### MAIN CASES - this catches > 50% of all browsers #### + + # Browser/major_version.minor_version.beta_version + - regex: '(AdobeAIR|FireWeb|Jasmine|ANTGalio|Midori|Fresco|Lobo|PaleMoon|Maxthon|Lynx|OmniWeb|Dillo|Camino|Demeter|Fluid|Fennec|Epiphany|Shiira|Sunrise|Spotify|Flock|Netscape|Lunascape|WebPilot|NetFront|Netfront|Konqueror|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|Opera Mini|iCab|NetNewsWire|ThunderBrowse|Iris|UP\.Browser|Bunjalloo|Google Earth|Raven for Mac|Openwave)/(\d+)\.(\d+)\.(\d+)' + + # Outlook 2007 + - regex: 'Microsoft Office Outlook 12\.\d+\.\d+|MSOffice 12' + family_replacement: 'Outlook' + v1_replacement: '2007' + + # Outlook 2010 + - regex: 'Microsoft Outlook 14\.\d+\.\d+|MSOffice 14' + family_replacement: 'Outlook' + v1_replacement: '2010' + + # Outlook 2013 + - regex: 'Microsoft Outlook 15\.\d+\.\d+' + family_replacement: 'Outlook' + v1_replacement: '2013' + + # Outlook 2016 + - regex: 'Microsoft Outlook (?:Mail )?16\.\d+\.\d+' + family_replacement: 'Outlook' + v1_replacement: '2016' + + # Windows Live Mail + - regex: 'Outlook-Express\/7\.0.*' + family_replacement: 'Windows Live Mail' + + # Apple Air Mail + - regex: '(Airmail) (\d+)\.(\d+)(?:\.(\d+))?' + + # Thunderbird + - regex: '(Thunderbird)/(\d+)\.(\d+)\.(\d+(?:pre)?)' + family_replacement: 'Thunderbird' + + # Vivaldi uses "Vivaldi" + - regex: '(Vivaldi)/(\d+)\.(\d+)\.(\d+)' + + # Edge/major_version.minor_version + - regex: '(Edge)/(\d+)\.(\d+)' + + # Brave Browser https://brave.com/ + - regex: '(brave)/(\d+)\.(\d+)\.(\d+) Chrome' + family_replacement: 'Brave' + + # Chrome/Chromium/major_version.minor_version.beta_version + - regex: '(Chromium|Chrome)/(\d+)\.(\d+)\.(\d+)' + + # Dolphin Browser + # @ref: http://www.dolphin.com + - regex: '\b(Dolphin)(?: |HDCN/|/INT\-)(\d+)\.(\d+)\.?(\d+)?' + + # Browser/major_version.minor_version + - regex: '(bingbot|Bolt|Jasmine|IceCat|Skyfire|Midori|Maxthon|Lynx|Arora|IBrowse|Dillo|Camino|Shiira|Fennec|Phoenix|Chrome|Flock|Netscape|Lunascape|Epiphany|WebPilot|Opera Mini|Opera|NetFront|Netfront|Konqueror|Googlebot|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|iCab|iTunes|MacAppStore|NetNewsWire|Space Bison|Stainless|Orca|Dolfin|BOLT|Minimo|Tizen Browser|Polaris|Abrowser|Planetweb|ICE Browser|mDolphin|qutebrowser|Otter|QupZilla)/(\d+)\.(\d+)\.?(\d+)?' + + # Chrome/Chromium/major_version.minor_version + - regex: '(Chromium|Chrome)/(\d+)\.(\d+)' + + ########## + # IE Mobile needs to happen before Android to catch cases such as: + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; ANZ821)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Orange)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Vodafone)... + ########## + + # IE Mobile + - regex: '(IEMobile)[ /](\d+)\.(\d+)' + family_replacement: 'IE Mobile' + + # Browser major_version.minor_version.beta_version (space instead of slash) + - regex: '(iRider|Crazy Browser|SkipStone|iCab|Lunascape|Sleipnir|Maemo Browser) (\d+)\.(\d+)\.(\d+)' + # Browser major_version.minor_version (space instead of slash) + - regex: '(iCab|Lunascape|Opera|Android|Jasmine|Polaris) (\d+)\.(\d+)\.?(\d+)?' + + # Kindle WebKit + - regex: '(Kindle)/(\d+)\.(\d+)' + + # weird android UAs + - regex: '(Android) Donut' + v1_replacement: '1' + v2_replacement: '2' + + - regex: '(Android) Eclair' + v1_replacement: '2' + v2_replacement: '1' + + - regex: '(Android) Froyo' + v1_replacement: '2' + v2_replacement: '2' + + - regex: '(Android) Gingerbread' + v1_replacement: '2' + v2_replacement: '3' + + - regex: '(Android) Honeycomb' + v1_replacement: '3' + + # desktop mode + # http://www.anandtech.com/show/3982/windows-phone-7-review + - regex: '(MSIE) (\d+)\.(\d+).*XBLWP7' + family_replacement: 'IE Large Screen' + + #### END MAIN CASES #### + + #### SPECIAL CASES #### + - regex: '(Obigo)InternetBrowser' + - regex: '(Obigo)\-Browser' + - regex: '(Obigo|OBIGO)[^\d]*(\d+)(?:.(\d+))?' + family_replacement: 'Obigo' + + - regex: '(MAXTHON|Maxthon) (\d+)\.(\d+)' + family_replacement: 'Maxthon' + - regex: '(Maxthon|MyIE2|Uzbl|Shiira)' + v1_replacement: '0' + + - regex: '(BrowseX) \((\d+)\.(\d+)\.(\d+)' + + - regex: '(NCSA_Mosaic)/(\d+)\.(\d+)' + family_replacement: 'NCSA Mosaic' + + # Polaris/d.d is above + - regex: '(POLARIS)/(\d+)\.(\d+)' + family_replacement: 'Polaris' + - regex: '(Embider)/(\d+)\.(\d+)' + family_replacement: 'Polaris' + + - regex: '(BonEcho)/(\d+)\.(\d+)\.?([ab]?\d+)?' + family_replacement: 'Bon Echo' + + # @note: iOS / OSX Applications + - regex: '(iPod|iPhone|iPad).+Version/(\d+)\.(\d+)(?:\.(\d+))?.* Safari' + family_replacement: 'Mobile Safari' + - regex: '(iPod|iPhone|iPad).+Version/(\d+)\.(\d+)(?:\.(\d+))?' + family_replacement: 'Mobile Safari UI/WKWebView' + - regex: '(iPod|iPhone|iPad);.*CPU.*OS (\d+)_(\d+)(?:_(\d+))?.*Mobile.* Safari' + family_replacement: 'Mobile Safari' + - regex: '(iPod|iPhone|iPad);.*CPU.*OS (\d+)_(\d+)(?:_(\d+))?.*Mobile' + family_replacement: 'Mobile Safari UI/WKWebView' + - regex: '(iPod|iPhone|iPad).* Safari' + family_replacement: 'Mobile Safari' + - regex: '(iPod|iPhone|iPad)' + family_replacement: 'Mobile Safari UI/WKWebView' + + - regex: '(AvantGo) (\d+).(\d+)' + + - regex: '(OneBrowser)/(\d+).(\d+)' + family_replacement: 'ONE Browser' + + - regex: '(Avant)' + v1_replacement: '1' + + # This is the Tesla Model S (see similar entry in device parsers) + - regex: '(QtCarBrowser)' + v1_replacement: '1' + + - regex: '^(iBrowser/Mini)(\d+).(\d+)' + family_replacement: 'iBrowser Mini' + - regex: '^(iBrowser|iRAPP)/(\d+).(\d+)' + + # nokia browsers + # based on: http://www.developer.nokia.com/Community/Wiki/User-Agent_headers_for_Nokia_devices + - regex: '^(Nokia)' + family_replacement: 'Nokia Services (WAP) Browser' + - regex: '(NokiaBrowser)/(\d+)\.(\d+).(\d+)\.(\d+)' + family_replacement: 'Nokia Browser' + - regex: '(NokiaBrowser)/(\d+)\.(\d+).(\d+)' + family_replacement: 'Nokia Browser' + - regex: '(NokiaBrowser)/(\d+)\.(\d+)' + family_replacement: 'Nokia Browser' + - regex: '(BrowserNG)/(\d+)\.(\d+).(\d+)' + family_replacement: 'Nokia Browser' + - regex: '(Series60)/5\.0' + family_replacement: 'Nokia Browser' + v1_replacement: '7' + v2_replacement: '0' + - regex: '(Series60)/(\d+)\.(\d+)' + family_replacement: 'Nokia OSS Browser' + - regex: '(S40OviBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Ovi Browser' + - regex: '(Nokia)[EN]?(\d+)' + + # BlackBerry devices + - regex: '(PlayBook).+RIM Tablet OS (\d+)\.(\d+)\.(\d+)' + family_replacement: 'BlackBerry WebKit' + - regex: '(Black[bB]erry|BB10).+Version/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'BlackBerry WebKit' + - regex: '(Black[bB]erry)\s?(\d+)' + family_replacement: 'BlackBerry' + + - regex: '(OmniWeb)/v(\d+)\.(\d+)' + + - regex: '(Blazer)/(\d+)\.(\d+)' + family_replacement: 'Palm Blazer' + + - regex: '(Pre)/(\d+)\.(\d+)' + family_replacement: 'Palm Pre' + + # fork of Links + - regex: '(ELinks)/(\d+)\.(\d+)' + - regex: '(ELinks) \((\d+)\.(\d+)' + - regex: '(Links) \((\d+)\.(\d+)' + + - regex: '(QtWeb) Internet Browser/(\d+)\.(\d+)' + + #- regex: '\(iPad;.+(Version)/(\d+)\.(\d+)(?:\.(\d+))?.*Safari/' + # family_replacement: 'iPad' + + # Phantomjs, should go before Safari + - regex: '(PhantomJS)/(\d+)\.(\d+)\.(\d+)' + + # WebKit Nightly + - regex: '(AppleWebKit)/(\d+)\.?(\d+)?\+ .* Safari' + family_replacement: 'WebKit Nightly' + + # Safari + - regex: '(Version)/(\d+)\.(\d+)(?:\.(\d+))?.*Safari/' + family_replacement: 'Safari' + # Safari didn't provide "Version/d.d.d" prior to 3.0 + - regex: '(Safari)/\d+' + + - regex: '(OLPC)/Update(\d+)\.(\d+)' + + - regex: '(OLPC)/Update()\.(\d+)' + v1_replacement: '0' + + - regex: '(SEMC\-Browser)/(\d+)\.(\d+)' + + - regex: '(Teleca)' + family_replacement: 'Teleca Browser' + + - regex: '(Phantom)/V(\d+)\.(\d+)' + family_replacement: 'Phantom Browser' + + - regex: 'Trident(.*)rv.(\d+)\.(\d+)' + family_replacement: 'IE' + + # Espial + - regex: '(Espial)/(\d+)(?:\.(\d+))?(?:\.(\d+))?' + + # Apple Mail + + # apple mail - not directly detectable, have it after Safari stuff + - regex: '(AppleWebKit)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Apple Mail' + + # AFTER THE EDGE CASES ABOVE! + # AFTER IE11 + # BEFORE all other IE + - regex: '(Firefox)/(\d+)\.(\d+)\.(\d+)' + - regex: '(Firefox)/(\d+)\.(\d+)(pre|[ab]\d+[a-z]*)?' + + - regex: '([MS]?IE) (\d+)\.(\d+)' + family_replacement: 'IE' + + - regex: '(python-requests)/(\d+)\.(\d+)' + family_replacement: 'Python Requests' + + - regex: '(Java)[/ ]{0,1}\d+\.(\d+)\.(\d+)[_-]*([a-zA-Z0-9]+)*' + + # Roku Digital-Video-Players https://www.roku.com/ + - regex: '^(Roku)/DVP-(\d+)\.(\d+)' + +os_parsers: + ########## + # HbbTV vendors + ########## + + # starts with the easy one : Panasonic seems consistent across years, hope it will continue + #HbbTV/1.1.1 (;Panasonic;VIERA 2011;f.532;0071-0802 2000-0000;) + #HbbTV/1.1.1 (;Panasonic;VIERA 2012;1.261;0071-3103 2000-0000;) + #HbbTV/1.2.1 (;Panasonic;VIERA 2013;3.672;4101-0003 0002-0000;) + #- regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Panasonic);VIERA ([0-9]{4});' + + # Sony is consistent too but do not place year like the other + # Opera/9.80 (Linux armv7l; HbbTV/1.1.1 (; Sony; KDL32W650A; PKG3.211EUA; 2013;); ) Presto/2.12.362 Version/12.11 + # Opera/9.80 (Linux mips; U; HbbTV/1.1.1 (; Sony; KDL40HX751; PKG1.902EUA; 2012;);; en) Presto/2.10.250 Version/11.60 + # Opera/9.80 (Linux mips; U; HbbTV/1.1.1 (; Sony; KDL22EX320; PKG4.017EUA; 2011;);; en) Presto/2.7.61 Version/11.00 + #- regex: 'HbbTV/\d+\.\d+\.\d+ \(; (Sony);.*;.*; ([0-9]{4});\)' + + + # LG is consistent too, but we need to add manually the year model + #Mozilla/5.0 (Unknown; Linux armv7l) AppleWebKit/537.1+ (KHTML, like Gecko) Safari/537.1+ HbbTV/1.1.1 ( ;LGE ;NetCast 4.0 ;03.20.30 ;1.0M ;) + #Mozilla/5.0 (DirectFB; Linux armv7l) AppleWebKit/534.26+ (KHTML, like Gecko) Version/5.0 Safari/534.26+ HbbTV/1.1.1 ( ;LGE ;NetCast 3.0 ;1.0 ;1.0M ;) + - regex: 'HbbTV/\d+\.\d+\.\d+ \( ;(LG)E ;NetCast 4.0' + os_v1_replacement: '2013' + - regex: 'HbbTV/\d+\.\d+\.\d+ \( ;(LG)E ;NetCast 3.0' + os_v1_replacement: '2012' + + # Samsung is on its way of normalizing their user-agent + # HbbTV/1.1.1 (;Samsung;SmartTV2013;T-FXPDEUC-1102.2;;) WebKit + # HbbTV/1.1.1 (;Samsung;SmartTV2013;T-MST12DEUC-1102.1;;) WebKit + # HbbTV/1.1.1 (;Samsung;SmartTV2012;;;) WebKit + # HbbTV/1.1.1 (;;;;;) Maple_2011 + - regex: 'HbbTV/1.1.1 \(;;;;;\) Maple_2011' + os_replacement: 'Samsung' + os_v1_replacement: '2011' + # manage the two models of 2013 + - regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Samsung);SmartTV([0-9]{4});.*FXPDEUC' + os_v2_replacement: 'UE40F7000' + - regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Samsung);SmartTV([0-9]{4});.*MST12DEUC' + os_v2_replacement: 'UE32F4500' + # generic Samsung (works starting in 2012) + #- regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Samsung);SmartTV([0-9]{4});' + + # Philips : not found any other way than a manual mapping + # Opera/9.80 (Linux mips; U; HbbTV/1.1.1 (; Philips; ; ; ; ) CE-HTML/1.0 NETTV/4.1.3 PHILIPSTV/1.1.1; en) Presto/2.10.250 Version/11.60 + # Opera/9.80 (Linux mips ; U; HbbTV/1.1.1 (; Philips; ; ; ; ) CE-HTML/1.0 NETTV/3.2.1; en) Presto/2.6.33 Version/10.70 + - regex: 'HbbTV/1.1.1 \(; (Philips);.*NETTV/4' + os_v1_replacement: '2013' + - regex: 'HbbTV/1.1.1 \(; (Philips);.*NETTV/3' + os_v1_replacement: '2012' + - regex: 'HbbTV/1.1.1 \(; (Philips);.*NETTV/2' + os_v1_replacement: '2011' + + # the HbbTV emulator developers use HbbTV/1.1.1 (;;;;;) firetv-firefox-plugin 1.1.20 + - regex: 'HbbTV/\d+\.\d+\.\d+.*(firetv)-firefox-plugin (\d+).(\d+).(\d+)' + os_replacement: 'FireHbbTV' + + # generic HbbTV, hoping to catch manufacturer name (always after 2nd comma) and the first string that looks like a 2011-2019 year + - regex: 'HbbTV/\d+\.\d+\.\d+ \(.*; ?([a-zA-Z]+) ?;.*(201[1-9]).*\)' + + ########## + # @note: Windows Phone needs to come before Windows NT 6.1 *and* before Android to catch cases such as: + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; ANZ821)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Orange)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Vodafone)... + ########## + + - regex: '(Windows Phone) (?:OS[ /])?(\d+)\.(\d+)' + + ########## + # Android + # can actually detect rooted android os. do we care? + ########## + - regex: '(Android)[ \-/](\d+)\.(\d+)(?:[.\-]([a-z0-9]+))?' + + - regex: '(Android) Donut' + os_v1_replacement: '1' + os_v2_replacement: '2' + + - regex: '(Android) Eclair' + os_v1_replacement: '2' + os_v2_replacement: '1' + + - regex: '(Android) Froyo' + os_v1_replacement: '2' + os_v2_replacement: '2' + + - regex: '(Android) Gingerbread' + os_v1_replacement: '2' + os_v2_replacement: '3' + + - regex: '(Android) Honeycomb' + os_v1_replacement: '3' + + # UCWEB + - regex: '^UCWEB.*; (Adr) (\d+)\.(\d+)(?:[.\-]([a-z0-9]+))?;' + os_replacement: 'Android' + - regex: '^UCWEB.*; (iPad OS|iPh OS) (\d+)_(\d+)(?:_(\d+))?;' + os_replacement: 'iOS' + - regex: '^UCWEB.*; (wds) (\d+)\.(\d+)(?:\.(\d+))?;' + os_replacement: 'Windows Phone' + # JUC + - regex: '^(JUC).*; ?U; ?(?:Android)?(\d+)\.(\d+)(?:[\.\-]([a-z0-9]+))?' + os_replacement: 'Android' + + ########## + # Kindle Android + ########## + - regex: '(Silk-Accelerated=[a-z]{4,5})' + os_replacement: 'Android' + + ########## + # Windows + # http://en.wikipedia.org/wiki/Windows_NT#Releases + # possibility of false positive when different marketing names share same NT kernel + # e.g. windows server 2003 and windows xp + # lots of ua strings have Windows NT 4.1 !?!?!?!? !?!? !? !????!?! !!! ??? !?!?! ? + # (very) roughly ordered in terms of frequency of occurence of regex (win xp currently most frequent, etc) + ########## + + # ie mobile desktop mode + # spoofs nt 6.1. must come before windows 7 + - regex: '(XBLWP7)' + os_replacement: 'Windows Phone' + + # @note: This needs to come before Windows NT 6.1 + - regex: '(Windows ?Mobile)' + os_replacement: 'Windows Mobile' + + - regex: '(Windows (?:NT 5\.2|NT 5\.1))' + os_replacement: 'Windows XP' + + - regex: '(Windows NT 6\.1)' + os_replacement: 'Windows 7' + + - regex: '(Windows NT 6\.0)' + os_replacement: 'Windows Vista' + + - regex: '(Win 9x 4\.90)' + os_replacement: 'Windows ME' + + - regex: '(Windows 98|Windows XP|Windows ME|Windows 95|Windows CE|Windows 7|Windows NT 4\.0|Windows Vista|Windows 2000|Windows 3.1)' + + - regex: '(Windows NT 6\.2; ARM;)' + os_replacement: 'Windows RT' + - regex: '(Windows NT 6\.2)' + os_replacement: 'Windows 8' + + - regex: '(Windows NT 6\.3; ARM;)' + os_replacement: 'Windows RT 8.1' + - regex: '(Windows NT 6\.3)' + os_replacement: 'Windows 8.1' + + - regex: '(Windows NT 6\.4)' + os_replacement: 'Windows 10' + - regex: '(Windows NT 10\.0)' + os_replacement: 'Windows 10' + + - regex: '(Windows NT 5\.0)' + os_replacement: 'Windows 2000' + + - regex: '(WinNT4.0)' + os_replacement: 'Windows NT 4.0' + + - regex: '(Windows ?CE)' + os_replacement: 'Windows CE' + + - regex: 'Win ?(95|98|3.1|NT|ME|2000)' + os_replacement: 'Windows $1' + + - regex: 'Win16' + os_replacement: 'Windows 3.1' + + - regex: 'Win32' + os_replacement: 'Windows 95' + + ########## + # Tizen OS from Samsung + # spoofs Android so pushing it above + ########## + - regex: '(Tizen)/(\d+)\.(\d+)' + + ########## + # Mac OS + # @ref: http://en.wikipedia.org/wiki/Mac_OS_X#Versions + # @ref: http://www.puredarwin.org/curious/versions + ########## + - regex: '((?:Mac ?|; )OS X)[\s/](?:(\d+)[_.](\d+)(?:[_.](\d+))?|Mach-O)' + os_replacement: 'Mac OS X' + # Leopard + - regex: ' (Dar)(win)/(9).(\d+).*\((?:i386|x86_64|Power Macintosh)\)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '5' + # Snow Leopard + - regex: ' (Dar)(win)/(10).(\d+).*\((?:i386|x86_64)\)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '6' + # Lion + - regex: ' (Dar)(win)/(11).(\d+).*\((?:i386|x86_64)\)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '7' + # Mountain Lion + - regex: ' (Dar)(win)/(12).(\d+).*\((?:i386|x86_64)\)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '8' + # Mavericks + - regex: ' (Dar)(win)/(13).(\d+).*\((?:i386|x86_64)\)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '9' + # Yosemite is Darwin/14.x but patch versions are inconsistent in the Darwin string; + # more accurately covered by CFNetwork regexes downstream + + # IE on Mac doesn't specify version number + - regex: 'Mac_PowerPC' + os_replacement: 'Mac OS' + + # builds before tiger don't seem to specify version? + + # ios devices spoof (mac os x), so including intel/ppc prefixes + - regex: '(?:PPC|Intel) (Mac OS X)' + + ########## + # iOS + # http://en.wikipedia.org/wiki/IOS_version_history + ########## + # keep this above generic iOS, since AppleTV UAs contain 'CPU OS' + - regex: '(Apple\s?TV)(?:/(\d+)\.(\d+))?' + os_replacement: 'ATV OS X' + + - regex: '(CPU OS|iPhone OS|CPU iPhone) +(\d+)[_\.](\d+)(?:[_\.](\d+))?' + os_replacement: 'iOS' + + # remaining cases are mostly only opera uas, so catch opera as to not catch iphone spoofs + - regex: '(iPhone|iPad|iPod); Opera' + os_replacement: 'iOS' + + # few more stragglers + - regex: '(iPhone|iPad|iPod).*Mac OS X.*Version/(\d+)\.(\d+)' + os_replacement: 'iOS' + + # CFNetwork/Darwin - The specific CFNetwork or Darwin version determines + # whether the os maps to Mac OS, or iOS, or just Darwin. + # See: http://user-agents.me/cfnetwork-version-list + - regex: '(CFNetwork)/(5)48\.0\.3.* Darwin/11\.0\.0' + os_replacement: 'iOS' + - regex: '(CFNetwork)/(5)48\.(0)\.4.* Darwin/(1)1\.0\.0' + os_replacement: 'iOS' + - regex: '(CFNetwork)/(5)48\.(1)\.4' + os_replacement: 'iOS' + - regex: '(CFNetwork)/(4)85\.1(3)\.9' + os_replacement: 'iOS' + - regex: '(CFNetwork)/(6)09\.(1)\.4' + os_replacement: 'iOS' + - regex: '(CFNetwork)/(6)(0)9' + os_replacement: 'iOS' + - regex: '(CFNetwork)/6(7)2\.(1)\.13' + os_replacement: 'iOS' + - regex: '(CFNetwork)/6(7)2\.(1)\.(1)4' + os_replacement: 'iOS' + - regex: '(CF)(Network)/6(7)(2)\.1\.15' + os_replacement: 'iOS' + os_v1_replacement: '7' + os_v2_replacement: '1' + - regex: '(CFNetwork)/6(7)2\.(0)\.(?:2|8)' + os_replacement: 'iOS' + - regex: '(CFNetwork)/709\.1' + os_replacement: 'iOS' + os_v1_replacement: '8' + os_v2_replacement: '0.b5' + - regex: '(CF)(Network)/711\.(\d)' + os_replacement: 'iOS' + os_v1_replacement: '8' + - regex: '(CF)(Network)/(720)\.(\d)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '10' + - regex: '(CF)(Network)/758\.(\d)' + os_replacement: 'iOS' + os_v1_replacement: '9' + + ########## + # CFNetwork iOS Apps + # @ref: https://en.wikipedia.org/wiki/Darwin_(operating_system)#Release_history + ########## + - regex: 'CFNetwork/.* Darwin/(9)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '1' + - regex: 'CFNetwork/.* Darwin/(10)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '4' + - regex: 'CFNetwork/.* Darwin/(11)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '5' + - regex: 'CFNetwork/.* Darwin/(13)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '6' + - regex: 'CFNetwork/6.* Darwin/(14)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '7' + - regex: 'CFNetwork/7.* Darwin/(14)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '8' + os_v2_replacement: '0' + - regex: 'CFNetwork/7.* Darwin/(15)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '9' + os_v2_replacement: '0' + # iOS Apps + - regex: '\b(iOS[ /]|iPhone(?:/| v|[ _]OS[/,]|; | OS : |\d,\d/|\d,\d; )|iPad/)(\d{1,2})[_\.](\d{1,2})(?:[_\.](\d+))?' + os_replacement: 'iOS' + + ########## + # Apple TV + ########## + - regex: '(tvOS)/(\d+).(\d+)' + os_replacement: 'tvOS' + + ########## + # Chrome OS + # if version 0.0.0, probably this stuff: + # http://code.google.com/p/chromium-os/issues/detail?id=11573 + # http://code.google.com/p/chromium-os/issues/detail?id=13790 + ########## + - regex: '(CrOS) [a-z0-9_]+ (\d+)\.(\d+)(?:\.(\d+))?' + os_replacement: 'Chrome OS' + + ########## + # Linux distros + ########## + - regex: '([Dd]ebian)' + os_replacement: 'Debian' + - regex: '(Linux Mint)(?:/(\d+))?' + - regex: '(Mandriva)(?: Linux)?/(?:[\d.-]+m[a-z]{2}(\d+).(\d))?' + + ########## + # Symbian + Symbian OS + # http://en.wikipedia.org/wiki/History_of_Symbian + ########## + - regex: '(Symbian[Oo][Ss])[/ ](\d+)\.(\d+)' + os_replacement: 'Symbian OS' + - regex: '(Symbian/3).+NokiaBrowser/7\.3' + os_replacement: 'Symbian^3 Anna' + - regex: '(Symbian/3).+NokiaBrowser/7\.4' + os_replacement: 'Symbian^3 Belle' + - regex: '(Symbian/3)' + os_replacement: 'Symbian^3' + - regex: '\b(Series 60|SymbOS|S60Version|S60V\d|S60\b)' + os_replacement: 'Symbian OS' + - regex: '(MeeGo)' + - regex: 'Symbian [Oo][Ss]' + os_replacement: 'Symbian OS' + - regex: 'Series40;' + os_replacement: 'Nokia Series 40' + - regex: 'Series30Plus;' + os_replacement: 'Nokia Series 30 Plus' + + ########## + # BlackBerry devices + ########## + - regex: '(BB10);.+Version/(\d+)\.(\d+)\.(\d+)' + os_replacement: 'BlackBerry OS' + - regex: '(Black[Bb]erry)[0-9a-z]+/(\d+)\.(\d+)\.(\d+)(?:\.(\d+))?' + os_replacement: 'BlackBerry OS' + - regex: '(Black[Bb]erry).+Version/(\d+)\.(\d+)\.(\d+)(?:\.(\d+))?' + os_replacement: 'BlackBerry OS' + - regex: '(RIM Tablet OS) (\d+)\.(\d+)\.(\d+)' + os_replacement: 'BlackBerry Tablet OS' + - regex: '(Play[Bb]ook)' + os_replacement: 'BlackBerry Tablet OS' + - regex: '(Black[Bb]erry)' + os_replacement: 'BlackBerry OS' + + ########## + # Firefox OS + ########## + - regex: '\((?:Mobile|Tablet);.+Gecko/18.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '1' + os_v2_replacement: '0' + os_v3_replacement: '1' + + - regex: '\((?:Mobile|Tablet);.+Gecko/18.1 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '1' + os_v2_replacement: '1' + + - regex: '\((?:Mobile|Tablet);.+Gecko/26.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '1' + os_v2_replacement: '2' + + - regex: '\((?:Mobile|Tablet);.+Gecko/28.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '1' + os_v2_replacement: '3' + + - regex: '\((?:Mobile|Tablet);.+Gecko/30.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '1' + os_v2_replacement: '4' + + - regex: '\((?:Mobile|Tablet);.+Gecko/32.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '2' + os_v2_replacement: '0' + + - regex: '\((?:Mobile|Tablet);.+Gecko/34.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '2' + os_v2_replacement: '1' + + # Firefox OS Generic + - regex: '\((?:Mobile|Tablet);.+Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + + + ########## + # BREW + # yes, Brew is lower-cased for Brew MP + ########## + - regex: '(BREW)[ /](\d+)\.(\d+)\.(\d+)' + - regex: '(BREW);' + - regex: '(Brew MP|BMP)[ /](\d+)\.(\d+)\.(\d+)' + os_replacement: 'Brew MP' + - regex: 'BMP;' + os_replacement: 'Brew MP' + + ########## + # Google TV + ########## + - regex: '(GoogleTV)(?: (\d+)\.(\d+)(?:\.(\d+))?|/[\da-z]+)' + + - regex: '(WebTV)/(\d+).(\d+)' + + ########## + # Misc mobile + ########## + - regex: '(hpw|web)OS/(\d+)\.(\d+)(?:\.(\d+))?' + os_replacement: 'webOS' + - regex: '(VRE);' + + ########## + # Generic patterns + # since the majority of os cases are very specific, these go last + ########## + - regex: '(Fedora|Red Hat|PCLinuxOS|Puppy|Ubuntu|Kindle|Bada|Lubuntu|BackTrack|Slackware|(?:Free|Open|Net|\b)BSD)[/ ](\d+)\.(\d+)(?:\.(\d+)(?:\.(\d+))?)?' + + # Gentoo Linux + Kernel Version + - regex: '(Linux)[ /](\d+)\.(\d+)(?:\.(\d+))?.*gentoo' + os_replacement: 'Gentoo' + + # Opera Mini Bada + - regex: '\((Bada);' + + # just os + - regex: '(Windows|Android|WeTab|Maemo)' + - regex: '(Ubuntu|Kubuntu|Arch Linux|CentOS|Slackware|Gentoo|openSUSE|SUSE|Red Hat|Fedora|PCLinuxOS|Mageia|(?:Free|Open|Net|\b)BSD)' + # Linux + Kernel Version + - regex: '(Linux)(?:[ /](\d+)\.(\d+)(?:\.(\d+))?)?' + - regex: 'SunOS' + os_replacement: 'Solaris' + + # Roku Digital-Video-Players https://www.roku.com/ + - regex: '^(Roku)/DVP-(\d+)\.(\d+)' + +device_parsers: + + ######### + # Mobile Spiders + # Catch the mobile crawler before checking for iPhones / Androids. + ######### + - regex: '(?:(?:iPhone|Windows CE|Android).*(?:(?:Bot|Yeti)-Mobile|YRSpider|bots?/\d|(?:bot|spider)\.html)|AdsBot-Google-Mobile.*iPhone)' + regex_flag: 'i' + device_replacement: 'Spider' + brand_replacement: 'Spider' + model_replacement: 'Smartphone' + - regex: '(?:DoCoMo|\bMOT\b|\bLG\b|Nokia|Samsung|SonyEricsson).*(?:(?:Bot|Yeti)-Mobile|bots?/\d|(?:bot|crawler)\.html|(?:jump|google|Wukong)bot|ichiro/mobile|/spider|YahooSeeker)' + regex_flag: 'i' + device_replacement: 'Spider' + brand_replacement: 'Spider' + model_replacement: 'Feature Phone' + + ######### + # WebBrowser for SmartWatch + # @ref: https://play.google.com/store/apps/details?id=se.vaggan.webbrowser&hl=en + ######### + - regex: '\bSmartWatch *\( *([^;]+) *; *([^;]+) *;' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ###################################################################### + # Android parsers + # + # @ref: https://support.google.com/googleplay/answer/1727131?hl=en + ###################################################################### + + # Android Application + - regex: 'Android Application[^\-]+ - (Sony) ?(Ericsson)? (.+) \w+ - ' + device_replacement: '$1 $2' + brand_replacement: '$1$2' + model_replacement: '$3' + - regex: 'Android Application[^\-]+ - (?:HTC|HUAWEI|LGE|LENOVO|MEDION|TCT) (HTC|HUAWEI|LG|LENOVO|MEDION|ALCATEL)[ _\-](.+) \w+ - ' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + - regex: 'Android Application[^\-]+ - ([^ ]+) (.+) \w+ - ' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # 3Q + # @ref: http://www.3q-int.com/ + ######### + - regex: '; *([BLRQ]C\d{4}[A-Z]+) +Build/' + device_replacement: '3Q $1' + brand_replacement: '3Q' + model_replacement: '$1' + - regex: '; *(?:3Q_)([^;/]+) +Build' + device_replacement: '3Q $1' + brand_replacement: '3Q' + model_replacement: '$1' + + ######### + # Acer + # @ref: http://us.acer.com/ac/en/US/content/group/tablets + ######### + - regex: 'Android [34].*; *(A100|A101|A110|A200|A210|A211|A500|A501|A510|A511|A700(?: Lite| 3G)?|A701|B1-A71|A1-\d{3}|B1-\d{3}|V360|V370|W500|W500P|W501|W501P|W510|W511|W700|Slider SL101|DA22[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Acer' + model_replacement: '$1' + - regex: '; *Acer Iconia Tab ([^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Acer' + model_replacement: '$1' + - regex: '; *(Z1[1235]0|E320[^/]*|S500|S510|Liquid[^;/]*|Iconia A\d+) Build' + device_replacement: '$1' + brand_replacement: 'Acer' + model_replacement: '$1' + - regex: '; *(Acer |ACER )([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Acer' + model_replacement: '$2' + + ######### + # Advent + # @ref: https://en.wikipedia.org/wiki/Advent_Vega + # @note: VegaBean and VegaComb (names derived from jellybean, honeycomb) are + # custom ROM builds for Vega + ######### + - regex: '; *(Advent )?(Vega(?:Bean|Comb)?).* Build' + device_replacement: '$1$2' + brand_replacement: 'Advent' + model_replacement: '$2' + + ######### + # Ainol + # @ref: http://www.ainol.com/plugin.php?identifier=ainol&module=product + ######### + - regex: '; *(Ainol )?((?:NOVO|[Nn]ovo)[^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Ainol' + model_replacement: '$2' + + ######### + # Airis + # @ref: http://airis.es/Tienda/Default.aspx?idG=001 + ######### + - regex: '; *AIRIS[ _\-]?([^/;\)]+) *(?:;|\)|Build)' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Airis' + model_replacement: '$1' + - regex: '; *(OnePAD[^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Airis' + model_replacement: '$1' + + ######### + # Airpad + # @ref: ?? + ######### + - regex: '; *Airpad[ \-]([^;/]+) Build' + device_replacement: 'Airpad $1' + brand_replacement: 'Airpad' + model_replacement: '$1' + + ######### + # Alcatel - TCT + # @ref: http://www.alcatelonetouch.com/global-en/products/smartphones.html + ######### + - regex: '; *(one ?touch) (EVO7|T10|T20) Build' + device_replacement: 'Alcatel One Touch $2' + brand_replacement: 'Alcatel' + model_replacement: 'One Touch $2' + - regex: '; *(?:alcatel[ _])?(?:(?:one[ _]?touch[ _])|ot[ \-])([^;/]+);? Build' + regex_flag: 'i' + device_replacement: 'Alcatel One Touch $1' + brand_replacement: 'Alcatel' + model_replacement: 'One Touch $1' + - regex: '; *(TCL)[ _]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + # operator specific models + - regex: '; *(Vodafone Smart II|Optimus_Madrid) Build' + device_replacement: 'Alcatel $1' + brand_replacement: 'Alcatel' + model_replacement: '$1' + - regex: '; *BASE_Lutea_3 Build' + device_replacement: 'Alcatel One Touch 998' + brand_replacement: 'Alcatel' + model_replacement: 'One Touch 998' + - regex: '; *BASE_Varia Build' + device_replacement: 'Alcatel One Touch 918D' + brand_replacement: 'Alcatel' + model_replacement: 'One Touch 918D' + + ######### + # Allfine + # @ref: http://www.myallfine.com/Products.asp + ######### + - regex: '; *((?:FINE|Fine)\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Allfine' + model_replacement: '$1' + + ######### + # Allview + # @ref: http://www.allview.ro/produse/droseries/lista-tablete-pc/ + ######### + - regex: '; *(ALLVIEW[ _]?|Allview[ _]?)((?:Speed|SPEED).*) Build/' + device_replacement: '$1$2' + brand_replacement: 'Allview' + model_replacement: '$2' + - regex: '; *(ALLVIEW[ _]?|Allview[ _]?)?(AX1_Shine|AX2_Frenzy) Build' + device_replacement: '$1$2' + brand_replacement: 'Allview' + model_replacement: '$2' + - regex: '; *(ALLVIEW[ _]?|Allview[ _]?)([^;/]*) Build' + device_replacement: '$1$2' + brand_replacement: 'Allview' + model_replacement: '$2' + + ######### + # Allwinner + # @ref: http://www.allwinner.com/ + # @models: A31 (13.3"),A20,A10, + ######### + - regex: '; *(A13-MID) Build' + device_replacement: '$1' + brand_replacement: 'Allwinner' + model_replacement: '$1' + - regex: '; *(Allwinner)[ _\-]?([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Allwinner' + model_replacement: '$1' + + ######### + # Amaway + # @ref: http://www.amaway.cn/ + ######### + - regex: '; *(A651|A701B?|A702|A703|A705|A706|A707|A711|A712|A713|A717|A722|A785|A801|A802|A803|A901|A902|A1002|A1003|A1006|A1007|A9701|A9703|Q710|Q80) Build' + device_replacement: '$1' + brand_replacement: 'Amaway' + model_replacement: '$1' + + ######### + # Amoi + # @ref: http://www.amoi.com/en/prd/prd_index.jspx + ######### + - regex: '; *(?:AMOI|Amoi)[ _]([^;/]+) Build' + device_replacement: 'Amoi $1' + brand_replacement: 'Amoi' + model_replacement: '$1' + - regex: '^(?:AMOI|Amoi)[ _]([^;/]+) Linux' + device_replacement: 'Amoi $1' + brand_replacement: 'Amoi' + model_replacement: '$1' + + ######### + # Aoc + # @ref: http://latin.aoc.com/media_tablet + ######### + - regex: '; *(MW(?:0[789]|10)[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Aoc' + model_replacement: '$1' + + ######### + # Aoson + # @ref: http://www.luckystar.com.cn/en/mid.aspx?page=1 + # @ref: http://www.luckystar.com.cn/en/mobiletel.aspx?page=1 + # @note: brand owned by luckystar + ######### + - regex: '; *(G7|M1013|M1015G|M11[CG]?|M-?12[B]?|M15|M19[G]?|M30[ACQ]?|M31[GQ]|M32|M33[GQ]|M36|M37|M38|M701T|M710|M712B|M713|M715G|M716G|M71(?:G|GS|T)?|M72[T]?|M73[T]?|M75[GT]?|M77G|M79T|M7L|M7LN|M81|M810|M81T|M82|M92|M92KS|M92S|M717G|M721|M722G|M723|M725G|M739|M785|M791|M92SK|M93D) Build' + device_replacement: 'Aoson $1' + brand_replacement: 'Aoson' + model_replacement: '$1' + - regex: '; *Aoson ([^;/]+) Build' + regex_flag: 'i' + device_replacement: 'Aoson $1' + brand_replacement: 'Aoson' + model_replacement: '$1' + + ######### + # Apanda + # @ref: http://www.apanda.com.cn/ + ######### + - regex: '; *[Aa]panda[ _\-]([^;/]+) Build' + device_replacement: 'Apanda $1' + brand_replacement: 'Apanda' + model_replacement: '$1' + + ######### + # Archos + # @ref: http://www.archos.com/de/products/tablets.html + # @ref: http://www.archos.com/de/products/smartphones/index.html + ######### + - regex: '; *(?:ARCHOS|Archos) ?(GAMEPAD.*?)(?: Build|[;/\(\)\-])' + device_replacement: 'Archos $1' + brand_replacement: 'Archos' + model_replacement: '$1' + - regex: 'ARCHOS; GOGI; ([^;]+);' + device_replacement: 'Archos $1' + brand_replacement: 'Archos' + model_replacement: '$1' + - regex: '(?:ARCHOS|Archos)[ _]?(.*?)(?: Build|[;/\(\)\-]|$)' + device_replacement: 'Archos $1' + brand_replacement: 'Archos' + model_replacement: '$1' + - regex: '; *(AN(?:7|8|9|10|13)[A-Z0-9]{1,4}) Build' + device_replacement: 'Archos $1' + brand_replacement: 'Archos' + model_replacement: '$1' + - regex: '; *(A28|A32|A43|A70(?:BHT|CHT|HB|S|X)|A101(?:B|C|IT)|A7EB|A7EB-WK|101G9|80G9) Build' + device_replacement: 'Archos $1' + brand_replacement: 'Archos' + model_replacement: '$1' + + ######### + # A-rival + # @ref: http://www.a-rival.de/de/ + ######### + - regex: '; *(PAD-FMD[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Arival' + model_replacement: '$1' + - regex: '; *(BioniQ) ?([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Arival' + model_replacement: '$1 $2' + + ######### + # Arnova + # @ref: http://arnovatech.com/ + ######### + - regex: '; *(AN\d[^;/]+|ARCHM\d+) Build' + device_replacement: 'Arnova $1' + brand_replacement: 'Arnova' + model_replacement: '$1' + - regex: '; *(?:ARNOVA|Arnova) ?([^;/]+) Build' + device_replacement: 'Arnova $1' + brand_replacement: 'Arnova' + model_replacement: '$1' + + ######### + # Assistant + # @ref: http://www.assistant.ua + ######### + - regex: '; *(?:ASSISTANT )?(AP)-?([1789]\d{2}[A-Z]{0,2}|80104) Build' + device_replacement: 'Assistant $1-$2' + brand_replacement: 'Assistant' + model_replacement: '$1-$2' + + ######### + # Asus + # @ref: http://www.asus.com/uk/Tablets_Mobile/ + ######### + - regex: '; *(ME17\d[^;/]*|ME3\d{2}[^;/]+|K00[A-Z]|Nexus 10|Nexus 7(?: 2013)?|PadFone[^;/]*|Transformer[^;/]*|TF\d{3}[^;/]*|eeepc) Build' + device_replacement: 'Asus $1' + brand_replacement: 'Asus' + model_replacement: '$1' + - regex: '; *ASUS[ _]*([^;/]+) Build' + device_replacement: 'Asus $1' + brand_replacement: 'Asus' + model_replacement: '$1' + + ######### + # Garmin-Asus + ######### + - regex: '; *Garmin-Asus ([^;/]+) Build' + device_replacement: 'Garmin-Asus $1' + brand_replacement: 'Garmin-Asus' + model_replacement: '$1' + - regex: '; *(Garminfone) Build' + device_replacement: 'Garmin $1' + brand_replacement: 'Garmin-Asus' + model_replacement: '$1' + + ######### + # Attab + # @ref: http://www.theattab.com/ + ######### + - regex: '; (@TAB-[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Attab' + model_replacement: '$1' + + ######### + # Audiosonic + # @ref: ?? + # @note: Take care with Docomo T-01 Toshiba + ######### + - regex: '; *(T-(?:07|[^0]\d)[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Audiosonic' + model_replacement: '$1' + + ######### + # Axioo + # @ref: http://www.axiooworld.com/ww/index.php + ######### + - regex: '; *(?:Axioo[ _\-]([^;/]+)|(picopad)[ _\-]([^;/]+)) Build' + regex_flag: 'i' + device_replacement: 'Axioo $1$2 $3' + brand_replacement: 'Axioo' + model_replacement: '$1$2 $3' + + ######### + # Azend + # @ref: http://azendcorp.com/index.php/products/portable-electronics + ######### + - regex: '; *(V(?:100|700|800)[^;/]*) Build' + device_replacement: '$1' + brand_replacement: 'Azend' + model_replacement: '$1' + + ######### + # Bak + # @ref: http://www.bakinternational.com/produtos.php?cat=80 + ######### + - regex: '; *(IBAK\-[^;/]*) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Bak' + model_replacement: '$1' + + ######### + # Bedove + # @ref: http://www.bedove.com/product.html + # @models: HY6501|HY5001|X12|X21|I5 + ######### + - regex: '; *(HY5001|HY6501|X12|X21|I5) Build' + device_replacement: 'Bedove $1' + brand_replacement: 'Bedove' + model_replacement: '$1' + + ######### + # Benss + # @ref: http://www.benss.net/ + ######### + - regex: '; *(JC-[^;/]*) Build' + device_replacement: 'Benss $1' + brand_replacement: 'Benss' + model_replacement: '$1' + + ######### + # Blackberry + # @ref: http://uk.blackberry.com/ + # @note: Android Apps seams to be used here + ######### + - regex: '; *(BB) ([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Blackberry' + model_replacement: '$2' + + ######### + # Blackbird + # @ref: http://iblackbird.co.kr + ######### + - regex: '; *(BlackBird)[ _](I8.*) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + - regex: '; *(BlackBird)[ _](.*) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Blaupunkt + # @ref: http://www.blaupunkt.com + ######### + # Endeavour + - regex: '; *([0-9]+BP[EM][^;/]*|Endeavour[^;/]+) Build' + device_replacement: 'Blaupunkt $1' + brand_replacement: 'Blaupunkt' + model_replacement: '$1' + + ######### + # Blu + # @ref: http://bluproducts.com + ######### + - regex: '; *((?:BLU|Blu)[ _\-])([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Blu' + model_replacement: '$2' + # BMOBILE = operator branded device + - regex: '; *(?:BMOBILE )?(Blu|BLU|DASH [^;/]+|VIVO 4\.3|TANK 4\.5) Build' + device_replacement: '$1' + brand_replacement: 'Blu' + model_replacement: '$1' + + ######### + # Blusens + # @ref: http://www.blusens.com/es/?sg=1&sv=al&roc=1 + ######### + # tablet + - regex: '; *(TOUCH\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Blusens' + model_replacement: '$1' + + ######### + # Bmobile + # @ref: http://bmobile.eu.com/?categoria=smartphones-2 + # @note: Might collide with Maxx as AX is used also there. + ######### + # smartphone + - regex: '; *(AX5\d+) Build' + device_replacement: '$1' + brand_replacement: 'Bmobile' + model_replacement: '$1' + + ######### + # bq + # @ref: http://bqreaders.com + ######### + - regex: '; *([Bb]q) ([^;/]+);? Build' + device_replacement: '$1 $2' + brand_replacement: 'bq' + model_replacement: '$2' + - regex: '; *(Maxwell [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'bq' + model_replacement: '$1' + + ######### + # Braun Phototechnik + # @ref: http://www.braun-phototechnik.de/en/products/list/~pcat.250/Tablet-PC.html + ######### + - regex: '; *((?:B-Tab|B-TAB) ?\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Braun' + model_replacement: '$1' + + ######### + # Broncho + # @ref: http://www.broncho.cn/ + ######### + - regex: '; *(Broncho) ([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Captiva + # @ref: http://www.captiva-power.de + ######### + - regex: '; *CAPTIVA ([^;/]+) Build' + device_replacement: 'Captiva $1' + brand_replacement: 'Captiva' + model_replacement: '$1' + + ######### + # Casio + # @ref: http://www.casiogzone.com/ + ######### + - regex: '; *(C771|CAL21|IS11CA) Build' + device_replacement: '$1' + brand_replacement: 'Casio' + model_replacement: '$1' + + ######### + # Cat + # @ref: http://www.cat-sound.com + ######### + - regex: '; *(?:Cat|CAT) ([^;/]+) Build' + device_replacement: 'Cat $1' + brand_replacement: 'Cat' + model_replacement: '$1' + - regex: '; *(?:Cat)(Nova.*) Build' + device_replacement: 'Cat $1' + brand_replacement: 'Cat' + model_replacement: '$1' + - regex: '; *(INM8002KP|ADM8000KP_[AB]) Build' + device_replacement: '$1' + brand_replacement: 'Cat' + model_replacement: 'Tablet PHOENIX 8.1J0' + + ######### + # Celkon + # @ref: http://www.celkonmobiles.com/?_a=products + # @models: A10, A19Q, A101, A105, A107, A107\+, A112, A118, A119, A119Q, A15, A19, A20, A200, A220, A225, A22 Race, A27, A58, A59, A60, A62, A63, A64, A66, A67, A69, A75, A77, A79, A8\+, A83, A85, A86, A87, A89 Ultima, A9\+, A90, A900, A95, A97i, A98, AR 40, AR 45, AR 50, ML5 + ######### + - regex: '; *(?:[Cc]elkon[ _\*]|CELKON[ _\*])([^;/\)]+) ?(?:Build|;|\))' + device_replacement: '$1' + brand_replacement: 'Celkon' + model_replacement: '$1' + - regex: 'Build/(?:[Cc]elkon)+_?([^;/_\)]+)' + device_replacement: '$1' + brand_replacement: 'Celkon' + model_replacement: '$1' + - regex: '; *(CT)-?(\d+) Build' + device_replacement: '$1$2' + brand_replacement: 'Celkon' + model_replacement: '$1$2' + # smartphones + - regex: '; *(A19|A19Q|A105|A107[^;/\)]*) ?(?:Build|;|\))' + device_replacement: '$1' + brand_replacement: 'Celkon' + model_replacement: '$1' + + ######### + # ChangJia + # @ref: http://www.cjshowroom.com/eproducts.aspx?classcode=004001001 + # @brief: China manufacturer makes tablets for different small brands + # (eg. http://www.zeepad.net/index.html) + ######### + - regex: '; *(TPC[0-9]{4,5}) Build' + device_replacement: '$1' + brand_replacement: 'ChangJia' + model_replacement: '$1' + + ######### + # Cloudfone + # @ref: http://www.cloudfonemobile.com/ + ######### + - regex: '; *(Cloudfone)[ _](Excite)([^ ][^;/]+) Build' + device_replacement: '$1 $2 $3' + brand_replacement: 'Cloudfone' + model_replacement: '$1 $2 $3' + - regex: '; *(Excite|ICE)[ _](\d+[^;/]+) Build' + device_replacement: 'Cloudfone $1 $2' + brand_replacement: 'Cloudfone' + model_replacement: 'Cloudfone $1 $2' + - regex: '; *(Cloudfone|CloudPad)[ _]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Cloudfone' + model_replacement: '$1 $2' + + ######### + # Cmx + # @ref: http://cmx.at/de/ + ######### + - regex: '; *((?:Aquila|Clanga|Rapax)[^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Cmx' + model_replacement: '$1' + + ######### + # CobyKyros + # @ref: http://cobykyros.com + # @note: Be careful with MID\d{3} from MpMan or Manta + ######### + - regex: '; *(?:CFW-|Kyros )?(MID[0-9]{4}(?:[ABC]|SR|TV)?)(\(3G\)-4G| GB 8K| 3G| 8K| GB)? *(?:Build|[;\)])' + device_replacement: 'CobyKyros $1$2' + brand_replacement: 'CobyKyros' + model_replacement: '$1$2' + + ######### + # Coolpad + # @ref: ?? + ######### + - regex: '; *([^;/]*)Coolpad[ _]([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Coolpad' + model_replacement: '$1$2' + + ######### + # Cube + # @ref: http://www.cube-tablet.com/buy-products.html + ######### + - regex: '; *(CUBE[ _])?([KU][0-9]+ ?GT.*|A5300) Build' + regex_flag: 'i' + device_replacement: '$1$2' + brand_replacement: 'Cube' + model_replacement: '$2' + + ######### + # Cubot + # @ref: http://www.cubotmall.com/ + ######### + - regex: '; *CUBOT ([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Cubot' + model_replacement: '$1' + - regex: '; *(BOBBY) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Cubot' + model_replacement: '$1' + + ######### + # Danew + # @ref: http://www.danew.com/produits-tablette.php + ######### + - regex: '; *(Dslide [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Danew' + model_replacement: '$1' + + ######### + # Dell + # @ref: http://www.dell.com + # @ref: http://www.softbank.jp/mobile/support/product/101dl/ + # @ref: http://www.softbank.jp/mobile/support/product/001dl/ + # @ref: http://developer.emnet.ne.jp/android.html + # @ref: http://www.dell.com/in/p/mobile-xcd28/pd + # @ref: http://www.dell.com/in/p/mobile-xcd35/pd + ######### + - regex: '; *(XCD)[ _]?(28|35) Build' + device_replacement: 'Dell $1$2' + brand_replacement: 'Dell' + model_replacement: '$1$2' + - regex: '; *(001DL) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: 'Streak' + - regex: '; *(?:Dell|DELL) (Streak) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: 'Streak' + - regex: '; *(101DL|GS01|Streak Pro[^;/]*) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: 'Streak Pro' + - regex: '; *([Ss]treak ?7) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: 'Streak 7' + - regex: '; *(Mini-3iX) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + - regex: '; *(?:Dell|DELL)[ _](Aero|Venue|Thunder|Mini.*|Streak[ _]Pro) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + - regex: '; *Dell[ _]([^;/]+) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + - regex: '; *Dell ([^;/]+) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + + ######### + # Denver + # @ref: http://www.denver-electronics.com/tablets1/ + ######### + - regex: '; *(TA[CD]-\d+[^;/]*) Build' + device_replacement: '$1' + brand_replacement: 'Denver' + model_replacement: '$1' + + ######### + # Dex + # @ref: http://dex.ua/ + ######### + - regex: '; *(iP[789]\d{2}(?:-3G)?|IP10\d{2}(?:-8GB)?) Build' + device_replacement: '$1' + brand_replacement: 'Dex' + model_replacement: '$1' + + ######### + # DNS AirTab + # @ref: http://www.dns-shop.ru/ + ######### + - regex: '; *(AirTab)[ _\-]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'DNS' + model_replacement: '$1 $2' + + ######### + # Docomo (Operator Branded Device) + # @ref: http://www.ipentec.com/document/document.aspx?page=android-useragent + ######### + - regex: '; *(F\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Fujitsu' + model_replacement: '$1' + - regex: '; *(HT-03A) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'Magic' + - regex: '; *(HT\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(L\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'LG' + model_replacement: '$1' + - regex: '; *(N\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Nec' + model_replacement: '$1' + - regex: '; *(P\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Panasonic' + model_replacement: '$1' + - regex: '; *(SC\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: '; *(SH\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Sharp' + model_replacement: '$1' + - regex: '; *(SO\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'SonyEricsson' + model_replacement: '$1' + - regex: '; *(T\-0[12][^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Toshiba' + model_replacement: '$1' + + ######### + # DOOV + # @ref: http://www.doov.com.cn/ + ######### + - regex: '; *(DOOV)[ _]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'DOOV' + model_replacement: '$2' + + ######### + # Enot + # @ref: http://www.enot.ua/ + ######### + - regex: '; *(Enot|ENOT)[ -]?([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Enot' + model_replacement: '$2' + + ######### + # Evercoss + # @ref: http://evercoss.com/android/ + ######### + - regex: '; *[^;/]+ Build/(?:CROSS|Cross)+[ _\-]([^\)]+)' + device_replacement: 'CROSS $1' + brand_replacement: 'Evercoss' + model_replacement: 'Cross $1' + - regex: '; *(CROSS|Cross)[ _\-]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Evercoss' + model_replacement: 'Cross $2' + + ######### + # Explay + # @ref: http://explay.ru/ + ######### + - regex: '; *Explay[_ ](.+?)(?:[\)]| Build)' + device_replacement: '$1' + brand_replacement: 'Explay' + model_replacement: '$1' + + ######### + # Fly + # @ref: http://www.fly-phone.com/ + ######### + - regex: '; *(IQ.*) Build' + device_replacement: '$1' + brand_replacement: 'Fly' + model_replacement: '$1' + - regex: '; *(Fly|FLY)[ _](IQ[^;]+|F[34]\d+[^;]*);? Build' + device_replacement: '$1 $2' + brand_replacement: 'Fly' + model_replacement: '$2' + + ######### + # Fujitsu + # @ref: http://www.fujitsu.com/global/ + ######### + - regex: '; *(M532|Q572|FJL21) Build/' + device_replacement: '$1' + brand_replacement: 'Fujitsu' + model_replacement: '$1' + + ######### + # Galapad + # @ref: http://www.galapad.net/product.html + ######### + - regex: '; *(G1) Build' + device_replacement: '$1' + brand_replacement: 'Galapad' + model_replacement: '$1' + + ######### + # Geeksphone + # @ref: http://www.geeksphone.com/ + ######### + - regex: '; *(Geeksphone) ([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Gfive + # @ref: http://www.gfivemobile.com/en + ######### + #- regex: '; *(G\'?FIVE) ([^;/]+) Build' # there is a problem with python yaml parser here + - regex: '; *(G[^F]?FIVE) ([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Gfive' + model_replacement: '$2' + + ######### + # Gionee + # @ref: http://www.gionee.com/ + ######### + - regex: '; *(Gionee)[ _\-]([^;/]+)(?:/[^;/]+)? Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Gionee' + model_replacement: '$2' + - regex: '; *(GN\d+[A-Z]?|INFINITY_PASSION|Ctrl_V1) Build' + device_replacement: 'Gionee $1' + brand_replacement: 'Gionee' + model_replacement: '$1' + - regex: '; *(E3) Build/JOP40D' + device_replacement: 'Gionee $1' + brand_replacement: 'Gionee' + model_replacement: '$1' + + ######### + # GoClever + # @ref: http://www.goclever.com + ######### + - regex: '; *((?:FONE|QUANTUM|INSIGNIA) \d+[^;/]*|PLAYTAB) Build' + device_replacement: 'GoClever $1' + brand_replacement: 'GoClever' + model_replacement: '$1' + - regex: '; *GOCLEVER ([^;/]+) Build' + device_replacement: 'GoClever $1' + brand_replacement: 'GoClever' + model_replacement: '$1' + + ######### + # Google + # @ref: http://www.google.de/glass/start/ + ######### + - regex: '; *(Glass \d+) Build' + device_replacement: '$1' + brand_replacement: 'Google' + model_replacement: '$1' + + ######### + # Gigabyte + # @ref: http://gsmart.gigabytecm.com/en/ + ######### + - regex: '; *(GSmart)[ -]([^/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Gigabyte' + model_replacement: '$1 $2' + + ######### + # Freescale development boards + # @ref: http://www.freescale.com/webapp/sps/site/prod_summary.jsp?code=IMX53QSB + ######### + - regex: '; *(imx5[13]_[^/]+) Build' + device_replacement: 'Freescale $1' + brand_replacement: 'Freescale' + model_replacement: '$1' + + ######### + # Haier + # @ref: http://www.haier.com/ + # @ref: http://www.haier.com/de/produkte/tablet/ + ######### + - regex: '; *Haier[ _\-]([^/]+) Build' + device_replacement: 'Haier $1' + brand_replacement: 'Haier' + model_replacement: '$1' + - regex: '; *(PAD1016) Build' + device_replacement: 'Haipad $1' + brand_replacement: 'Haipad' + model_replacement: '$1' + + ######### + # Haipad + # @ref: http://www.haipad.net/ + # @models: V7P|M7SM7S|M9XM9X|M7XM7X|M9|M8|M7-M|M1002|M7|M701 + ######### + - regex: '; *(M701|M7|M8|M9) Build' + device_replacement: 'Haipad $1' + brand_replacement: 'Haipad' + model_replacement: '$1' + + ######### + # Hannspree + # @ref: http://www.hannspree.eu/ + # @models: SN10T1|SN10T2|SN70T31B|SN70T32W + ######### + - regex: '; *(SN\d+T[^;\)/]*)(?: Build|[;\)])' + device_replacement: 'Hannspree $1' + brand_replacement: 'Hannspree' + model_replacement: '$1' + + ######### + # HCLme + # @ref: http://www.hclmetablet.com/india/ + ######### + - regex: 'Build/HCL ME Tablet ([^;\)]+)[\);]' + device_replacement: 'HCLme $1' + brand_replacement: 'HCLme' + model_replacement: '$1' + - regex: '; *([^;\/]+) Build/HCL' + device_replacement: 'HCLme $1' + brand_replacement: 'HCLme' + model_replacement: '$1' + + ######### + # Hena + # @ref: http://www.henadigital.com/en/product/index.asp?id=6 + ######### + - regex: '; *(MID-?\d{4}C[EM]) Build' + device_replacement: 'Hena $1' + brand_replacement: 'Hena' + model_replacement: '$1' + + ######### + # Hisense + # @ref: http://www.hisense.com/ + ######### + - regex: '; *(EG\d{2,}|HS-[^;/]+|MIRA[^;/]+) Build' + device_replacement: 'Hisense $1' + brand_replacement: 'Hisense' + model_replacement: '$1' + - regex: '; *(andromax[^;/]+) Build' + regex_flag: 'i' + device_replacement: 'Hisense $1' + brand_replacement: 'Hisense' + model_replacement: '$1' + + ######### + # hitech + # @ref: http://www.hitech-mobiles.com/ + ######### + - regex: '; *(?:AMAZE[ _](S\d+)|(S\d+)[ _]AMAZE) Build' + device_replacement: 'AMAZE $1$2' + brand_replacement: 'hitech' + model_replacement: 'AMAZE $1$2' + + ######### + # HP + # @ref: http://www.hp.com/ + ######### + - regex: '; *(PlayBook) Build' + device_replacement: 'HP $1' + brand_replacement: 'HP' + model_replacement: '$1' + - regex: '; *HP ([^/]+) Build' + device_replacement: 'HP $1' + brand_replacement: 'HP' + model_replacement: '$1' + - regex: '; *([^/]+_tenderloin) Build' + device_replacement: 'HP TouchPad' + brand_replacement: 'HP' + model_replacement: 'TouchPad' + + ######### + # Huawei + # @ref: http://www.huaweidevice.com + # @note: Needs to be before HTC due to Desire HD Build on U8815 + ######### + - regex: '; *(HUAWEI |Huawei-)?([UY][^;/]+) Build/(?:Huawei|HUAWEI)([UY][^\);]+)\)' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *([^;/]+) Build[/ ]Huawei(MT1-U06|[A-Z]+\d+[^\);]+)[^\);]*\)' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *(S7|M860) Build' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: '$1' + - regex: '; *((?:HUAWEI|Huawei)[ \-]?)(MediaPad) Build' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *((?:HUAWEI[ _]?|Huawei[ _])?Ascend[ _])([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *((?:HUAWEI|Huawei)[ _\-]?)((?:G700-|MT-)[^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *((?:HUAWEI|Huawei)[ _\-]?)([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *(MediaPad[^;]+|SpringBoard) Build/Huawei' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: '$1' + - regex: '; *([^;]+) Build/Huawei' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: '$1' + - regex: '; *([Uu])([89]\d{3}) Build' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: 'U$2' + - regex: '; *(?:Ideos |IDEOS )(S7) Build' + device_replacement: 'Huawei Ideos$1' + brand_replacement: 'Huawei' + model_replacement: 'Ideos$1' + - regex: '; *(?:Ideos |IDEOS )([^;/]+\s*|\s*)Build' + device_replacement: 'Huawei Ideos$1' + brand_replacement: 'Huawei' + model_replacement: 'Ideos$1' + - regex: '; *(Orange Daytona|Pulse|Pulse Mini|Vodafone 858|C8500|C8600|C8650|C8660|Nexus 6P) Build' + device_replacement: 'Huawei $1' + brand_replacement: 'Huawei' + model_replacement: '$1' + + ######### + # HTC + # @ref: http://www.htc.com/www/products/ + # @ref: http://en.wikipedia.org/wiki/List_of_HTC_phones + ######### + + - regex: '; *HTC[ _]([^;]+); Windows Phone' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + + # Android HTC with Version Number matcher + # ; HTC_0P3Z11/1.12.161.3 Build + # ;HTC_A3335 V2.38.841.1 Build + - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[ _/]([^ _/]+))?(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))' + device_replacement: 'HTC $1 $2' + brand_replacement: 'HTC' + model_replacement: '$1 $2' + - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+))?)?(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))' + device_replacement: 'HTC $1 $2 $3' + brand_replacement: 'HTC' + model_replacement: '$1 $2 $3' + - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+))?)?)?(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))' + device_replacement: 'HTC $1 $2 $3 $4' + brand_replacement: 'HTC' + model_replacement: '$1 $2 $3 $4' + + # Android HTC without Version Number matcher + - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/;]+)(?: *Build|[;\)]| - )' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/]+)(?:[ _/]([^ _/;\)]+))?(?: *Build|[;\)]| - )' + device_replacement: 'HTC $1 $2' + brand_replacement: 'HTC' + model_replacement: '$1 $2' + - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/;\)]+))?)?(?: *Build|[;\)]| - )' + device_replacement: 'HTC $1 $2 $3' + brand_replacement: 'HTC' + model_replacement: '$1 $2 $3' + - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ /;]+))?)?)?(?: *Build|[;\)]| - )' + device_replacement: 'HTC $1 $2 $3 $4' + brand_replacement: 'HTC' + model_replacement: '$1 $2 $3 $4' + + # HTC Streaming Player + - regex: 'HTC Streaming Player [^\/]*/[^\/]*/ htc_([^/]+) /' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + # general matcher for anything else + - regex: '(?:[;,] *|^)(?:htccn_chs-)?HTC[ _-]?([^;]+?)(?: *Build|clay|Android|-?Mozilla| Opera| Profile| UNTRUSTED|[;/\(\)]|$)' + regex_flag: 'i' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + # Android matchers without HTC + - regex: '; *(A6277|ADR6200|ADR6300|ADR6350|ADR6400[A-Z]*|ADR6425[A-Z]*|APX515CKT|ARIA|Desire[^_ ]*|Dream|EndeavorU|Eris|Evo|Flyer|HD2|Hero|HERO200|Hero CDMA|HTL21|Incredible|Inspire[A-Z0-9]*|Legend|Liberty|Nexus ?(?:One|HD2)|One|One S C2|One[ _]?(?:S|V|X\+?)\w*|PC36100|PG06100|PG86100|S31HT|Sensation|Wildfire)(?: Build|[/;\(\)])' + regex_flag: 'i' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(ADR6200|ADR6400L|ADR6425LVW|Amaze|DesireS?|EndeavorU|Eris|EVO|Evo\d[A-Z]+|HD2|IncredibleS?|Inspire[A-Z0-9]*|Inspire[A-Z0-9]*|Sensation[A-Z0-9]*|Wildfire)[ _-](.+?)(?:[/;\)]|Build|MIUI|1\.0)' + regex_flag: 'i' + device_replacement: 'HTC $1 $2' + brand_replacement: 'HTC' + model_replacement: '$1 $2' + + ######### + # Hyundai + # @ref: http://www.hyundaitechnologies.com + ######### + - regex: '; *HYUNDAI (T\d[^/]*) Build' + device_replacement: 'Hyundai $1' + brand_replacement: 'Hyundai' + model_replacement: '$1' + - regex: '; *HYUNDAI ([^;/]+) Build' + device_replacement: 'Hyundai $1' + brand_replacement: 'Hyundai' + model_replacement: '$1' + # X900? http://www.amazon.com/Hyundai-X900-Retina-Android-Bluetooth/dp/B00AO07H3O + - regex: '; *(X700|Hold X|MB-6900) Build' + device_replacement: 'Hyundai $1' + brand_replacement: 'Hyundai' + model_replacement: '$1' + + ######### + # iBall + # @ref: http://www.iball.co.in/Category/Mobiles/22 + ######### + - regex: '; *(?:iBall[ _\-])?(Andi)[ _]?(\d[^;/]*) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'iBall' + model_replacement: '$1 $2' + - regex: '; *(IBall)(?:[ _]([^;/]+)|) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'iBall' + model_replacement: '$2' + + ######### + # IconBIT + # @ref: http://www.iconbit.com/catalog/tablets/ + ######### + - regex: '; *(NT-\d+[^ ;/]*|Net[Tt]AB [^;/]+|Mercury [A-Z]+|iconBIT)(?: S/N:[^;/]+)? Build' + device_replacement: '$1' + brand_replacement: 'IconBIT' + model_replacement: '$1' + + ######### + # IMO + # @ref: http://www.ponselimo.com/ + ######### + - regex: '; *(IMO)[ _]([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'IMO' + model_replacement: '$2' + + ######### + # i-mobile + # @ref: http://www.i-mobilephone.com/ + ######### + - regex: '; *i-?mobile[ _]([^/]+) Build/' + regex_flag: 'i' + device_replacement: 'i-mobile $1' + brand_replacement: 'imobile' + model_replacement: '$1' + - regex: '; *(i-(?:style|note)[^/]*) Build/' + regex_flag: 'i' + device_replacement: 'i-mobile $1' + brand_replacement: 'imobile' + model_replacement: '$1' + + ######### + # Impression + # @ref: http://impression.ua/planshetnye-kompyutery + ######### + - regex: '; *(ImPAD) ?(\d+(?:.)*) Build' + device_replacement: '$1 $2' + brand_replacement: 'Impression' + model_replacement: '$1 $2' + + ######### + # Infinix + # @ref: http://www.infinixmobility.com/index.html + ######### + - regex: '; *(Infinix)[ _]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Infinix' + model_replacement: '$2' + + ######### + # Informer + # @ref: ?? + ######### + - regex: '; *(Informer)[ \-]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Informer' + model_replacement: '$2' + + ######### + # Intenso + # @ref: http://www.intenso.de + # @models: 7":TAB 714,TAB 724;8":TAB 814,TAB 824;10":TAB 1004 + ######### + - regex: '; *(TAB) ?([78][12]4) Build' + device_replacement: 'Intenso $1' + brand_replacement: 'Intenso' + model_replacement: '$1 $2' + + ######### + # Intex + # @ref: http://intexmobile.in/index.aspx + # @note: Zync also offers a "Cloud Z5" device + ######### + # smartphones + - regex: '; *(?:Intex[ _])?(AQUA|Aqua)([ _\.\-])([^;/]+) *(?:Build|;)' + device_replacement: '$1$2$3' + brand_replacement: 'Intex' + model_replacement: '$1 $3' + # matches "INTEX CLOUD X1" + - regex: '; *(?:INTEX|Intex)(?:[_ ]([^\ _;/]+))(?:[_ ]([^\ _;/]+))? *(?:Build|;)' + device_replacement: '$1 $2' + brand_replacement: 'Intex' + model_replacement: '$1 $2' + # tablets + - regex: '; *([iI]Buddy)[ _]?(Connect)(?:_|\?_| )?([^;/]*) *(?:Build|;)' + device_replacement: '$1 $2 $3' + brand_replacement: 'Intex' + model_replacement: 'iBuddy $2 $3' + - regex: '; *(I-Buddy)[ _]([^;/]+) *(?:Build|;)' + device_replacement: '$1 $2' + brand_replacement: 'Intex' + model_replacement: 'iBuddy $2' + + ######### + # iOCEAN + # @ref: http://www.iocean.cc/ + ######### + - regex: '; *(iOCEAN) ([^/]+) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'iOCEAN' + model_replacement: '$2' + + ######### + # i.onik + # @ref: http://www.i-onik.de/ + ######### + - regex: '; *(TP\d+(?:\.\d+)?\-\d[^;/]+) Build' + device_replacement: 'ionik $1' + brand_replacement: 'ionik' + model_replacement: '$1' + + ######### + # IRU.ru + # @ref: http://www.iru.ru/catalog/soho/planetable/ + ######### + - regex: '; *(M702pro) Build' + device_replacement: '$1' + brand_replacement: 'Iru' + model_replacement: '$1' + + ######### + # Ivio + # @ref: http://www.ivio.com/mobile.php + # @models: DG80,DG20,DE38,DE88,MD70 + ######### + - regex: '; *(DE88Plus|MD70) Build' + device_replacement: '$1' + brand_replacement: 'Ivio' + model_replacement: '$1' + - regex: '; *IVIO[_\-]([^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Ivio' + model_replacement: '$1' + + ######### + # Jaytech + # @ref: http://www.jay-tech.de/jaytech/servlet/frontend/ + ######### + - regex: '; *(TPC-\d+|JAY-TECH) Build' + device_replacement: '$1' + brand_replacement: 'Jaytech' + model_replacement: '$1' + + ######### + # Jiayu + # @ref: http://www.ejiayu.com/en/Product.html + ######### + - regex: '; *(JY-[^;/]+|G[234]S?) Build' + device_replacement: '$1' + brand_replacement: 'Jiayu' + model_replacement: '$1' + + ######### + # JXD + # @ref: http://www.jxd.hk/ + ######### + - regex: '; *(JXD)[ _\-]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'JXD' + model_replacement: '$2' + + ######### + # Karbonn + # @ref: http://www.karbonnmobiles.com/products_tablet.php + ######### + - regex: '; *Karbonn[ _]?([^;/]+) *(?:Build|;)' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Karbonn' + model_replacement: '$1' + - regex: '; *([^;]+) Build/Karbonn' + device_replacement: '$1' + brand_replacement: 'Karbonn' + model_replacement: '$1' + - regex: '; *(A11|A39|A37|A34|ST8|ST10|ST7|Smart Tab3|Smart Tab2|Titanium S\d) +Build' + device_replacement: '$1' + brand_replacement: 'Karbonn' + model_replacement: '$1' + + ######### + # KDDI (Operator Branded Device) + # @ref: http://www.ipentec.com/document/document.aspx?page=android-useragent + ######### + - regex: '; *(IS01|IS03|IS05|IS\d{2}SH) Build' + device_replacement: '$1' + brand_replacement: 'Sharp' + model_replacement: '$1' + - regex: '; *(IS04) Build' + device_replacement: '$1' + brand_replacement: 'Regza' + model_replacement: '$1' + - regex: '; *(IS06|IS\d{2}PT) Build' + device_replacement: '$1' + brand_replacement: 'Pantech' + model_replacement: '$1' + - regex: '; *(IS11S) Build' + device_replacement: '$1' + brand_replacement: 'SonyEricsson' + model_replacement: 'Xperia Acro' + - regex: '; *(IS11CA) Build' + device_replacement: '$1' + brand_replacement: 'Casio' + model_replacement: 'GzOne $1' + - regex: '; *(IS11LG) Build' + device_replacement: '$1' + brand_replacement: 'LG' + model_replacement: 'Optimus X' + - regex: '; *(IS11N) Build' + device_replacement: '$1' + brand_replacement: 'Medias' + model_replacement: '$1' + - regex: '; *(IS11PT) Build' + device_replacement: '$1' + brand_replacement: 'Pantech' + model_replacement: 'MIRACH' + - regex: '; *(IS12F) Build' + device_replacement: '$1' + brand_replacement: 'Fujitsu' + model_replacement: 'Arrows ES' + # @ref: https://ja.wikipedia.org/wiki/IS12M + - regex: '; *(IS12M) Build' + device_replacement: '$1' + brand_replacement: 'Motorola' + model_replacement: 'XT909' + - regex: '; *(IS12S) Build' + device_replacement: '$1' + brand_replacement: 'SonyEricsson' + model_replacement: 'Xperia Acro HD' + - regex: '; *(ISW11F) Build' + device_replacement: '$1' + brand_replacement: 'Fujitsu' + model_replacement: 'Arrowz Z' + - regex: '; *(ISW11HT) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'EVO' + - regex: '; *(ISW11K) Build' + device_replacement: '$1' + brand_replacement: 'Kyocera' + model_replacement: 'DIGNO' + - regex: '; *(ISW11M) Build' + device_replacement: '$1' + brand_replacement: 'Motorola' + model_replacement: 'Photon' + - regex: '; *(ISW11SC) Build' + device_replacement: '$1' + brand_replacement: 'Samsung' + model_replacement: 'GALAXY S II WiMAX' + - regex: '; *(ISW12HT) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'EVO 3D' + - regex: '; *(ISW13HT) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'J' + - regex: '; *(ISW?[0-9]{2}[A-Z]{0,2}) Build' + device_replacement: '$1' + brand_replacement: 'KDDI' + model_replacement: '$1' + - regex: '; *(INFOBAR [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'KDDI' + model_replacement: '$1' + + ######### + # Kingcom + # @ref: http://www.e-kingcom.com + ######### + - regex: '; *(JOYPAD|Joypad)[ _]([^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: 'Kingcom' + model_replacement: '$1 $2' + + ######### + # Kobo + # @ref: https://en.wikipedia.org/wiki/Kobo_Inc. + # @ref: http://www.kobo.com/devices#tablets + ######### + - regex: '; *(Vox|VOX|Arc|K080) Build/' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Kobo' + model_replacement: '$1' + - regex: '\b(Kobo Touch)\b' + device_replacement: '$1' + brand_replacement: 'Kobo' + model_replacement: '$1' + + ######### + # K-Touch + # @ref: ?? + ######### + - regex: '; *(K-Touch)[ _]([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Ktouch' + model_replacement: '$2' + + ######### + # KT Tech + # @ref: http://www.kttech.co.kr + ######### + - regex: '; *((?:EV|KM)-S\d+[A-Z]?) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'KTtech' + model_replacement: '$1' + + ######### + # Kyocera + # @ref: http://www.android.com/devices/?country=all&m=kyocera + ######### + - regex: '; *(Zio|Hydro|Torque|Event|EVENT|Echo|Milano|Rise|URBANO PROGRESSO|WX04K|WX06K|WX10K|KYL21|101K|C5[12]\d{2}) Build/' + device_replacement: '$1' + brand_replacement: 'Kyocera' + model_replacement: '$1' + + ######### + # Lava + # @ref: http://www.lavamobiles.com/ + ######### + - regex: '; *(?:LAVA[ _])?IRIS[ _\-]?([^/;\)]+) *(?:;|\)|Build)' + regex_flag: 'i' + device_replacement: 'Iris $1' + brand_replacement: 'Lava' + model_replacement: 'Iris $1' + - regex: '; *LAVA[ _]([^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Lava' + model_replacement: '$1' + + ######### + # Lemon + # @ref: http://www.lemonmobiles.com/products.php?type=1 + ######### + - regex: '; *(?:(Aspire A1)|(?:LEMON|Lemon)[ _]([^;/]+))_? Build' + device_replacement: 'Lemon $1$2' + brand_replacement: 'Lemon' + model_replacement: '$1$2' + + ######### + # Lenco + # @ref: http://www.lenco.com/c/tablets/ + ######### + - regex: '; *(TAB-1012) Build/' + device_replacement: 'Lenco $1' + brand_replacement: 'Lenco' + model_replacement: '$1' + - regex: '; Lenco ([^;/]+) Build/' + device_replacement: 'Lenco $1' + brand_replacement: 'Lenco' + model_replacement: '$1' + + ######### + # Lenovo + # @ref: http://support.lenovo.com/en_GB/downloads/default.page?# + ######### + - regex: '; *(A1_07|A2107A-H|S2005A-H|S1-37AH0) Build' + device_replacement: '$1' + brand_replacement: 'Lenovo' + model_replacement: '$1' + - regex: '; *(Idea[Tp]ab)[ _]([^;/]+);? Build' + device_replacement: 'Lenovo $1 $2' + brand_replacement: 'Lenovo' + model_replacement: '$1 $2' + - regex: '; *(Idea(?:Tab|pad)) ?([^;/]+) Build' + device_replacement: 'Lenovo $1 $2' + brand_replacement: 'Lenovo' + model_replacement: '$1 $2' + - regex: '; *(ThinkPad) ?(Tablet) Build/' + device_replacement: 'Lenovo $1 $2' + brand_replacement: 'Lenovo' + model_replacement: '$1 $2' + - regex: '; *(?:LNV-)?(?:=?[Ll]enovo[ _\-]?|LENOVO[ _])+(.+?)(?:Build|[;/\)])' + device_replacement: 'Lenovo $1' + brand_replacement: 'Lenovo' + model_replacement: '$1' + - regex: '[;,] (?:Vodafone )?(SmartTab) ?(II) ?(\d+) Build/' + device_replacement: 'Lenovo $1 $2 $3' + brand_replacement: 'Lenovo' + model_replacement: '$1 $2 $3' + - regex: '; *(?:Ideapad )?K1 Build/' + device_replacement: 'Lenovo Ideapad K1' + brand_replacement: 'Lenovo' + model_replacement: 'Ideapad K1' + - regex: '; *(3GC101|3GW10[01]|A390) Build/' + device_replacement: '$1' + brand_replacement: 'Lenovo' + model_replacement: '$1' + - regex: '\b(?:Lenovo|LENOVO)+[ _\-]?([^,;:/ ]+)' + device_replacement: 'Lenovo $1' + brand_replacement: 'Lenovo' + model_replacement: '$1' + + ######### + # Lexibook + # @ref: http://www.lexibook.com/fr + ######### + - regex: '; *(MFC\d+)[A-Z]{2}([^;,/]*),? Build' + device_replacement: '$1$2' + brand_replacement: 'Lexibook' + model_replacement: '$1$2' + + ######### + # LG + # @ref: http://www.lg.com/uk/mobile + ######### + - regex: '; *(E[34][0-9]{2}|LS[6-8][0-9]{2}|VS[6-9][0-9]+[^;/]+|Nexus 4|Nexus 5X?|GT540f?|Optimus (?:2X|G|4X HD)|OptimusX4HD) *(?:Build|;)' + device_replacement: '$1' + brand_replacement: 'LG' + model_replacement: '$1' + - regex: '[;:] *(L-\d+[A-Z]|LGL\d+[A-Z]?)(?:/V\d+)? *(?:Build|[;\)])' + device_replacement: '$1' + brand_replacement: 'LG' + model_replacement: '$1' + - regex: '; *(LG-)([A-Z]{1,2}\d{2,}[^,;/\)\(]*?)(?:Build| V\d+|[,;/\)\(]|$)' + device_replacement: '$1$2' + brand_replacement: 'LG' + model_replacement: '$2' + - regex: '; *(LG[ \-]|LG)([^;/]+)[;/]? Build' + device_replacement: '$1$2' + brand_replacement: 'LG' + model_replacement: '$2' + - regex: '^(LG)-([^;/]+)/ Mozilla/.*; Android' + device_replacement: '$1 $2' + brand_replacement: 'LG' + model_replacement: '$2' + + ######### + # Malata + # @ref: http://www.malata.com/en/products.aspx?classid=680 + ######### + - regex: '; *((?:SMB|smb)[^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'Malata' + model_replacement: '$1' + - regex: '; *(?:Malata|MALATA) ([^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'Malata' + model_replacement: '$1' + + ######### + # Manta + # @ref: http://www.manta.com.pl/en + ######### + - regex: '; *(MS[45][0-9]{3}|MID0[568][NS]?|MID[1-9]|MID[78]0[1-9]|MID970[1-9]|MID100[1-9]) Build/' + device_replacement: '$1' + brand_replacement: 'Manta' + model_replacement: '$1' + + ######### + # Match + # @ref: http://www.match.net.cn/products.asp + ######### + - regex: '; *(M1052|M806|M9000|M9100|M9701|MID100|MID120|MID125|MID130|MID135|MID140|MID701|MID710|MID713|MID727|MID728|MID731|MID732|MID733|MID735|MID736|MID737|MID760|MID800|MID810|MID820|MID830|MID833|MID835|MID860|MID900|MID930|MID933|MID960|MID980) Build/' + device_replacement: '$1' + brand_replacement: 'Match' + model_replacement: '$1' + + ######### + # Maxx + # @ref: http://www.maxxmobile.in/ + # @models: Maxx MSD7-Play, Maxx MX245+ Trance, Maxx AX8 Race, Maxx MSD7 3G- AX50, Maxx Genx Droid 7 - AX40, Maxx AX5 Duo, + # Maxx AX3 Duo, Maxx AX3, Maxx AX8 Note II (Note 2), Maxx AX8 Note I, Maxx AX8, Maxx AX5 Plus, Maxx MSD7 Smarty, + # Maxx AX9Z Race, + # Maxx MT150, Maxx MQ601, Maxx M2020, Maxx Sleek MX463neo, Maxx MX525, Maxx MX192-Tune, Maxx Genx Droid 7 AX353, + # @note: Need more User-Agents!!! + ######### + - regex: '; *(GenxDroid7|MSD7.*|AX\d.*|Tab 701|Tab 722) Build/' + device_replacement: 'Maxx $1' + brand_replacement: 'Maxx' + model_replacement: '$1' + + ######### + # Mediacom + # @ref: http://www.mediacomeurope.it/ + ######### + - regex: '; *(M-PP[^;/]+|PhonePad ?\d{2,}[^;/]+) Build' + device_replacement: 'Mediacom $1' + brand_replacement: 'Mediacom' + model_replacement: '$1' + - regex: '; *(M-MP[^;/]+|SmartPad ?\d{2,}[^;/]+) Build' + device_replacement: 'Mediacom $1' + brand_replacement: 'Mediacom' + model_replacement: '$1' + + ######### + # Medion + # @ref: http://www.medion.com/en/ + ######### + - regex: '; *(?:MD_)?LIFETAB[ _]([^;/]+) Build' + regex_flag: 'i' + device_replacement: 'Medion Lifetab $1' + brand_replacement: 'Medion' + model_replacement: 'Lifetab $1' + - regex: '; *MEDION ([^;/]+) Build' + device_replacement: 'Medion $1' + brand_replacement: 'Medion' + model_replacement: '$1' + + ######### + # Meizu + # @ref: http://www.meizu.com + ######### + - regex: '; *(M030|M031|M035|M040|M065|m9) Build' + device_replacement: 'Meizu $1' + brand_replacement: 'Meizu' + model_replacement: '$1' + - regex: '; *(?:meizu_|MEIZU )(.+?) *(?:Build|[;\)])' + device_replacement: 'Meizu $1' + brand_replacement: 'Meizu' + model_replacement: '$1' + + ######### + # Micromax + # @ref: http://www.micromaxinfo.com + ######### + - regex: '; *(?:Micromax[ _](A111|A240)|(A111|A240)) Build' + regex_flag: 'i' + device_replacement: 'Micromax $1$2' + brand_replacement: 'Micromax' + model_replacement: '$1$2' + - regex: '; *Micromax[ _](A\d{2,3}[^;/]*) Build' + regex_flag: 'i' + device_replacement: 'Micromax $1' + brand_replacement: 'Micromax' + model_replacement: '$1' + # be carefull here with Acer e.g. A500 + - regex: '; *(A\d{2}|A[12]\d{2}|A90S|A110Q) Build' + regex_flag: 'i' + device_replacement: 'Micromax $1' + brand_replacement: 'Micromax' + model_replacement: '$1' + - regex: '; *Micromax[ _](P\d{3}[^;/]*) Build' + regex_flag: 'i' + device_replacement: 'Micromax $1' + brand_replacement: 'Micromax' + model_replacement: '$1' + - regex: '; *(P\d{3}|P\d{3}\(Funbook\)) Build' + regex_flag: 'i' + device_replacement: 'Micromax $1' + brand_replacement: 'Micromax' + model_replacement: '$1' + + ######### + # Mito + # @ref: http://new.mitomobile.com/ + ######### + - regex: '; *(MITO)[ _\-]?([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Mito' + model_replacement: '$2' + + ######### + # Mobistel + # @ref: http://www.mobistel.com/ + ######### + - regex: '; *(Cynus)[ _](F5|T\d|.+?) *(?:Build|[;/\)])' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Mobistel' + model_replacement: '$1 $2' + + ######### + # Modecom + # @ref: http://www.modecom.eu/tablets/portal/ + ######### + - regex: '; *(MODECOM )?(FreeTab) ?([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1$2 $3' + brand_replacement: 'Modecom' + model_replacement: '$2 $3' + - regex: '; *(MODECOM )([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Modecom' + model_replacement: '$2' + + ######### + # Motorola + # @ref: http://www.motorola.com/us/shop-all-mobile-phones/ + ######### + - regex: '; *(MZ\d{3}\+?|MZ\d{3} 4G|Xoom|XOOM[^;/]*) Build' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: '$1' + - regex: '; *(Milestone )(XT[^;/]*) Build' + device_replacement: 'Motorola $1$2' + brand_replacement: 'Motorola' + model_replacement: '$2' + - regex: '; *(Motoroi ?x|Droid X|DROIDX) Build' + regex_flag: 'i' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: 'DROID X' + - regex: '; *(Droid[^;/]*|DROID[^;/]*|Milestone[^;/]*|Photon|Triumph|Devour|Titanium) Build' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: '$1' + - regex: '; *(A555|A85[34][^;/]*|A95[356]|ME[58]\d{2}\+?|ME600|ME632|ME722|MB\d{3}\+?|MT680|MT710|MT870|MT887|MT917|WX435|WX453|WX44[25]|XT\d{3,4}[A-Z\+]*|CL[iI]Q|CL[iI]Q XT) Build' + device_replacement: '$1' + brand_replacement: 'Motorola' + model_replacement: '$1' + - regex: '; *(Motorola MOT-|Motorola[ _\-]|MOT\-?)([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Motorola' + model_replacement: '$2' + - regex: '; *(Moto[_ ]?|MOT\-)([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Motorola' + model_replacement: '$2' + + ######### + # MpMan + # @ref: http://www.mpmaneurope.com + ######### + - regex: '; *((?:MP[DQ]C|MPG\d{1,4}|MP\d{3,4}|MID(?:(?:10[234]|114|43|7[247]|8[24]|7)C|8[01]1))[^;/]*) Build' + device_replacement: '$1' + brand_replacement: 'Mpman' + model_replacement: '$1' + + ######### + # MSI + # @ref: http://www.msi.com/product/windpad/ + ######### + - regex: '; *(?:MSI[ _])?(Primo\d+|Enjoy[ _\-][^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Msi' + model_replacement: '$1' + + ######### + # Multilaser + # http://www.multilaser.com.br/listagem_produtos.php?cat=5 + ######### + - regex: '; *Multilaser[ _]([^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Multilaser' + model_replacement: '$1' + + ######### + # MyPhone + # @ref: http://myphone.com.ph/ + ######### + - regex: '; *(My)[_]?(Pad)[ _]([^;/]+) Build' + device_replacement: '$1$2 $3' + brand_replacement: 'MyPhone' + model_replacement: '$1$2 $3' + - regex: '; *(My)\|?(Phone)[ _]([^;/]+) Build' + device_replacement: '$1$2 $3' + brand_replacement: 'MyPhone' + model_replacement: '$3' + - regex: '; *(A\d+)[ _](Duo)? Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'MyPhone' + model_replacement: '$1 $2' + + ######### + # Mytab + # @ref: http://www.mytab.eu/en/category/mytab-products/ + ######### + - regex: '; *(myTab[^;/]*) Build' + device_replacement: '$1' + brand_replacement: 'Mytab' + model_replacement: '$1' + + ######### + # Nabi + # @ref: https://www.nabitablet.com + ######### + - regex: '; *(NABI2?-)([^;/]+) Build/' + device_replacement: '$1$2' + brand_replacement: 'Nabi' + model_replacement: '$2' + + ######### + # Nec Medias + # @ref: http://www.n-keitai.com/ + ######### + - regex: '; *(N-\d+[CDE]) Build/' + device_replacement: '$1' + brand_replacement: 'Nec' + model_replacement: '$1' + - regex: '; ?(NEC-)(.*) Build/' + device_replacement: '$1$2' + brand_replacement: 'Nec' + model_replacement: '$2' + - regex: '; *(LT-NA7) Build/' + device_replacement: '$1' + brand_replacement: 'Nec' + model_replacement: 'Lifetouch Note' + + ######### + # Nextbook + # @ref: http://nextbookusa.com + ######### + - regex: '; *(NXM\d+[A-z0-9_]*|Next\d[A-z0-9_ \-]*|NEXT\d[A-z0-9_ \-]*|Nextbook [A-z0-9_ ]*|DATAM803HC|M805)(?: Build|[\);])' + device_replacement: '$1' + brand_replacement: 'Nextbook' + model_replacement: '$1' + + ######### + # Nokia + # @ref: http://www.nokia.com + ######### + - regex: '; *(Nokia)([ _\-]*)([^;/]*) Build' + regex_flag: 'i' + device_replacement: '$1$2$3' + brand_replacement: 'Nokia' + model_replacement: '$3' + + ######### + # Nook + # @ref: + # TODO nook browser/1.0 + ######### + - regex: '; *(Nook ?|Barnes & Noble Nook |BN )([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Nook' + model_replacement: '$2' + - regex: '; *(NOOK )?(BNRV200|BNRV200A|BNTV250|BNTV250A|BNTV400|BNTV600|LogicPD Zoom2) Build' + device_replacement: '$1$2' + brand_replacement: 'Nook' + model_replacement: '$2' + - regex: '; Build/(Nook)' + device_replacement: '$1' + brand_replacement: 'Nook' + model_replacement: 'Tablet' + + ######### + # Olivetti + # @ref: http://www.olivetti.de/EN/Page/t02/view_html?idp=348 + ######### + - regex: '; *(OP110|OliPad[^;/]+) Build' + device_replacement: 'Olivetti $1' + brand_replacement: 'Olivetti' + model_replacement: '$1' + + ######### + # Omega + # @ref: http://omega-technology.eu/en/produkty/346/tablets + # @note: MID tablets might get matched by CobyKyros first + # @models: (T107|MID(?:700[2-5]|7031|7108|7132|750[02]|8001|8500|9001|971[12]) + ######### + - regex: '; *OMEGA[ _\-](MID[^;/]+) Build' + device_replacement: 'Omega $1' + brand_replacement: 'Omega' + model_replacement: '$1' + - regex: '^(MID7500|MID\d+) Mozilla/5\.0 \(iPad;' + device_replacement: 'Omega $1' + brand_replacement: 'Omega' + model_replacement: '$1' + + ######### + # OpenPeak + # @ref: https://support.google.com/googleplay/answer/1727131?hl=en + ######### + - regex: '; *((?:CIUS|cius)[^;/]*) Build' + device_replacement: 'Openpeak $1' + brand_replacement: 'Openpeak' + model_replacement: '$1' + + ######### + # Oppo + # @ref: http://en.oppo.com/products/ + ######### + - regex: '; *(Find ?(?:5|7a)|R8[012]\d{1,2}|T703\d{0,1}|U70\d{1,2}T?|X90\d{1,2}) Build' + device_replacement: 'Oppo $1' + brand_replacement: 'Oppo' + model_replacement: '$1' + - regex: '; *OPPO ?([^;/]+) Build/' + device_replacement: 'Oppo $1' + brand_replacement: 'Oppo' + model_replacement: '$1' + + ######### + # Odys + # @ref: http://odys.de + ######### + - regex: '; *(?:Odys\-|ODYS\-|ODYS )([^;/]+) Build' + device_replacement: 'Odys $1' + brand_replacement: 'Odys' + model_replacement: '$1' + - regex: '; *(SELECT) ?(7) Build' + device_replacement: 'Odys $1 $2' + brand_replacement: 'Odys' + model_replacement: '$1 $2' + - regex: '; *(PEDI)_(PLUS)_(W) Build' + device_replacement: 'Odys $1 $2 $3' + brand_replacement: 'Odys' + model_replacement: '$1 $2 $3' + # Weltbild - Tablet PC 4 = Cat Phoenix = Odys Tablet PC 4? + - regex: '; *(AEON|BRAVIO|FUSION|FUSION2IN1|Genio|EOS10|IEOS[^;/]*|IRON|Loox|LOOX|LOOX Plus|Motion|NOON|NOON_PRO|NEXT|OPOS|PEDI[^;/]*|PRIME[^;/]*|STUDYTAB|TABLO|Tablet-PC-4|UNO_X8|XELIO[^;/]*|Xelio ?\d+ ?[Pp]ro|XENO10|XPRESS PRO) Build' + device_replacement: 'Odys $1' + brand_replacement: 'Odys' + model_replacement: '$1' + + ######### + # Orion + # @ref: http://www.orion.ua/en/products/computer-products/tablet-pcs.html + ######### + - regex: '; *(TP-\d+) Build/' + device_replacement: 'Orion $1' + brand_replacement: 'Orion' + model_replacement: '$1' + + ######### + # PackardBell + # @ref: http://www.packardbell.com/pb/en/AE/content/productgroup/tablets + ######### + - regex: '; *(G100W?) Build/' + device_replacement: 'PackardBell $1' + brand_replacement: 'PackardBell' + model_replacement: '$1' + + ######### + # Panasonic + # @ref: http://panasonic.jp/mobile/ + # @models: T11, T21, T31, P11, P51, Eluga Power, Eluga DL1 + # @models: (tab) Toughpad FZ-A1, Toughpad JT-B1 + ######### + - regex: '; *(Panasonic)[_ ]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + # Toughpad + - regex: '; *(FZ-A1B|JT-B1) Build' + device_replacement: 'Panasonic $1' + brand_replacement: 'Panasonic' + model_replacement: '$1' + # Eluga Power + - regex: '; *(dL1|DL1) Build' + device_replacement: 'Panasonic $1' + brand_replacement: 'Panasonic' + model_replacement: '$1' + + ######### + # Pantech + # @href: http://www.pantech.co.kr/en/prod/prodList.do?gbrand=PANTECH + # @href: http://www.pantech.co.kr/en/prod/prodList.do?gbrand=VEGA + # @models: ADR8995, ADR910L, ADR930VW, C790, CDM8992, CDM8999, IS06, IS11PT, P2000, P2020, P2030, P4100, P5000, P6010, P6020, P6030, P7000, P7040, P8000, P8010, P9020, P9050, P9060, P9070, P9090, PT001, PT002, PT003, TXT8040, TXT8045, VEGA PTL21 + ######### + - regex: '; *(SKY[ _])?(IM\-[AT]\d{3}[^;/]+).* Build/' + device_replacement: 'Pantech $1$2' + brand_replacement: 'Pantech' + model_replacement: '$1$2' + - regex: '; *((?:ADR8995|ADR910L|ADR930L|ADR930VW|PTL21|P8000)(?: 4G)?) Build/' + device_replacement: '$1' + brand_replacement: 'Pantech' + model_replacement: '$1' + - regex: '; *Pantech([^;/]+).* Build/' + device_replacement: 'Pantech $1' + brand_replacement: 'Pantech' + model_replacement: '$1' + + ######### + # Papayre + # @ref: http://grammata.es/ + ######### + - regex: '; *(papyre)[ _\-]([^;/]+) Build/' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Papyre' + model_replacement: '$2' + + ######### + # Pearl + # @ref: http://www.pearl.de/c-1540.shtml + ######### + - regex: '; *(?:Touchlet )?(X10\.[^;/]+) Build/' + device_replacement: 'Pearl $1' + brand_replacement: 'Pearl' + model_replacement: '$1' + + ######### + # Phicomm + # @ref: http://www.phicomm.com.cn/ + ######### + - regex: '; PHICOMM (i800) Build/' + device_replacement: 'Phicomm $1' + brand_replacement: 'Phicomm' + model_replacement: '$1' + - regex: '; PHICOMM ([^;/]+) Build/' + device_replacement: 'Phicomm $1' + brand_replacement: 'Phicomm' + model_replacement: '$1' + - regex: '; *(FWS\d{3}[^;/]+) Build/' + device_replacement: 'Phicomm $1' + brand_replacement: 'Phicomm' + model_replacement: '$1' + + ######### + # Philips + # @ref: http://www.support.philips.com/support/catalog/products.jsp?_dyncharset=UTF-8&country=&categoryid=MOBILE_PHONES_SMART_SU_CN_CARE&userLanguage=en&navCount=2&groupId=PC_PRODUCTS_AND_PHONES_GR_CN_CARE&catalogType=&navAction=push&userCountry=cn&title=Smartphones&cateId=MOBILE_PHONES_CA_CN_CARE + # @TODO: Philips Tablets User-Agents missing! + # @ref: http://www.support.philips.com/support/catalog/products.jsp?_dyncharset=UTF-8&country=&categoryid=ENTERTAINMENT_TABLETS_SU_CN_CARE&userLanguage=en&navCount=0&groupId=&catalogType=&navAction=push&userCountry=cn&title=Entertainment+Tablets&cateId=TABLETS_CA_CN_CARE + ######### + # @note: this a best guess according to available philips models. Need more User-Agents + - regex: '; *(D633|D822|D833|T539|T939|V726|W335|W336|W337|W3568|W536|W5510|W626|W632|W6350|W6360|W6500|W732|W736|W737|W7376|W820|W832|W8355|W8500|W8510|W930) Build' + device_replacement: '$1' + brand_replacement: 'Philips' + model_replacement: '$1' + - regex: '; *(?:Philips|PHILIPS)[ _]([^;/]+) Build' + device_replacement: 'Philips $1' + brand_replacement: 'Philips' + model_replacement: '$1' + + ######### + # Pipo + # @ref: http://www.pipo.cn/En/ + ######### + - regex: 'Android 4\..*; *(M[12356789]|U[12368]|S[123])\ ?(pro)? Build' + device_replacement: 'Pipo $1$2' + brand_replacement: 'Pipo' + model_replacement: '$1$2' + + ######### + # Ployer + # @ref: http://en.ployer.cn/ + ######### + - regex: '; *(MOMO[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Ployer' + model_replacement: '$1' + + ######### + # Polaroid/ Acho + # @ref: http://polaroidstore.com/store/start.asp?category_id=382&category_id2=0&order=title&filter1=&filter2=&filter3=&view=all + ######### + - regex: '; *(?:Polaroid[ _])?((?:MIDC\d{3,}|PMID\d{2,}|PTAB\d{3,})[^;/]*)(\/[^;/]*)? Build/' + device_replacement: '$1' + brand_replacement: 'Polaroid' + model_replacement: '$1' + - regex: '; *(?:Polaroid )(Tablet) Build/' + device_replacement: '$1' + brand_replacement: 'Polaroid' + model_replacement: '$1' + + ######### + # Pomp + # @ref: http://pompmobileshop.com/ + ######### + #~ TODO + - regex: '; *(POMP)[ _\-](.+?) *(?:Build|[;/\)])' + device_replacement: '$1 $2' + brand_replacement: 'Pomp' + model_replacement: '$2' + + ######### + # Positivo + # @ref: http://www.positivoinformatica.com.br/www/pessoal/tablet-ypy/ + ######### + - regex: '; *(TB07STA|TB10STA|TB07FTA|TB10FTA) Build/' + device_replacement: '$1' + brand_replacement: 'Positivo' + model_replacement: '$1' + - regex: '; *(?:Positivo )?((?:YPY|Ypy)[^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'Positivo' + model_replacement: '$1' + + ######### + # POV + # @ref: http://www.pointofview-online.com/default2.php + # @TODO: Smartphone Models MOB-3515, MOB-5045-B missing + ######### + - regex: '; *(MOB-[^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'POV' + model_replacement: '$1' + - regex: '; *POV[ _\-]([^;/]+) Build/' + device_replacement: 'POV $1' + brand_replacement: 'POV' + model_replacement: '$1' + - regex: '; *((?:TAB-PLAYTAB|TAB-PROTAB|PROTAB|PlayTabPro|Mobii[ _\-]|TAB-P)[^;/]*) Build/' + device_replacement: 'POV $1' + brand_replacement: 'POV' + model_replacement: '$1' + + ######### + # Prestigio + # @ref: http://www.prestigio.com/catalogue/MultiPhones + # @ref: http://www.prestigio.com/catalogue/MultiPads + ######### + - regex: '; *(?:Prestigio )?((?:PAP|PMP)\d[^;/]+) Build/' + device_replacement: 'Prestigio $1' + brand_replacement: 'Prestigio' + model_replacement: '$1' + + ######### + # Proscan + # @ref: http://www.proscanvideo.com/products-search.asp?itemClass=TABLET&itemnmbr= + ######### + - regex: '; *(PLT[0-9]{4}.*) Build/' + device_replacement: '$1' + brand_replacement: 'Proscan' + model_replacement: '$1' + + ######### + # QMobile + # @ref: http://www.qmobile.com.pk/ + ######### + - regex: '; *(A2|A5|A8|A900)_?(Classic)? Build' + device_replacement: '$1 $2' + brand_replacement: 'Qmobile' + model_replacement: '$1 $2' + - regex: '; *(Q[Mm]obile)_([^_]+)_([^_]+) Build' + device_replacement: 'Qmobile $2 $3' + brand_replacement: 'Qmobile' + model_replacement: '$2 $3' + - regex: '; *(Q\-?[Mm]obile)[_ ](A[^;/]+) Build' + device_replacement: 'Qmobile $2' + brand_replacement: 'Qmobile' + model_replacement: '$2' + + ######### + # Qmobilevn + # @ref: http://qmobile.vn/san-pham.html + ######### + - regex: '; *(Q\-Smart)[ _]([^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: 'Qmobilevn' + model_replacement: '$2' + - regex: '; *(Q\-?[Mm]obile)[ _\-](S[^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: 'Qmobilevn' + model_replacement: '$2' + + ######### + # Quanta + # @ref: ? + ######### + - regex: '; *(TA1013) Build' + device_replacement: '$1' + brand_replacement: 'Quanta' + model_replacement: '$1' + + ######### + # Rockchip + # @ref: http://www.rock-chips.com/a/cn/product/index.html + # @note: manufacturer sells chipsets - I assume that these UAs are dev-boards + ######### + - regex: '; *(RK\d+),? Build/' + device_replacement: '$1' + brand_replacement: 'Rockchip' + model_replacement: '$1' + - regex: ' Build/(RK\d+)' + device_replacement: '$1' + brand_replacement: 'Rockchip' + model_replacement: '$1' + + ######### + # Samsung Android Devices + # @ref: http://www.samsung.com/us/mobile/cell-phones/all-products + ######### + - regex: '; *(SAMSUNG |Samsung )?((?:Galaxy (?:Note II|S\d)|GT-I9082|GT-I9205|GT-N7\d{3}|SM-N9005)[^;/]*)\/?[^;/]* Build/' + device_replacement: 'Samsung $1$2' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '; *(Google )?(Nexus [Ss](?: 4G)?) Build/' + device_replacement: 'Samsung $1$2' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '; *(SAMSUNG |Samsung )([^\/]*)\/[^ ]* Build/' + device_replacement: 'Samsung $2' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '; *(Galaxy(?: Ace| Nexus| S ?II+|Nexus S| with MCR 1.2| Mini Plus 4G)?) Build/' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: '; *(SAMSUNG[ _\-] *)+([^;/]+) Build' + device_replacement: 'Samsung $2' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '; *(SAMSUNG-)?(GT\-[BINPS]\d{4}[^\/]*)(\/[^ ]*) Build' + device_replacement: 'Samsung $1$2$3' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '(?:; *|^)((?:GT\-[BIiNPS]\d{4}|I9\d{2}0[A-Za-z\+]?\b)[^;/\)]*?)(?:Build|Linux|MIUI|[;/\)])' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: '; (SAMSUNG-)([A-Za-z0-9\-]+).* Build/' + device_replacement: 'Samsung $1$2' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '; *((?:SCH|SGH|SHV|SHW|SPH|SC|SM)\-[A-Za-z0-9 ]+)(/?[^ ]*)? Build' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: ' ((?:SCH)\-[A-Za-z0-9 ]+)(/?[^ ]*)? Build' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: '; *(Behold ?(?:2|II)|YP\-G[^;/]+|EK-GC100|SCL21|I9300) Build' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + + ######### + # Sharp + # @ref: http://www.sharp-phone.com/en/index.html + # @ref: http://www.android.com/devices/?country=all&m=sharp + ######### + - regex: '; *(SH\-?\d\d[^;/]+|SBM\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Sharp' + model_replacement: '$1' + - regex: '; *(SHARP[ -])([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Sharp' + model_replacement: '$2' + + ######### + # Simvalley + # @ref: http://www.simvalley-mobile.de/ + ######### + - regex: '; *(SPX[_\-]\d[^;/]*) Build/' + device_replacement: '$1' + brand_replacement: 'Simvalley' + model_replacement: '$1' + - regex: '; *(SX7\-PEARL\.GmbH) Build/' + device_replacement: '$1' + brand_replacement: 'Simvalley' + model_replacement: '$1' + - regex: '; *(SP[T]?\-\d{2}[^;/]*) Build/' + device_replacement: '$1' + brand_replacement: 'Simvalley' + model_replacement: '$1' + + ######### + # SK Telesys + # @ref: http://www.sk-w.com/phone/phone_list.jsp + # @ref: http://www.android.com/devices/?country=all&m=sk-telesys + ######### + - regex: '; *(SK\-.*) Build/' + device_replacement: '$1' + brand_replacement: 'SKtelesys' + model_replacement: '$1' + + ######### + # Skytex + # @ref: http://skytex.com/android + ######### + - regex: '; *(?:SKYTEX|SX)-([^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Skytex' + model_replacement: '$1' + - regex: '; *(IMAGINE [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Skytex' + model_replacement: '$1' + + ######### + # SmartQ + # @ref: http://en.smartdevices.com.cn/Products/ + # @models: Z8, X7, U7H, U7, T30, T20, Ten3, V5-II, T7-3G, SmartQ5, K7, S7, Q8, T19, Ten2, Ten, R10, T7, R7, V5, V7, SmartQ7 + ######### + - regex: '; *(SmartQ) ?([^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Smartbitt + # @ref: http://www.smartbitt.com/ + # @missing: SBT Useragents + ######### + - regex: '; *(WF7C|WF10C|SBT[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Smartbitt' + model_replacement: '$1' + + ######### + # Softbank (Operator Branded Devices) + # @ref: http://www.ipentec.com/document/document.aspx?page=android-useragent + ######### + - regex: '; *(SBM(?:003SH|005SH|006SH|007SH|102SH)) Build' + device_replacement: '$1' + brand_replacement: 'Sharp' + model_replacement: '$1' + - regex: '; *(003P|101P|101P11C|102P) Build' + device_replacement: '$1' + brand_replacement: 'Panasonic' + model_replacement: '$1' + - regex: '; *(00\dZ) Build/' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + - regex: '; HTC(X06HT) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(001HT|X06HT) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(201M) Build' + device_replacement: '$1' + brand_replacement: 'Motorola' + model_replacement: 'XT902' + + ######### + # Trekstor + # @ref: http://www.trekstor.co.uk/surftabs-en.html + # @note: Must come before SonyEricsson + ######### + - regex: '; *(ST\d{4}.*)Build/ST' + device_replacement: 'Trekstor $1' + brand_replacement: 'Trekstor' + model_replacement: '$1' + - regex: '; *(ST\d{4}.*) Build/' + device_replacement: 'Trekstor $1' + brand_replacement: 'Trekstor' + model_replacement: '$1' + + ######### + # SonyEricsson + # @note: Must come before nokia since they also use symbian + # @ref: http://www.android.com/devices/?country=all&m=sony-ericssons + # @TODO: type! + ######### + # android matchers + - regex: '; *(Sony ?Ericsson ?)([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'SonyEricsson' + model_replacement: '$2' + - regex: '; *((?:SK|ST|E|X|LT|MK|MT|WT)\d{2}[a-z0-9]*(?:-o)?|R800i|U20i) Build' + device_replacement: '$1' + brand_replacement: 'SonyEricsson' + model_replacement: '$1' + # TODO X\d+ is wrong + - regex: '; *(Xperia (?:A8|Arc|Acro|Active|Live with Walkman|Mini|Neo|Play|Pro|Ray|X\d+)[^;/]*) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'SonyEricsson' + model_replacement: '$1' + + ######### + # Sony + # @ref: http://www.sonymobile.co.jp/index.html + # @ref: http://www.sonymobile.com/global-en/products/phones/ + # @ref: http://www.sony.jp/tablet/ + ######### + - regex: '; Sony (Tablet[^;/]+) Build' + device_replacement: 'Sony $1' + brand_replacement: 'Sony' + model_replacement: '$1' + - regex: '; Sony ([^;/]+) Build' + device_replacement: 'Sony $1' + brand_replacement: 'Sony' + model_replacement: '$1' + - regex: '; *(Sony)([A-Za-z0-9\-]+) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + - regex: '; *(Xperia [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1' + - regex: '; *(C(?:1[0-9]|2[0-9]|53|55|6[0-9])[0-9]{2}|D[25]\d{3}|D6[56]\d{2}) Build' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1' + - regex: '; *(SGP\d{3}|SGPT\d{2}) Build' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1' + - regex: '; *(NW-Z1000Series) Build' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1' + + ########## + # Sony PlayStation + # @ref: http://playstation.com + # The Vita spoofs the Kindle + ########## + - regex: 'PLAYSTATION 3' + device_replacement: 'PlayStation 3' + brand_replacement: 'Sony' + model_replacement: 'PlayStation 3' + - regex: '(PlayStation (?:Portable|Vita|\d+))' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1' + + ######### + # Spice + # @ref: http://www.spicemobilephones.co.in/ + ######### + - regex: '; *((?:CSL_Spice|Spice|SPICE|CSL)[ _\-]?)?([Mm][Ii])([ _\-])?(\d{3}[^;/]*) Build/' + device_replacement: '$1$2$3$4' + brand_replacement: 'Spice' + model_replacement: 'Mi$4' + + ######### + # Sprint (Operator Branded Devices) + # @ref: + ######### + - regex: '; *(Sprint )(.+?) *(?:Build|[;/])' + device_replacement: '$1$2' + brand_replacement: 'Sprint' + model_replacement: '$2' + - regex: '\b(Sprint)[: ]([^;,/ ]+)' + device_replacement: '$1$2' + brand_replacement: 'Sprint' + model_replacement: '$2' + + ######### + # Tagi + # @ref: ?? + ######### + - regex: '; *(TAGI[ ]?)(MID) ?([^;/]+) Build/' + device_replacement: '$1$2$3' + brand_replacement: 'Tagi' + model_replacement: '$2$3' + + ######### + # Tecmobile + # @ref: http://www.tecmobile.com/ + ######### + - regex: '; *(Oyster500|Opal 800) Build' + device_replacement: 'Tecmobile $1' + brand_replacement: 'Tecmobile' + model_replacement: '$1' + + ######### + # Tecno + # @ref: www.tecno-mobile.com/‎ + ######### + - regex: '; *(TECNO[ _])([^;/]+) Build/' + device_replacement: '$1$2' + brand_replacement: 'Tecno' + model_replacement: '$2' + + ######### + # Telechips, Techvision evaluation boards + # @ref: + ######### + - regex: '; *Android for (Telechips|Techvision) ([^ ]+) ' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Telstra + # @ref: http://www.telstra.com.au/home-phone/thub-2/ + # @ref: https://support.google.com/googleplay/answer/1727131?hl=en + ######### + - regex: '; *(T-Hub2) Build/' + device_replacement: '$1' + brand_replacement: 'Telstra' + model_replacement: '$1' + + ######### + # Terra + # @ref: http://www.wortmann.de/ + ######### + - regex: '; *(PAD) ?(100[12]) Build/' + device_replacement: 'Terra $1$2' + brand_replacement: 'Terra' + model_replacement: '$1$2' + + ######### + # Texet + # @ref: http://www.texet.ru/tablet/ + ######### + - regex: '; *(T[BM]-\d{3}[^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'Texet' + model_replacement: '$1' + + ######### + # Thalia + # @ref: http://www.thalia.de/shop/tolino-shine-ereader/show/ + ######### + - regex: '; *(tolino [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Thalia' + model_replacement: '$1' + - regex: '; *Build/.* (TOLINO_BROWSER)' + device_replacement: '$1' + brand_replacement: 'Thalia' + model_replacement: 'Tolino Shine' + + ######### + # Thl + # @ref: http://en.thl.com.cn/Mobile + # @ref: http://thlmobilestore.com + ######### + - regex: '; *(?:CJ[ -])?(ThL|THL)[ -]([^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: 'Thl' + model_replacement: '$2' + - regex: '; *(T100|T200|T5|W100|W200|W8s) Build/' + device_replacement: '$1' + brand_replacement: 'Thl' + model_replacement: '$1' + + ######### + # T-Mobile (Operator Branded Devices) + ######### + # @ref: https://en.wikipedia.org/wiki/HTC_Hero + - regex: '; *(T-Mobile[ _]G2[ _]Touch) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'Hero' + # @ref: https://en.wikipedia.org/wiki/HTC_Desire_Z + - regex: '; *(T-Mobile[ _]G2) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'Desire Z' + - regex: '; *(T-Mobile myTouch Q) Build' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: 'U8730' + - regex: '; *(T-Mobile myTouch) Build' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: 'U8680' + - regex: '; *(T-Mobile_Espresso) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'Espresso' + - regex: '; *(T-Mobile G1) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'Dream' + - regex: '\b(T-Mobile ?)?(myTouch)[ _]?([34]G)[ _]?([^\/]*) (?:Mozilla|Build)' + device_replacement: '$1$2 $3 $4' + brand_replacement: 'HTC' + model_replacement: '$2 $3 $4' + - regex: '\b(T-Mobile)_([^_]+)_(.*) Build' + device_replacement: '$1 $2 $3' + brand_replacement: 'Tmobile' + model_replacement: '$2 $3' + - regex: '\b(T-Mobile)[_ ]?(.*?)Build' + device_replacement: '$1 $2' + brand_replacement: 'Tmobile' + model_replacement: '$2' + + ######### + # Tomtec + # @ref: http://www.tom-tec.eu/pages/tablets.php + ######### + - regex: ' (ATP[0-9]{4}) Build' + device_replacement: '$1' + brand_replacement: 'Tomtec' + model_replacement: '$1' + + ######### + # Tooky + # @ref: http://www.tookymobile.com/ + ######### + - regex: ' *(TOOKY)[ _\-]([^;/]+) ?(?:Build|;)' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Tooky' + model_replacement: '$2' + + ######### + # Toshiba + # @ref: http://www.toshiba.co.jp/ + # @missing: LT170, Thrive 7, TOSHIBA STB10 + ######### + - regex: '\b(TOSHIBA_AC_AND_AZ|TOSHIBA_FOLIO_AND_A|FOLIO_AND_A)' + device_replacement: '$1' + brand_replacement: 'Toshiba' + model_replacement: 'Folio 100' + - regex: '; *([Ff]olio ?100) Build/' + device_replacement: '$1' + brand_replacement: 'Toshiba' + model_replacement: 'Folio 100' + - regex: '; *(AT[0-9]{2,3}(?:\-A|LE\-A|PE\-A|SE|a)?|AT7-A|AT1S0|Hikari-iFrame/WDPF-[^;/]+|THRiVE|Thrive) Build/' + device_replacement: 'Toshiba $1' + brand_replacement: 'Toshiba' + model_replacement: '$1' + + ######### + # Touchmate + # @ref: http://touchmatepc.com/new/ + ######### + - regex: '; *(TM-MID\d+[^;/]+|TOUCHMATE|MID-750) Build' + device_replacement: '$1' + brand_replacement: 'Touchmate' + model_replacement: '$1' + # @todo: needs verification user-agents missing + - regex: '; *(TM-SM\d+[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Touchmate' + model_replacement: '$1' + + ######### + # Treq + # @ref: http://www.treq.co.id/product + ######### + - regex: '; *(A10 [Bb]asic2?) Build/' + device_replacement: '$1' + brand_replacement: 'Treq' + model_replacement: '$1' + - regex: '; *(TREQ[ _\-])([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1$2' + brand_replacement: 'Treq' + model_replacement: '$2' + + ######### + # Umeox + # @ref: http://umeox.com/ + # @models: A936|A603|X-5|X-3 + ######### + # @todo: guessed markers + - regex: '; *(X-?5|X-?3) Build/' + device_replacement: '$1' + brand_replacement: 'Umeox' + model_replacement: '$1' + # @todo: guessed markers + - regex: '; *(A502\+?|A936|A603|X1|X2) Build/' + device_replacement: '$1' + brand_replacement: 'Umeox' + model_replacement: '$1' + + ######### + # Versus + # @ref: http://versusuk.com/support.html + ######### + - regex: '(TOUCH(?:TAB|PAD).+?) Build/' + regex_flag: 'i' + device_replacement: 'Versus $1' + brand_replacement: 'Versus' + model_replacement: '$1' + + ######### + # Vertu + # @ref: http://www.vertu.com/ + ######### + - regex: '(VERTU) ([^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: 'Vertu' + model_replacement: '$2' + + ######### + # Videocon + # @ref: http://www.videoconmobiles.com + ######### + - regex: '; *(Videocon)[ _\-]([^;/]+) *(?:Build|;)' + device_replacement: '$1 $2' + brand_replacement: 'Videocon' + model_replacement: '$2' + - regex: ' (VT\d{2}[A-Za-z]*) Build' + device_replacement: '$1' + brand_replacement: 'Videocon' + model_replacement: '$1' + + ######### + # Viewsonic + # @ref: http://viewsonic.com + ######### + - regex: '; *((?:ViewPad|ViewPhone|VSD)[^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'Viewsonic' + model_replacement: '$1' + - regex: '; *(ViewSonic-)([^;/]+) Build/' + device_replacement: '$1$2' + brand_replacement: 'Viewsonic' + model_replacement: '$2' + - regex: '; *(GTablet.*) Build/' + device_replacement: '$1' + brand_replacement: 'Viewsonic' + model_replacement: '$1' + + ######### + # vivo + # @ref: http://vivo.cn/ + ######### + - regex: '; *([Vv]ivo)[ _]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'vivo' + model_replacement: '$2' + + ######### + # Vodafone (Operator Branded Devices) + # @ref: ?? + ######### + - regex: '(Vodafone) (.*) Build/' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Walton + # @ref: http://www.waltonbd.com/ + ######### + - regex: '; *(?:Walton[ _\-])?(Primo[ _\-][^;/]+) Build' + regex_flag: 'i' + device_replacement: 'Walton $1' + brand_replacement: 'Walton' + model_replacement: '$1' + + ######### + # Wiko + # @ref: http://fr.wikomobile.com/collection.php?s=Smartphones + ######### + - regex: '; *(?:WIKO[ \-])?(CINK\+?|BARRY|BLOOM|DARKFULL|DARKMOON|DARKNIGHT|DARKSIDE|FIZZ|HIGHWAY|IGGY|OZZY|RAINBOW|STAIRWAY|SUBLIM|WAX|CINK [^;/]+) Build/' + regex_flag: 'i' + device_replacement: 'Wiko $1' + brand_replacement: 'Wiko' + model_replacement: '$1' + + ######### + # WellcoM + # @ref: ?? + ######### + - regex: '; *WellcoM-([^;/]+) Build' + device_replacement: 'Wellcom $1' + brand_replacement: 'Wellcom' + model_replacement: '$1' + + ########## + # WeTab + # @ref: http://wetab.mobi/ + ########## + - regex: '(?:(WeTab)-Browser|; (wetab) Build)' + device_replacement: '$1' + brand_replacement: 'WeTab' + model_replacement: 'WeTab' + + ######### + # Wolfgang + # @ref: http://wolfgangmobile.com/ + ######### + - regex: '; *(AT-AS[^;/]+) Build' + device_replacement: 'Wolfgang $1' + brand_replacement: 'Wolfgang' + model_replacement: '$1' + + ######### + # Woxter + # @ref: http://www.woxter.es/es-es/categories/index + ######### + - regex: '; *(?:Woxter|Wxt) ([^;/]+) Build' + device_replacement: 'Woxter $1' + brand_replacement: 'Woxter' + model_replacement: '$1' + + ######### + # Yarvik Zania + # @ref: http://yarvik.com + ######### + - regex: '; *(?:Xenta |Luna )?(TAB[234][0-9]{2}|TAB0[78]-\d{3}|TAB0?9-\d{3}|TAB1[03]-\d{3}|SMP\d{2}-\d{3}) Build/' + device_replacement: 'Yarvik $1' + brand_replacement: 'Yarvik' + model_replacement: '$1' + + ######### + # Yifang + # @note: Needs to be at the very last as manufacturer builds for other brands. + # @ref: http://www.yifangdigital.com/ + # @models: M1010, M1011, M1007, M1008, M1005, M899, M899LP, M909, M8000, + # M8001, M8002, M8003, M849, M815, M816, M819, M805, M878, M780LPW, + # M778, M7000, M7000AD, M7000NBD, M7001, M7002, M7002KBD, M777, M767, + # M789, M799, M769, M757, M755, M753, M752, M739, M729, M723, M712, M727 + ######### + - regex: '; *([A-Z]{2,4})(M\d{3,}[A-Z]{2})([^;\)\/]*)(?: Build|[;\)])' + device_replacement: 'Yifang $1$2$3' + brand_replacement: 'Yifang' + model_replacement: '$2' + + ######### + # XiaoMi + # @ref: http://www.xiaomi.com/event/buyphone + ######### + - regex: '; *((MI|HM|MI-ONE|Redmi)[ -](NOTE |Note )?[^;/]*) (Build|MIUI)/' + device_replacement: 'XiaoMi $1' + brand_replacement: 'XiaoMi' + model_replacement: '$1' + + ######### + # Xolo + # @ref: http://www.xolo.in/ + ######### + - regex: '; *XOLO[ _]([^;/]*tab.*) Build' + regex_flag: 'i' + device_replacement: 'Xolo $1' + brand_replacement: 'Xolo' + model_replacement: '$1' + - regex: '; *XOLO[ _]([^;/]+) Build' + regex_flag: 'i' + device_replacement: 'Xolo $1' + brand_replacement: 'Xolo' + model_replacement: '$1' + - regex: '; *(q\d0{2,3}[a-z]?) Build' + regex_flag: 'i' + device_replacement: 'Xolo $1' + brand_replacement: 'Xolo' + model_replacement: '$1' + + ######### + # Xoro + # @ref: http://www.xoro.de/produkte/ + ######### + - regex: '; *(PAD ?[79]\d+[^;/]*|TelePAD\d+[^;/]) Build' + device_replacement: 'Xoro $1' + brand_replacement: 'Xoro' + model_replacement: '$1' + + ######### + # Zopo + # @ref: http://www.zopomobiles.com/products.html + ######### + - regex: '; *(?:(?:ZOPO|Zopo)[ _]([^;/]+)|(ZP ?(?:\d{2}[^;/]+|C2))|(C[2379])) Build' + device_replacement: '$1$2$3' + brand_replacement: 'Zopo' + model_replacement: '$1$2$3' + + ######### + # ZiiLabs + # @ref: http://www.ziilabs.com/products/platforms/androidreferencetablets.php + ######### + - regex: '; *(ZiiLABS) (Zii[^;/]*) Build' + device_replacement: '$1 $2' + brand_replacement: 'ZiiLabs' + model_replacement: '$2' + - regex: '; *(Zii)_([^;/]*) Build' + device_replacement: '$1 $2' + brand_replacement: 'ZiiLabs' + model_replacement: '$2' + + ######### + # ZTE + # @ref: http://www.ztedevices.com/ + ######### + - regex: '; *(ARIZONA|(?:ATLAS|Atlas) W|D930|Grand (?:[SX][^;]*|Era|Memo[^;]*)|JOE|(?:Kis|KIS)\b[^;]*|Libra|Light [^;]*|N8[056][01]|N850L|N8000|N9[15]\d{2}|N9810|NX501|Optik|(?:Vip )Racer[^;]*|RacerII|RACERII|San Francisco[^;]*|V9[AC]|V55|V881|Z[679][0-9]{2}[A-z]?) Build' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + - regex: '; *([A-Z]\d+)_USA_[^;]* Build' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + - regex: '; *(SmartTab\d+)[^;]* Build' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + - regex: '; *(?:Blade|BLADE|ZTE-BLADE)([^;/]*) Build' + device_replacement: 'ZTE Blade$1' + brand_replacement: 'ZTE' + model_replacement: 'Blade$1' + - regex: '; *(?:Skate|SKATE|ZTE-SKATE)([^;/]*) Build' + device_replacement: 'ZTE Skate$1' + brand_replacement: 'ZTE' + model_replacement: 'Skate$1' + - regex: '; *(Orange |Optimus )(Monte Carlo|San Francisco) Build' + device_replacement: '$1$2' + brand_replacement: 'ZTE' + model_replacement: '$1$2' + - regex: '; *(?:ZXY-ZTE_|ZTE\-U |ZTE[\- _]|ZTE-C[_ ])([^;/]+) Build' + device_replacement: 'ZTE $1' + brand_replacement: 'ZTE' + model_replacement: '$1' + # operator specific + - regex: '; (BASE) (lutea|Lutea 2|Tab[^;]*) Build' + device_replacement: '$1 $2' + brand_replacement: 'ZTE' + model_replacement: '$1 $2' + - regex: '; (Avea inTouch 2|soft stone|tmn smart a7|Movistar[ _]Link) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + - regex: '; *(vp9plus)\)' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + + ########## + # Zync + # @ref: http://www.zync.in/index.php/our-products/tablet-phablets + ########## + - regex: '; ?(Cloud[ _]Z5|z1000|Z99 2G|z99|z930|z999|z990|z909|Z919|z900) Build/' + device_replacement: '$1' + brand_replacement: 'Zync' + model_replacement: '$1' + + ########## + # Kindle + # @note: Needs to be after Sony Playstation Vita as this UA contains Silk/3.2 + # @ref: https://developer.amazon.com/sdk/fire/specifications.html + # @ref: http://amazonsilk.wordpress.com/useful-bits/silk-user-agent/ + ########## + - regex: '; ?(KFOT|Kindle Fire) Build\b' + device_replacement: 'Kindle Fire' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire' + - regex: '; ?(KFOTE|Amazon Kindle Fire2) Build\b' + device_replacement: 'Kindle Fire 2' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire 2' + - regex: '; ?(KFTT) Build\b' + device_replacement: 'Kindle Fire HD' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HD 7"' + - regex: '; ?(KFJWI) Build\b' + device_replacement: 'Kindle Fire HD 8.9" WiFi' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HD 8.9" WiFi' + - regex: '; ?(KFJWA) Build\b' + device_replacement: 'Kindle Fire HD 8.9" 4G' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HD 8.9" 4G' + - regex: '; ?(KFSOWI) Build\b' + device_replacement: 'Kindle Fire HD 7" WiFi' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HD 7" WiFi' + - regex: '; ?(KFTHWI) Build\b' + device_replacement: 'Kindle Fire HDX 7" WiFi' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HDX 7" WiFi' + - regex: '; ?(KFTHWA) Build\b' + device_replacement: 'Kindle Fire HDX 7" 4G' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HDX 7" 4G' + - regex: '; ?(KFAPWI) Build\b' + device_replacement: 'Kindle Fire HDX 8.9" WiFi' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HDX 8.9" WiFi' + - regex: '; ?(KFAPWA) Build\b' + device_replacement: 'Kindle Fire HDX 8.9" 4G' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HDX 8.9" 4G' + - regex: '; ?Amazon ([^;/]+) Build\b' + device_replacement: '$1' + brand_replacement: 'Amazon' + model_replacement: '$1' + - regex: '; ?(Kindle) Build\b' + device_replacement: 'Kindle' + brand_replacement: 'Amazon' + model_replacement: 'Kindle' + - regex: '; ?(Silk)/(\d+)\.(\d+)(?:\.([0-9\-]+))? Build\b' + device_replacement: 'Kindle Fire' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire$2' + - regex: ' (Kindle)/(\d+\.\d+)' + device_replacement: 'Kindle' + brand_replacement: 'Amazon' + model_replacement: '$1 $2' + - regex: ' (Silk|Kindle)/(\d+)\.' + device_replacement: 'Kindle' + brand_replacement: 'Amazon' + model_replacement: 'Kindle' + + ######### + # Devices from chinese manufacturer(s) + # @note: identified by x-wap-profile http://218.249.47.94/Xianghe/.* + ######### + - regex: '(sprd)\-([^/]+)/' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + # @ref: http://eshinechina.en.alibaba.com/ + - regex: '; *(H\d{2}00\+?) Build' + device_replacement: '$1' + brand_replacement: 'Hero' + model_replacement: '$1' + - regex: '; *(iphone|iPhone5) Build/' + device_replacement: 'Xianghe $1' + brand_replacement: 'Xianghe' + model_replacement: '$1' + - regex: '; *(e\d{4}[a-z]?_?v\d+|v89_[^;/]+)[^;/]+ Build/' + device_replacement: 'Xianghe $1' + brand_replacement: 'Xianghe' + model_replacement: '$1' + + ######### + # Cellular + # @ref: + # @note: Operator branded devices + ######### + - regex: '\bUSCC[_\-]?([^ ;/\)]+)' + device_replacement: '$1' + brand_replacement: 'Cellular' + model_replacement: '$1' + + ###################################################################### + # Windows Phone Parsers + ###################################################################### + + ######### + # Alcatel Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:ALCATEL)[^;]*; *([^;,\)]+)' + device_replacement: 'Alcatel $1' + brand_replacement: 'Alcatel' + model_replacement: '$1' + + ######### + # Asus Windows Phones + ######### + #~ - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:ASUS|Asus)[^;]*; *([^;,\)]+)' + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:ASUS|Asus)[^;]*; *([^;,\)]+)' + device_replacement: 'Asus $1' + brand_replacement: 'Asus' + model_replacement: '$1' + + ######### + # Dell Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:DELL|Dell)[^;]*; *([^;,\)]+)' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + + ######### + # HTC Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:HTC|Htc|HTC_blocked[^;]*)[^;]*; *(?:HTC)?([^;,\)]+)' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + + ######### + # Huawei Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:HUAWEI)[^;]*; *(?:HUAWEI )?([^;,\)]+)' + device_replacement: 'Huawei $1' + brand_replacement: 'Huawei' + model_replacement: '$1' + + ######### + # LG Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:LG|Lg)[^;]*; *(?:LG[ \-])?([^;,\)]+)' + device_replacement: 'LG $1' + brand_replacement: 'LG' + model_replacement: '$1' + + ######### + # Noka Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:NOKIA|Nokia)[^;]*; *(?:NOKIA ?|Nokia ?|LUMIA ?|[Ll]umia ?)*(\d{3,}[^;\)]*)' + device_replacement: 'Lumia $1' + brand_replacement: 'Nokia' + model_replacement: 'Lumia $1' + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:NOKIA|Nokia)[^;]*; *(RM-\d{3,})' + device_replacement: 'Nokia $1' + brand_replacement: 'Nokia' + model_replacement: '$1' + - regex: '(?:Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)]|WPDesktop;) ?(?:ARM; ?Touch; ?|Touch; ?)?(?:NOKIA|Nokia)[^;]*; *(?:NOKIA ?|Nokia ?|LUMIA ?|[Ll]umia ?)*([^;\)]+)' + device_replacement: 'Nokia $1' + brand_replacement: 'Nokia' + model_replacement: '$1' + + ######### + # Microsoft Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:Microsoft(?: Corporation)?)[^;]*; *([^;,\)]+)' + device_replacement: 'Microsoft $1' + brand_replacement: 'Microsoft' + model_replacement: '$1' + + ######### + # Samsung Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:SAMSUNG)[^;]*; *(?:SAMSUNG )?([^;,\.\)]+)' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + + ######### + # Toshiba Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:TOSHIBA|FujitsuToshibaMobileCommun)[^;]*; *([^;,\)]+)' + device_replacement: 'Toshiba $1' + brand_replacement: 'Toshiba' + model_replacement: '$1' + + ######### + # Generic Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?([^;]+); *([^;,\)]+)' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ###################################################################### + # Other Devices Parser + ###################################################################### + + ######### + # Samsung Bada Phones + ######### + - regex: '(?:^|; )SAMSUNG\-([A-Za-z0-9\-]+).* Bada/' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + + ######### + # Firefox OS + ######### + - regex: '\(Mobile; ALCATEL ?(One|ONE) ?(Touch|TOUCH) ?([^;/]+)(?:/[^;]+)?; rv:[^\)]+\) Gecko/[^\/]+ Firefox/' + device_replacement: 'Alcatel $1 $2 $3' + brand_replacement: 'Alcatel' + model_replacement: 'One Touch $3' + - regex: '\(Mobile; (?:ZTE([^;]+)|(OpenC)); rv:[^\)]+\) Gecko/[^\/]+ Firefox/' + device_replacement: 'ZTE $1$2' + brand_replacement: 'ZTE' + model_replacement: '$1$2' + + ########## + # NOKIA + # @note: NokiaN8-00 comes before iphone. Sometimes spoofs iphone + ########## + - regex: 'Nokia(N[0-9]+)([A-z_\-][A-z0-9_\-]*)' + device_replacement: 'Nokia $1' + brand_replacement: 'Nokia' + model_replacement: '$1$2' + - regex: '(?:NOKIA|Nokia)(?:\-| *)(?:([A-Za-z0-9]+)\-[0-9a-f]{32}|([A-Za-z0-9\-]+)(?:UCBrowser)|([A-Za-z0-9\-]+))' + device_replacement: 'Nokia $1$2$3' + brand_replacement: 'Nokia' + model_replacement: '$1$2$3' + - regex: 'Lumia ([A-Za-z0-9\-]+)' + device_replacement: 'Lumia $1' + brand_replacement: 'Nokia' + model_replacement: 'Lumia $1' + # UCWEB Browser on Symbian + - regex: '\(Symbian; U; S60 V5; [A-z]{2}\-[A-z]{2}; (SonyEricsson|Samsung|Nokia|LG)([^;/]+)\)' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + # Nokia Symbian + - regex: '\(Symbian(?:/3)?; U; ([^;]+);' + device_replacement: 'Nokia $1' + brand_replacement: 'Nokia' + model_replacement: '$1' + + ########## + # BlackBerry + # @ref: http://www.useragentstring.com/pages/BlackBerry/ + ########## + - regex: 'BB10; ([A-Za-z0-9\- ]+)\)' + device_replacement: 'BlackBerry $1' + brand_replacement: 'BlackBerry' + model_replacement: '$1' + - regex: 'Play[Bb]ook.+RIM Tablet OS' + device_replacement: 'BlackBerry Playbook' + brand_replacement: 'BlackBerry' + model_replacement: 'Playbook' + - regex: 'Black[Bb]erry ([0-9]+);' + device_replacement: 'BlackBerry $1' + brand_replacement: 'BlackBerry' + model_replacement: '$1' + - regex: 'Black[Bb]erry([0-9]+)' + device_replacement: 'BlackBerry $1' + brand_replacement: 'BlackBerry' + model_replacement: '$1' + - regex: 'Black[Bb]erry;' + device_replacement: 'BlackBerry' + brand_replacement: 'BlackBerry' + + ########## + # PALM / HP + # @note: some palm devices must come before iphone. sometimes spoofs iphone in ua + ########## + - regex: '(Pre|Pixi)/\d+\.\d+' + device_replacement: 'Palm $1' + brand_replacement: 'Palm' + model_replacement: '$1' + - regex: 'Palm([0-9]+)' + device_replacement: 'Palm $1' + brand_replacement: 'Palm' + model_replacement: '$1' + - regex: 'Treo([A-Za-z0-9]+)' + device_replacement: 'Palm Treo $1' + brand_replacement: 'Palm' + model_replacement: 'Treo $1' + - regex: 'webOS.*(P160U(?:NA)?)/(\d+).(\d+)' + device_replacement: 'HP Veer' + brand_replacement: 'HP' + model_replacement: 'Veer' + - regex: '(Touch[Pp]ad)/\d+\.\d+' + device_replacement: 'HP TouchPad' + brand_replacement: 'HP' + model_replacement: 'TouchPad' + - regex: 'HPiPAQ([A-Za-z0-9]+)/\d+.\d+' + device_replacement: 'HP iPAQ $1' + brand_replacement: 'HP' + model_replacement: 'iPAQ $1' + - regex: 'PDA; (PalmOS)/sony/model ([a-z]+)/Revision' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1 $2' + + ########## + # AppleTV + # No built in browser that I can tell + # Stack Overflow indicated iTunes-AppleTV/4.1 as a known UA for app available and I'm seeing it in live traffic + ########## + - regex: '(Apple\s?TV)' + device_replacement: 'AppleTV' + brand_replacement: 'Apple' + model_replacement: 'AppleTV' + + ######### + # Tesla Model S + ######### + - regex: '(QtCarBrowser)' + device_replacement: 'Tesla Model S' + brand_replacement: 'Tesla' + model_replacement: 'Model S' + + ########## + # iSTUFF + # @note: complete but probably catches spoofs + # ipad and ipod must be parsed before iphone + # cannot determine specific device type from ua string. (3g, 3gs, 4, etc) + ########## + # @note: on some ua the device can be identified e.g. iPhone5,1 + - regex: '((?:iPhone|iPad|iPod)\d+,\d+)' + device_replacement: '$1' + brand_replacement: 'Apple' + model_replacement: '$1' + # @note: iPad needs to be before iPhone + - regex: '(iPad)(?:;| Simulator;)' + device_replacement: '$1' + brand_replacement: 'Apple' + model_replacement: '$1' + - regex: '(iPod)(?:;| touch;| Simulator;)' + device_replacement: '$1' + brand_replacement: 'Apple' + model_replacement: '$1' + - regex: '(iPhone)(?:;| Simulator;)' + device_replacement: '$1' + brand_replacement: 'Apple' + model_replacement: '$1' + # @note: desktop applications show device info + - regex: 'CFNetwork/.* Darwin/\d.*\(((?:Mac|iMac|PowerMac|PowerBook)[^\d]*)(\d+)(?:,|%2C)(\d+)' + device_replacement: '$1$2,$3' + brand_replacement: 'Apple' + model_replacement: '$1$2,$3' + # @note: iOS applications do not show device info + - regex: 'CFNetwork/.* Darwin/\d' + device_replacement: 'iOS-Device' + brand_replacement: 'Apple' + model_replacement: 'iOS-Device' + + ########## + # Acer + ########## + - regex: 'acer_([A-Za-z0-9]+)_' + device_replacement: 'Acer $1' + brand_replacement: 'Acer' + model_replacement: '$1' + + ########## + # Alcatel + ########## + - regex: '(?:ALCATEL|Alcatel)-([A-Za-z0-9\-]+)' + device_replacement: 'Alcatel $1' + brand_replacement: 'Alcatel' + model_replacement: '$1' + + ########## + # Amoi + ########## + - regex: '(?:Amoi|AMOI)\-([A-Za-z0-9]+)' + device_replacement: 'Amoi $1' + brand_replacement: 'Amoi' + model_replacement: '$1' + + ########## + # Asus + ########## + - regex: '(?:; |\/|^)((?:Transformer (?:Pad|Prime) |Transformer |PadFone[ _]?)[A-Za-z0-9]*)' + device_replacement: 'Asus $1' + brand_replacement: 'Asus' + model_replacement: '$1' + - regex: '(?:asus.*?ASUS|Asus|ASUS|asus)[\- ;]*((?:Transformer (?:Pad|Prime) |Transformer |Padfone |Nexus[ _])?[A-Za-z0-9]+)' + device_replacement: 'Asus $1' + brand_replacement: 'Asus' + model_replacement: '$1' + + + ########## + # Bird + ########## + - regex: '\bBIRD[ \-\.]([A-Za-z0-9]+)' + device_replacement: 'Bird $1' + brand_replacement: 'Bird' + model_replacement: '$1' + + ########## + # Dell + ########## + - regex: '\bDell ([A-Za-z0-9]+)' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + + ########## + # DoCoMo + ########## + - regex: 'DoCoMo/2\.0 ([A-Za-z0-9]+)' + device_replacement: 'DoCoMo $1' + brand_replacement: 'DoCoMo' + model_replacement: '$1' + - regex: '([A-Za-z0-9]+)_W;FOMA' + device_replacement: 'DoCoMo $1' + brand_replacement: 'DoCoMo' + model_replacement: '$1' + - regex: '([A-Za-z0-9]+);FOMA' + device_replacement: 'DoCoMo $1' + brand_replacement: 'DoCoMo' + model_replacement: '$1' + + ########## + # htc + ########## + - regex: '\b(?:HTC/|HTC/[a-z0-9]+/)?HTC[ _\-;]? *(.*?)(?:-?Mozilla|fingerPrint|[;/\(\)]|$)' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + + ########## + # Huawei + ########## + - regex: 'Huawei([A-Za-z0-9]+)' + device_replacement: 'Huawei $1' + brand_replacement: 'Huawei' + model_replacement: '$1' + - regex: 'HUAWEI-([A-Za-z0-9]+)' + device_replacement: 'Huawei $1' + brand_replacement: 'Huawei' + model_replacement: '$1' + - regex: 'vodafone([A-Za-z0-9]+)' + device_replacement: 'Huawei Vodafone $1' + brand_replacement: 'Huawei' + model_replacement: 'Vodafone $1' + + ########## + # i-mate + ########## + - regex: 'i\-mate ([A-Za-z0-9]+)' + device_replacement: 'i-mate $1' + brand_replacement: 'i-mate' + model_replacement: '$1' + + ########## + # kyocera + ########## + - regex: 'Kyocera\-([A-Za-z0-9]+)' + device_replacement: 'Kyocera $1' + brand_replacement: 'Kyocera' + model_replacement: '$1' + - regex: 'KWC\-([A-Za-z0-9]+)' + device_replacement: 'Kyocera $1' + brand_replacement: 'Kyocera' + model_replacement: '$1' + + ########## + # lenovo + ########## + - regex: 'Lenovo[_\-]([A-Za-z0-9]+)' + device_replacement: 'Lenovo $1' + brand_replacement: 'Lenovo' + model_replacement: '$1' + + ########## + # HbbTV (European and Australian standard) + # written before the LG regexes, as LG is making HbbTV too + ########## + - regex: '(HbbTV)/[0-9]+\.[0-9]+\.[0-9]+ \([^;]*; *(LG)E *; *([^;]*) *;[^;]*;[^;]*;\)' + device_replacement: '$1' + brand_replacement: '$2' + model_replacement: '$3' + - regex: '(HbbTV)/1\.1\.1.*CE-HTML/1\.\d;(Vendor/)*(THOM[^;]*?)[;\s](?:.*SW-Version/.*)*(LF[^;]+);?' + device_replacement: '$1' + brand_replacement: 'Thomson' + model_replacement: '$4' + - regex: '(HbbTV)(?:/1\.1\.1)?(?: ?\(;;;;;\))?; *CE-HTML(?:/1\.\d)?; *([^ ]+) ([^;]+);' + device_replacement: '$1' + brand_replacement: '$2' + model_replacement: '$3' + - regex: '(HbbTV)/1\.1\.1 \(;;;;;\) Maple_2011' + device_replacement: '$1' + brand_replacement: 'Samsung' + - regex: '(HbbTV)/[0-9]+\.[0-9]+\.[0-9]+ \([^;]*; *(?:CUS:([^;]*)|([^;]+)) *; *([^;]*) *;.*;' + device_replacement: '$1' + brand_replacement: '$2$3' + model_replacement: '$4' + - regex: '(HbbTV)/[0-9]+\.[0-9]+\.[0-9]+' + device_replacement: '$1' + + ########## + # LGE NetCast TV + ########## + - regex: 'LGE; (?:Media\/)?([^;]*);[^;]*;[^;]*;?\); "?LG NetCast(\.TV|\.Media|)-\d+' + device_replacement: 'NetCast$2' + brand_replacement: 'LG' + model_replacement: '$1' + + ########## + # InettvBrowser + ########## + - regex: 'InettvBrowser/[0-9]+\.[0-9A-Z]+ \([^;]*;(Sony)([^;]*);[^;]*;[^\)]*\)' + device_replacement: 'Inettv' + brand_replacement: '$1' + model_replacement: '$2' + - regex: 'InettvBrowser/[0-9]+\.[0-9A-Z]+ \([^;]*;([^;]*);[^;]*;[^\)]*\)' + device_replacement: 'Inettv' + brand_replacement: 'Generic_Inettv' + model_replacement: '$1' + - regex: '(?:InettvBrowser|TSBNetTV|NETTV|HBBTV)' + device_replacement: 'Inettv' + brand_replacement: 'Generic_Inettv' + + ########## + # lg + ########## + # LG Symbian Phones + - regex: 'Series60/\d\.\d (LG)[\-]?([A-Za-z0-9 \-]+)' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + # other LG phones + - regex: '\b(?:LGE[ \-]LG\-(?:AX)?|LGE |LGE?-LG|LGE?[ \-]|LG[ /\-]|lg[\-])([A-Za-z0-9]+)\b' + device_replacement: 'LG $1' + brand_replacement: 'LG' + model_replacement: '$1' + - regex: '(?:^LG[\-]?|^LGE[\-/]?)([A-Za-z]+[0-9]+[A-Za-z]*)' + device_replacement: 'LG $1' + brand_replacement: 'LG' + model_replacement: '$1' + - regex: '^LG([0-9]+[A-Za-z]*)' + device_replacement: 'LG $1' + brand_replacement: 'LG' + model_replacement: '$1' + + ########## + # microsoft + ########## + - regex: '(KIN\.[^ ]+) (\d+)\.(\d+)' + device_replacement: 'Microsoft $1' + brand_replacement: 'Microsoft' + model_replacement: '$1' + - regex: '(?:MSIE|XBMC).*\b(Xbox)\b' + device_replacement: '$1' + brand_replacement: 'Microsoft' + model_replacement: '$1' + - regex: '; ARM; Trident/6\.0; Touch[\);]' + device_replacement: 'Microsoft Surface RT' + brand_replacement: 'Microsoft' + model_replacement: 'Surface RT' + + ########## + # motorola + ########## + - regex: 'Motorola\-([A-Za-z0-9]+)' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: '$1' + - regex: 'MOTO\-([A-Za-z0-9]+)' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: '$1' + - regex: 'MOT\-([A-z0-9][A-z0-9\-]*)' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: '$1' + + ########## + # nintendo + ########## + - regex: 'Nintendo WiiU' + device_replacement: 'Nintendo Wii U' + brand_replacement: 'Nintendo' + model_replacement: 'Wii U' + - regex: 'Nintendo (DS|3DS|DSi|Wii);' + device_replacement: 'Nintendo $1' + brand_replacement: 'Nintendo' + model_replacement: '$1' + + ########## + # pantech + ########## + - regex: '(?:Pantech|PANTECH)[ _-]?([A-Za-z0-9\-]+)' + device_replacement: 'Pantech $1' + brand_replacement: 'Pantech' + model_replacement: '$1' + + ########## + # philips + ########## + - regex: 'Philips([A-Za-z0-9]+)' + device_replacement: 'Philips $1' + brand_replacement: 'Philips' + model_replacement: '$1' + - regex: 'Philips ([A-Za-z0-9]+)' + device_replacement: 'Philips $1' + brand_replacement: 'Philips' + model_replacement: '$1' + + ########## + # Samsung + ########## + # Samsung Symbian Devices + - regex: 'SymbianOS/9\.\d.* Samsung[/\-]([A-Za-z0-9 \-]+)' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: '(Samsung)(SGH)(i[0-9]+)' + device_replacement: '$1 $2$3' + brand_replacement: '$1' + model_replacement: '$2-$3' + - regex: 'SAMSUNG-ANDROID-MMS/([^;/]+)' + device_replacement: '$1' + brand_replacement: 'Samsung' + model_replacement: '$1' + # Other Samsung + #- regex: 'SAMSUNG(?:; |-)([A-Za-z0-9\-]+)' + - regex: 'SAMSUNG(?:; |[ -/])([A-Za-z0-9\-]+)' + regex_flag: 'i' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + + ########## + # Sega + ########## + - regex: '(Dreamcast)' + device_replacement: 'Sega $1' + brand_replacement: 'Sega' + model_replacement: '$1' + + ########## + # Siemens mobile + ########## + - regex: '^SIE-([A-Za-z0-9]+)' + device_replacement: 'Siemens $1' + brand_replacement: 'Siemens' + model_replacement: '$1' + + ########## + # Softbank + ########## + - regex: 'Softbank/[12]\.0/([A-Za-z0-9]+)' + device_replacement: 'Softbank $1' + brand_replacement: 'Softbank' + model_replacement: '$1' + + ########## + # SonyEricsson + ########## + - regex: 'SonyEricsson ?([A-Za-z0-9\-]+)' + device_replacement: 'Ericsson $1' + brand_replacement: 'SonyEricsson' + model_replacement: '$1' + + ########## + # Sony + ########## + - regex: 'Android [^;]+; ([^ ]+) (Sony)/' + device_replacement: '$2 $1' + brand_replacement: '$2' + model_replacement: '$1' + - regex: '(Sony)(?:BDP\/|\/)?([^ /;\)]+)[ /;\)]' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Puffin Browser Device detect + # A=Android, I=iOS, P=Phone, T=Tablet + # AT=Android+Tablet + ######### + - regex: 'Puffin/[\d\.]+IT' + device_replacement: 'iPad' + brand_replacement: 'Apple' + model_replacement: 'iPad' + - regex: 'Puffin/[\d\.]+IP' + device_replacement: 'iPhone' + brand_replacement: 'Apple' + model_replacement: 'iPhone' + - regex: 'Puffin/[\d\.]+AT' + device_replacement: 'Generic Tablet' + brand_replacement: 'Generic' + model_replacement: 'Tablet' + - regex: 'Puffin/[\d\.]+AP' + device_replacement: 'Generic Smartphone' + brand_replacement: 'Generic' + model_replacement: 'Smartphone' + + ######### + # Android General Device Matching (far from perfect) + ######### + - regex: 'Android[\- ][\d]+\.[\d]+; [A-Za-z]{2}\-[A-Za-z]{0,2}; WOWMobile (.+) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + - regex: 'Android[\- ][\d]+\.[\d]+\-update1; [A-Za-z]{2}\-[A-Za-z]{0,2} *; *(.+?) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *[A-Za-z]{2}[_\-][A-Za-z]{0,2}\-? *; *(.+?) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *[A-Za-z]{0,2}\- *; *(.+?) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + # No build info at all - "Build" follows locale immediately + - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *[a-z]{0,2}[_\-]?[A-Za-z]{0,2};? Build' + device_replacement: 'Generic Smartphone' + brand_replacement: 'Generic' + model_replacement: 'Smartphone' + - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *\-?[A-Za-z]{2}; *(.+?) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}(?:;.*)?; *(.+?) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + + ########## + # Google TV + ########## + - regex: '(GoogleTV)' + brand_replacement: 'Generic_Inettv' + model_replacement: '$1' + + ########## + # WebTV + ########## + - regex: '(WebTV)/\d+.\d+' + brand_replacement: 'Generic_Inettv' + model_replacement: '$1' + # Roku Digital-Video-Players https://www.roku.com/ + - regex: '^(Roku)/DVP-\d+\.\d+' + brand_replacement: 'Generic_Inettv' + model_replacement: '$1' + + ########## + # Generic Tablet + ########## + - regex: '(Android 3\.\d|Opera Tablet|Tablet; .+Firefox/|Android.*(?:Tab|Pad))' + regex_flag: 'i' + device_replacement: 'Generic Tablet' + brand_replacement: 'Generic' + model_replacement: 'Tablet' + + ########## + # Generic Smart Phone + ########## + - regex: '(Symbian|\bS60(Version|V\d)|\bS60\b|\((Series 60|Windows Mobile|Palm OS|Bada); Opera Mini|Windows CE|Opera Mobi|BREW|Brew|Mobile; .+Firefox/|iPhone OS|Android|MobileSafari|Windows *Phone|\(webOS/|PalmOS)' + device_replacement: 'Generic Smartphone' + brand_replacement: 'Generic' + model_replacement: 'Smartphone' + - regex: '(hiptop|avantgo|plucker|xiino|blazer|elaine)' + regex_flag: 'i' + device_replacement: 'Generic Smartphone' + brand_replacement: 'Generic' + model_replacement: 'Smartphone' + + ########## + # Spiders (this is hack...) + ########## + - regex: '(bot|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.*/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes)' + regex_flag: 'i' + device_replacement: 'Spider' + brand_replacement: 'Spider' + model_replacement: 'Desktop' + + ########## + # Generic Feature Phone + # take care to do case insensitive matching + ########## + - regex: '^(1207|3gso|4thp|501i|502i|503i|504i|505i|506i|6310|6590|770s|802s|a wa|acer|acs\-|airn|alav|asus|attw|au\-m|aur |aus |abac|acoo|aiko|alco|alca|amoi|anex|anny|anyw|aptu|arch|argo|bmobile|bell|bird|bw\-n|bw\-u|beck|benq|bilb|blac|c55/|cdm\-|chtm|capi|comp|cond|dall|dbte|dc\-s|dica|ds\-d|ds12|dait|devi|dmob|doco|dopo|dorado|el(?:38|39|48|49|50|55|58|68)|el[3456]\d{2}dual|erk0|esl8|ex300|ez40|ez60|ez70|ezos|ezze|elai|emul|eric|ezwa|fake|fly\-|fly_|g\-mo|g1 u|g560|gf\-5|grun|gene|go.w|good|grad|hcit|hd\-m|hd\-p|hd\-t|hei\-|hp i|hpip|hs\-c|htc |htc\-|htca|htcg)' + regex_flag: 'i' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' + - regex: '^(htcp|htcs|htct|htc_|haie|hita|huaw|hutc|i\-20|i\-go|i\-ma|i\-mobile|i230|iac|iac\-|iac/|ig01|im1k|inno|iris|jata|kddi|kgt|kgt/|kpt |kwc\-|klon|lexi|lg g|lg\-a|lg\-b|lg\-c|lg\-d|lg\-f|lg\-g|lg\-k|lg\-l|lg\-m|lg\-o|lg\-p|lg\-s|lg\-t|lg\-u|lg\-w|lg/k|lg/l|lg/u|lg50|lg54|lge\-|lge/|leno|m1\-w|m3ga|m50/|maui|mc01|mc21|mcca|medi|meri|mio8|mioa|mo01|mo02|mode|modo|mot |mot\-|mt50|mtp1|mtv |mate|maxo|merc|mits|mobi|motv|mozz|n100|n101|n102|n202|n203|n300|n302|n500|n502|n505|n700|n701|n710|nec\-|nem\-|newg|neon)' + regex_flag: 'i' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' + - regex: '^(netf|noki|nzph|o2 x|o2\-x|opwv|owg1|opti|oran|ot\-s|p800|pand|pg\-1|pg\-2|pg\-3|pg\-6|pg\-8|pg\-c|pg13|phil|pn\-2|pt\-g|palm|pana|pire|pock|pose|psio|qa\-a|qc\-2|qc\-3|qc\-5|qc\-7|qc07|qc12|qc21|qc32|qc60|qci\-|qwap|qtek|r380|r600|raks|rim9|rove|s55/|sage|sams|sc01|sch\-|scp\-|sdk/|se47|sec\-|sec0|sec1|semc|sgh\-|shar|sie\-|sk\-0|sl45|slid|smb3|smt5|sp01|sph\-|spv |spv\-|sy01|samm|sany|sava|scoo|send|siem|smar|smit|soft|sony|t\-mo|t218|t250|t600|t610|t618|tcl\-|tdg\-|telm|tim\-|ts70|tsm\-|tsm3|tsm5|tx\-9|tagt)' + regex_flag: 'i' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' + - regex: '^(talk|teli|topl|tosh|up.b|upg1|utst|v400|v750|veri|vk\-v|vk40|vk50|vk52|vk53|vm40|vx98|virg|vertu|vite|voda|vulc|w3c |w3c\-|wapj|wapp|wapu|wapm|wig |wapi|wapr|wapv|wapy|wapa|waps|wapt|winc|winw|wonu|x700|xda2|xdag|yas\-|your|zte\-|zeto|aste|audi|avan|blaz|brew|brvw|bumb|ccwa|cell|cldc|cmd\-|dang|eml2|fetc|hipt|http|ibro|idea|ikom|ipaq|jbro|jemu|jigs|keji|kyoc|kyok|libw|m\-cr|midp|mmef|moto|mwbp|mywa|newt|nok6|o2im|pant|pdxg|play|pluc|port|prox|rozo|sama|seri|smal|symb|treo|upsi|vx52|vx53|vx60|vx61|vx70|vx80|vx81|vx83|vx85|wap\-|webc|whit|wmlb|xda\-|xda_)' + regex_flag: 'i' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' + - regex: '^(Ice)$' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' + - regex: '(wap[\-\ ]browser|maui|netfront|obigo|teleca|up\.browser|midp|Opera Mini)' + regex_flag: 'i' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java new file mode 100644 index 00000000000..52a10840387 --- /dev/null +++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -0,0 +1,174 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.test.ESTestCase; +import org.junit.BeforeClass; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class UserAgentProcessorFactoryTests extends ESTestCase { + + private static Map userAgentParsers; + + private static String regexWithoutDevicesFilename = "regexes_without_devices.yaml"; + private static Path userAgentConfigDir; + + @BeforeClass + public static void createUserAgentParsers() throws IOException { + Path configDir = createTempDir(); + userAgentConfigDir = configDir.resolve("ingest-useragent"); + Files.createDirectories(userAgentConfigDir); + + // Copy file, leaving out the device parsers at the end + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(UserAgentProcessor.class.getResourceAsStream("/regexes.yaml"), StandardCharsets.UTF_8)); + BufferedWriter writer = Files.newBufferedWriter(userAgentConfigDir.resolve(regexWithoutDevicesFilename));) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("device_parsers:")) { + break; + } + + writer.write(line); + writer.newLine(); + } + } + + userAgentParsers = IngestUserAgentPlugin.createUserAgentParsers(userAgentConfigDir, new UserAgentCache(1000)); + } + + public void testBuildDefaults() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + + String processorTag = randomAsciiOfLength(10); + + UserAgentProcessor processor = factory.create(processorTag, config); + assertThat(processor.getTag(), equalTo(processorTag)); + assertThat(processor.getField(), equalTo("_field")); + assertThat(processor.getTargetField(), equalTo("useragent")); + assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0)); + assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0)); + assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0)); + assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class))); + } + + public void testBuildTargetField() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("target_field", "_target_field"); + + UserAgentProcessor processor = factory.create(null, config); + assertThat(processor.getField(), equalTo("_field")); + assertThat(processor.getTargetField(), equalTo("_target_field")); + } + + public void testBuildRegexFile() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("regex_file", regexWithoutDevicesFilename); + + UserAgentProcessor processor = factory.create(null, config); + assertThat(processor.getField(), equalTo("_field")); + assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0)); + assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0)); + assertThat(processor.getUaParser().getDevicePatterns().size(), equalTo(0)); + } + + public void testBuildNonExistingRegexFile() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("regex_file", "does-not-exist.yaml"); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config)); + assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yaml] doesn't exist (has to exist at node startup)")); + } + + public void testBuildFields() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Set properties = EnumSet.noneOf(UserAgentProcessor.Property.class); + List fieldNames = new ArrayList<>(); + int numFields = scaledRandomIntBetween(1, UserAgentProcessor.Property.values().length); + for (int i = 0; i < numFields; i++) { + UserAgentProcessor.Property property = UserAgentProcessor.Property.values()[i]; + properties.add(property); + fieldNames.add(property.name().toLowerCase(Locale.ROOT)); + } + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("properties", fieldNames); + + UserAgentProcessor processor = factory.create(null, config); + assertThat(processor.getField(), equalTo("_field")); + assertThat(processor.getProperties(), equalTo(properties)); + } + + public void testInvalidProperty() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("properties", Collections.singletonList("invalid")); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config)); + assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, " + + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]")); + } + + public void testInvalidPropertiesType() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("properties", "invalid"); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config)); + assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]")); + } +} diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java new file mode 100644 index 00000000000..d9b5eed059a --- /dev/null +++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -0,0 +1,161 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.ingest.RandomDocumentPicks; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.useragent.UserAgentProcessor; +import org.elasticsearch.test.ESTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.io.InputStream; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.is; + +public class UserAgentProcessorTests extends ESTestCase { + + private static UserAgentProcessor processor; + + @BeforeClass + public static void setupProcessor() throws IOException { + InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yaml"); + assertNotNull(regexStream); + + UserAgentParser parser = new UserAgentParser(randomAsciiOfLength(10), regexStream, new UserAgentCache(1000)); + + processor = new UserAgentProcessor(randomAsciiOfLength(10), "source_field", "target_field", parser, + EnumSet.allOf(UserAgentProcessor.Property.class)); + } + + @SuppressWarnings("unchecked") + public void testCommonBrowser() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Chrome")); + assertThat(target.get("major"), is("33")); + assertThat(target.get("minor"), is("0")); + assertThat(target.get("patch"), is("1750")); + assertNull(target.get("build")); + + assertThat(target.get("os"), is("Mac OS X 10.9.2")); + assertThat(target.get("os_name"), is("Mac OS X")); + assertThat(target.get("os_major"), is("10")); + assertThat(target.get("os_minor"), is("9")); + + assertThat(target.get("device"), is("Other")); + } + + @SuppressWarnings("unchecked") + public void testUncommonDevice() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10+ " + + "(KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Android")); + assertThat(target.get("major"), is("3")); + assertThat(target.get("minor"), is("0")); + assertNull(target.get("patch")); + assertNull(target.get("build")); + + assertThat(target.get("os"), is("Android 3.0")); + assertThat(target.get("os_name"), is("Android")); + assertThat(target.get("os_major"), is("3")); + assertThat(target.get("os_minor"), is("0")); + + assertThat(target.get("device"), is("Motorola Xoom")); + } + + @SuppressWarnings("unchecked") + public void testSpider() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("EasouSpider")); + assertNull(target.get("major")); + assertNull(target.get("minor")); + assertNull(target.get("patch")); + assertNull(target.get("build")); + + assertThat(target.get("os"), is("Other")); + assertThat(target.get("os_name"), is("Other")); + assertNull(target.get("os_major")); + assertNull(target.get("os_minor")); + + assertThat(target.get("device"), is("Spider")); + } + + @SuppressWarnings("unchecked") + public void testUnknown() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Something I made up v42.0.1"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Other")); + assertNull(target.get("major")); + assertNull(target.get("minor")); + assertNull(target.get("patch")); + assertNull(target.get("build")); + + assertThat(target.get("os"), is("Other")); + assertThat(target.get("os_name"), is("Other")); + assertNull(target.get("os_major")); + assertNull(target.get("os_minor")); + + assertThat(target.get("device"), is("Other")); + } +} + diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java new file mode 100644 index 00000000000..010f85f671a --- /dev/null +++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; + +import java.io.IOException; + +public class UserAgentRestIT extends ESRestTestCase { + + public UserAgentRestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ESRestTestCase.createParameters(0, 1); + } +} diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml new file mode 100644 index 00000000000..d37ad674cb9 --- /dev/null +++ b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml @@ -0,0 +1,11 @@ +"ingest-useragent plugin installed": + - do: + cluster.state: {} + + - set: {master_node: master} + + - do: + nodes.info: {} + + - match: { nodes.$master.plugins.0.name: ingest-useragent } + - match: { nodes.$master.ingest.processors.0.type: useragent } diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml new file mode 100644 index 00000000000..13451a075d7 --- /dev/null +++ b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml @@ -0,0 +1,86 @@ +--- +"Test user agent processor with defaults": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "useragent" : { + "field" : "field1" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.useragent.name: "Chrome" } + - match: { _source.useragent.os: "Mac OS X 10.9.2" } + - match: { _source.useragent.os_name: "Mac OS X" } + - match: { _source.useragent.os_major: "10" } + - match: { _source.useragent.os_minor: "9" } + - match: { _source.useragent.major: "33" } + - match: { _source.useragent.minor: "0" } + - match: { _source.useragent.patch: "1750" } + - match: { _source.useragent.device: "Other" } + +--- +"Test user agent processor with parameters": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "useragent" : { + "field" : "field1", + "target_field": "field2", + "properties": ["os"] + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.field2.os: "Mac OS X 10.9.2" } + - is_false: _source.useragent + - is_false: _source.field2.name + - is_false: _source.field2.os_name + - is_false: _source.field2.os_major + - is_false: _source.field2.os_minor + - is_false: _source.field2.major + - is_false: _source.field2.minor + - is_false: _source.field2.patch + - is_false: _source.field2.device diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml new file mode 100644 index 00000000000..3472cd03c3f --- /dev/null +++ b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml @@ -0,0 +1,42 @@ +--- +"Test user agent processor with custom regex file": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "useragent" : { + "field": "field1", + "regex_file": "test-regexes.yaml" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.useragent.name: "Test" } + - match: { _source.useragent.os: "Other" } + - match: { _source.useragent.os_name: "Other" } + - match: { _source.useragent.device: "Other" } + - is_false: _source.useragent.os_major + - is_false: _source.useragent.os_minor + - is_false: _source.useragent.major + - is_false: _source.useragent.minor + - is_false: _source.useragent.patch diff --git a/plugins/ingest-useragent/test/test-regexes.yaml b/plugins/ingest-useragent/test/test-regexes.yaml new file mode 100644 index 00000000000..e41dec700c0 --- /dev/null +++ b/plugins/ingest-useragent/test/test-regexes.yaml @@ -0,0 +1,3 @@ +user_agent_parsers: + - regex: '.*' + family_replacement: 'Test' \ No newline at end of file diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java index 75ae79d5f7d..b27c3fad2b8 100644 --- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java +++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.plugin.example; -import org.elasticsearch.client.Client; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Table; import org.elasticsearch.common.inject.Inject; @@ -40,7 +39,7 @@ public class ExampleCatAction extends AbstractCatAction { @Inject public ExampleCatAction(Settings settings, RestController controller, ExamplePluginConfiguration config) { - super(settings, controller); + super(settings); this.config = config; controller.registerHandler(GET, "/_cat/configured_example", this); } @@ -53,11 +52,12 @@ public class ExampleCatAction extends AbstractCatAction { table.endRow(); try { channel.sendResponse(RestTable.buildResponse(table, channel)); - } catch (Throwable e) { + } catch (Exception e) { try { channel.sendResponse(new BytesRestResponse(channel, e)); - } catch (Throwable e1) { - logger.error("failed to send failure response", e1); + } catch (Exception inner) { + inner.addSuppressed(e); + logger.error("failed to send failure response", inner); } } } diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java index c2f1214e72b..634a4ca6dfa 100644 --- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java +++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java @@ -35,9 +35,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.hamcrest.Matchers.equalTo; -/** - * - */ public class JavaScriptScriptMultiThreadedTests extends ESTestCase { public void testExecutableNoRuntimeParams() throws Exception { final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS); @@ -64,9 +61,9 @@ public class JavaScriptScriptMultiThreadedTests extends ESTestCase { long result = ((Number) script.run()).longValue(); assertThat(result, equalTo(addition)); } - } catch (Throwable t) { + } catch (Exception e) { failed.set(true); - logger.error("failed", t); + logger.error("failed", e); } finally { latch.countDown(); } @@ -106,9 +103,9 @@ public class JavaScriptScriptMultiThreadedTests extends ESTestCase { long result = ((Number) script.run()).longValue(); assertThat(result, equalTo(addition)); } - } catch (Throwable t) { + } catch (Exception e) { failed.set(true); - logger.error("failed", t); + logger.error("failed", e); } finally { latch.countDown(); } @@ -147,9 +144,9 @@ public class JavaScriptScriptMultiThreadedTests extends ESTestCase { long result = ((Number) se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testExecutableNoRuntimeParams", "js", compiled), runtimeVars).run()).longValue(); assertThat(result, equalTo(addition)); } - } catch (Throwable t) { + } catch (Exception e) { failed.set(true); - logger.error("failed", t); + logger.error("failed", e); } finally { latch.countDown(); } diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java index b642b7b0a73..5a16c06d4dc 100644 --- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java +++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java @@ -261,7 +261,7 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri } /** Evaluates with reduced privileges */ - private final PyObject evalRestricted(final PyCode code) { + private PyObject evalRestricted(final PyCode code) { // eval the script with reduced privileges return AccessController.doPrivileged(new PrivilegedAction() { @Override diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java index abf9f661a6d..0a887bc9a7e 100644 --- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java +++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java @@ -66,9 +66,9 @@ public class PythonScriptMultiThreadedTests extends ESTestCase { long result = ((Number) script.run()).longValue(); assertThat(result, equalTo(addition)); } - } catch (Throwable t) { + } catch (Exception e) { failed.set(true); - logger.error("failed", t); + logger.error("failed", e); } finally { latch.countDown(); } @@ -109,9 +109,9 @@ public class PythonScriptMultiThreadedTests extends ESTestCase { // long result = ((Number) script.run(runtimeVars)).longValue(); // assertThat(result, equalTo(addition)); // } -// } catch (Throwable t) { +// } catch (Exception e) { // failed.set(true); -// logger.error("failed", t); +// logger.error("failed", e); // } finally { // latch.countDown(); // } @@ -151,9 +151,9 @@ public class PythonScriptMultiThreadedTests extends ESTestCase { long result = ((Number) se.executable(compiledScript, runtimeVars).run()).longValue(); assertThat(result, equalTo(addition)); } - } catch (Throwable t) { + } catch (Exception e) { failed.set(true); - logger.error("failed", t); + logger.error("failed", e); } finally { latch.countDown(); } diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java index 79174e54c62..06e51686823 100644 --- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java +++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java @@ -482,7 +482,7 @@ public class AttachmentMapper extends FieldMapper { String parsedContent; try { parsedContent = TikaImpl.parse(content, metadata, indexedChars); - } catch (Throwable e) { + } catch (Exception e) { // #18: we could ignore errors when Tika does not parse data if (!ignoreErrors) { logger.trace("exception caught", e); @@ -508,8 +508,8 @@ public class AttachmentMapper extends FieldMapper { } context = context.createExternalValueContext(language); languageMapper.parse(context); - } catch(Throwable t) { - logger.debug("Cannot detect language: [{}]", t.getMessage()); + } catch(Exception e) { + logger.debug("Cannot detect language: [{}]", e.getMessage()); } } diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java index 8c86800e52b..6b80baa7c28 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java @@ -116,7 +116,7 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { .endObject() .endObject(); - byte[] mapping = mappingBuilder.bytes().toBytes(); + byte[] mapping = BytesReference.toBytes(mappingBuilder.bytes()); MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()); DocumentMapper docMapper = mapperService.parse("mail", new CompressedXContent(mapping), true); // this should not throw an exception diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java index fbbdeb83a7d..b32a6ab79a0 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java @@ -58,7 +58,7 @@ public class TikaDocTests extends ESTestCase { assertNotNull(parsedContent); assertFalse(parsedContent.isEmpty()); logger.debug("extracted content: {}", parsedContent); - } catch (Throwable e) { + } catch (Exception e) { throw new RuntimeException("parsing of filename: " + fileName.getFileName() + " failed", e); } } diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml index 458990cc90c..f180f51dfc5 100644 --- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml @@ -54,7 +54,7 @@ search: index: test body: - fields: [file.content_type,file.name] + stored_fields: [file.content_type,file.name] - match: { hits.total: 1 } - match: { hits.hits.0.fields: { file.content_type: ["text/my-dummy-content-type"], file.name: ["my-dummy-name-txt"] }} diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml index dc6f800c078..c2d42be3b9a 100644 --- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml @@ -57,7 +57,7 @@ setup: query: match: file.content: "apache tika" - fields: [] + stored_fields: [] highlight: fields: file.content: {} diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml index 69991b9d0c0..99f427c1ab9 100644 --- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml +++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml @@ -38,7 +38,7 @@ setup: search: index: test body: - fields: [file.content, file.author, file.date, file.content_length, file.content_type] + stored_fields: [file.content, file.author, file.date, file.content_length, file.content_type] - match: { hits.total: 1 } - match: { hits.hits.0.fields: { file.content: ["Test elasticsearch\n"], @@ -65,7 +65,7 @@ setup: search: index: test body: - fields: [file.content, file.author, file.date, file.content_length, file.content_type] + stored_fields: [file.content, file.author, file.date, file.content_length, file.content_type] - match: { hits.total: 1 } - match: { hits.hits.0.fields: { file.content: ["Test elasticsearch\n"], diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java index 761fb5fd144..956645aebd2 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java @@ -83,8 +83,8 @@ public class SizeFieldMapperUpgradeTests extends ESIntegTestCase { ElasticsearchAssertions.assertHitCount(countResponse, 3L); final SearchResponse sizeResponse = client().prepareSearch(indexName) - .addField("_source") - .addField("_size") + .addStoredField("_source") + .addStoredField("_size") .get(); ElasticsearchAssertions.assertHitCount(sizeResponse, 3L); for (SearchHit hit : sizeResponse.getHits().getHits()) { diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java index 2809b8588f1..64193e0b2f9 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.repositories.RepositoryName; import org.elasticsearch.repositories.RepositorySettings; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URISyntaxException; @@ -56,7 +57,8 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore { public AzureBlobStore(RepositoryName name, Settings settings, RepositorySettings repositorySettings, AzureStorageService client) throws URISyntaxException, StorageException { super(settings); - this.client = client.start(); + this.client = client; + client.start(); this.container = getValue(repositorySettings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING); this.repositoryName = name.getName(); this.accountName = getValue(repositorySettings, Repository.ACCOUNT_SETTING, Storage.ACCOUNT_SETTING); @@ -127,7 +129,7 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore { this.client.deleteBlob(this.accountName, this.locMode, container, blob); } - public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException + public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException, IOException { return this.client.getInputStream(this.accountName, this.locMode, container, blob); } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java index 13db36aeb5a..46972482a6a 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URISyntaxException; @@ -75,7 +76,7 @@ public interface AzureStorageService { void deleteBlob(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException; InputStream getInputStream(String account, LocationMode mode, String container, String blob) - throws URISyntaxException, StorageException; + throws URISyntaxException, StorageException, IOException; OutputStream getOutputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException; @@ -86,5 +87,5 @@ public interface AzureStorageService { void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException; - AzureStorageService start(); + void start(); } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java index 09dbc6520a1..80e62b90ad3 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java @@ -45,7 +45,7 @@ import java.net.URISyntaxException; import java.util.HashMap; import java.util.Map; -public class AzureStorageServiceImpl extends AbstractLifecycleComponent +public class AzureStorageServiceImpl extends AbstractLifecycleComponent implements AzureStorageService { final AzureStorageSettings primaryStorageSettings; diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index 4d3459cdcd4..1333c755e7c 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -65,7 +65,7 @@ public class AzureRepository extends BlobStoreRepository { private static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB); - public final static String TYPE = "azure"; + public static final String TYPE = "azure"; public static final class Repository { public static final Setting ACCOUNT_SETTING = Setting.simpleString("account", Property.NodeScope); diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java index 8160c560325..21df62cd768 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java @@ -31,6 +31,8 @@ import org.elasticsearch.common.settings.Settings; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URISyntaxException; @@ -41,7 +43,7 @@ import java.util.concurrent.ConcurrentHashMap; /** * In memory storage for unit tests */ -public class AzureStorageServiceMock extends AbstractLifecycleComponent +public class AzureStorageServiceMock extends AbstractLifecycleComponent implements AzureStorageService { protected Map blobs = new ConcurrentHashMap<>(); @@ -79,7 +81,10 @@ public class AzureStorageServiceMock extends AbstractLifecycleComponent listBlobsByPrefix(final @Nullable String prefix) throws IOException { + public Map listBlobsByPrefix(@Nullable final String prefix) throws IOException { FileStatus[] files = store.execute(new Operation() { @Override public FileStatus[] run(FileContext fileContext) throws IOException { diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index 15cd55c823d..752a2abc3ce 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -31,7 +31,7 @@ import java.util.function.Function; /** * */ -public interface AwsS3Service extends LifecycleComponent { +public interface AwsS3Service extends LifecycleComponent { // Global AWS settings (shared between discovery-ec2 and repository-s3) // Each setting starting with `cloud.aws` also exists in discovery-ec2 project. Don't forget to update diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 287973a3a90..1a0c2992b15 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -44,7 +44,7 @@ import java.util.Map; /** * */ -public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service { +public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service { /** * (acceskey, endpoint) -> client diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java index 7dc6f3b6a83..ea71dc152f9 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java @@ -46,9 +46,6 @@ import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.util.Map; -/** - * - */ public class S3BlobContainer extends AbstractBlobContainer { protected final S3BlobStore blobStore; @@ -74,7 +71,7 @@ public class S3BlobContainer extends AbstractBlobContainer { }); } catch (AmazonS3Exception e) { return false; - } catch (Throwable e) { + } catch (Exception e) { throw new BlobStoreException("failed to check if blob exists", e); } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index fe184a06aef..e3b7e8296e0 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -57,7 +57,7 @@ import java.util.function.Function; */ public class S3Repository extends BlobStoreRepository { - public final static String TYPE = "s3"; + public static final String TYPE = "s3"; /** * Global S3 repositories settings. Starting with: repositories.s3 diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index 2e196610c1a..d1c43f15adb 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -52,7 +52,7 @@ import static org.hamcrest.Matchers.notNullValue; /** */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0) -abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase { +public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase { @Override public Settings nodeSettings(int nodeOrdinal) { @@ -512,7 +512,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase multiObjectDeleteRequest.setKeys(keys); client.deleteObjects(multiObjectDeleteRequest); } - } catch (Throwable ex) { + } catch (Exception ex) { logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region); } } diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java index 9e29d6f091a..6016b0ca335 100644 --- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java +++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java @@ -28,7 +28,7 @@ import java.util.Collection; import static org.hamcrest.Matchers.is; -abstract public class AbstractAzureFsTestCase extends ESIntegTestCase { +public abstract class AbstractAzureFsTestCase extends ESIntegTestCase { @Override protected Collection> nodePlugins() { return pluginList(SMBStorePlugin.class); diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java index a319aaabb70..d028dfd573a 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java @@ -40,7 +40,7 @@ public class SeccompTests extends ESTestCase { if (!JNANatives.LOCAL_SECCOMP_ALL) { try { Seccomp.init(createTempDir()); - } catch (Throwable e) { + } catch (Exception e) { throw new RuntimeException("unable to forcefully apply seccomp to test thread", e); } } diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index e5117fa0aa0..af1f311dd23 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -26,7 +26,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.Terminal; -import org.elasticsearch.cli.UserError; +import org.elasticsearch.cli.UserException; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.PathUtils; @@ -320,7 +320,7 @@ public class InstallPluginCommandTests extends ESTestCase { public void testUnknownPlugin() throws Exception { Tuple env = createEnv(fs, temp); - UserError e = expectThrows(UserError.class, () -> installPlugin("foo", env.v1())); + UserException e = expectThrows(UserException.class, () -> installPlugin("foo", env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("Unknown plugin foo")); } @@ -350,7 +350,7 @@ public class InstallPluginCommandTests extends ESTestCase { Tuple env = createEnv(fs, temp); Path pluginDir = createPluginDir(temp); String pluginZip = createPlugin("lang-groovy", pluginDir); - UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1())); + UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("is a system module")); assertInstallCleaned(env.v2()); } @@ -385,7 +385,7 @@ public class InstallPluginCommandTests extends ESTestCase { Path pluginDir = createPluginDir(temp); String pluginZip = createPlugin("fake", pluginDir); installPlugin(pluginZip, env.v1()); - UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1())); + UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("already exists")); assertInstallCleaned(env.v2()); } @@ -407,7 +407,7 @@ public class InstallPluginCommandTests extends ESTestCase { Path binDir = pluginDir.resolve("bin"); Files.createFile(binDir); String pluginZip = createPlugin("fake", pluginDir); - UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1())); + UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("not a directory")); assertInstallCleaned(env.v2()); } @@ -419,7 +419,7 @@ public class InstallPluginCommandTests extends ESTestCase { Files.createDirectories(dirInBinDir); Files.createFile(dirInBinDir.resolve("somescript")); String pluginZip = createPlugin("fake", pluginDir); - UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1())); + UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin")); assertInstallCleaned(env.v2()); } @@ -490,7 +490,7 @@ public class InstallPluginCommandTests extends ESTestCase { Path configDir = pluginDir.resolve("config"); Files.createFile(configDir); String pluginZip = createPlugin("fake", pluginDir); - UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1())); + UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("not a directory")); assertInstallCleaned(env.v2()); } @@ -502,7 +502,7 @@ public class InstallPluginCommandTests extends ESTestCase { Files.createDirectories(dirInConfigDir); Files.createFile(dirInConfigDir.resolve("myconfig.yml")); String pluginZip = createPlugin("fake", pluginDir); - UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1())); + UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in config dir for plugin")); assertInstallCleaned(env.v2()); } @@ -534,7 +534,7 @@ public class InstallPluginCommandTests extends ESTestCase { Path pluginDir = createPluginDir(temp); Files.createFile(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES)); String pluginZip = writeZip(pluginDir, null); - UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1())); + UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); assertTrue(e.getMessage(), e.getMessage().contains("`elasticsearch` directory is missing in the plugin zip")); assertInstallCleaned(env.v2()); } @@ -580,16 +580,16 @@ public class InstallPluginCommandTests extends ESTestCase { public void testInstallMisspelledOfficialPlugins() throws Exception { Tuple env = createEnv(fs, temp); - UserError e = expectThrows(UserError.class, () -> installPlugin("xpack", env.v1())); + UserException e = expectThrows(UserException.class, () -> installPlugin("xpack", env.v1())); assertThat(e.getMessage(), containsString("Unknown plugin xpack, did you mean [x-pack]?")); - e = expectThrows(UserError.class, () -> installPlugin("analysis-smartnc", env.v1())); + e = expectThrows(UserException.class, () -> installPlugin("analysis-smartnc", env.v1())); assertThat(e.getMessage(), containsString("Unknown plugin analysis-smartnc, did you mean [analysis-smartcn]?")); - e = expectThrows(UserError.class, () -> installPlugin("repository", env.v1())); + e = expectThrows(UserException.class, () -> installPlugin("repository", env.v1())); assertThat(e.getMessage(), containsString("Unknown plugin repository, did you mean any of [repository-s3, repository-gcs]?")); - e = expectThrows(UserError.class, () -> installPlugin("unknown_plugin", env.v1())); + e = expectThrows(UserException.class, () -> installPlugin("unknown_plugin", env.v1())); assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin")); } diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java index 3a4639fa839..e2910be64f0 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java @@ -27,7 +27,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.cli.UserError; +import org.elasticsearch.cli.UserException; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -72,7 +72,7 @@ public class RemovePluginCommandTests extends ESTestCase { } public void testMissing() throws Exception { - UserError e = expectThrows(UserError.class, () -> removePlugin("dne", home)); + UserException e = expectThrows(UserException.class, () -> removePlugin("dne", home)); assertTrue(e.getMessage(), e.getMessage().contains("plugin dne not found")); assertRemoveCleaned(env); } @@ -102,7 +102,7 @@ public class RemovePluginCommandTests extends ESTestCase { public void testBinNotDir() throws Exception { Files.createDirectories(env.pluginsFile().resolve("elasticsearch")); - UserError e = expectThrows(UserError.class, () -> removePlugin("elasticsearch", home)); + UserException e = expectThrows(UserException.class, () -> removePlugin("elasticsearch", home)); assertTrue(e.getMessage(), e.getMessage().contains("not a directory")); assertTrue(Files.exists(env.pluginsFile().resolve("elasticsearch"))); // did not remove assertTrue(Files.exists(env.binFile().resolve("elasticsearch"))); diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java index 4199a5d67cd..e7b5d1c4501 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java @@ -23,10 +23,10 @@ import org.apache.lucene.util.IOUtils; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodeService; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESTestCase; @@ -65,14 +65,14 @@ public class TribeUnitTests extends ESTestCase { .put(baseSettings) .put("cluster.name", "tribe1") .put("node.name", "tribe1_node") - .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), random().nextLong()) + .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong()) .build()).start(); tribe2 = new TribeClientNode( Settings.builder() .put(baseSettings) .put("cluster.name", "tribe2") .put("node.name", "tribe2_node") - .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), random().nextLong()) + .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong()) .build()).start(); } diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml b/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml index d4fa8d8d130..19b2a7b5dd9 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml +++ b/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml @@ -1,5 +1,5 @@ cluster.name: tribe_node_cluster tribe.t1.cluster.name: tribe1 tribe.t2.cluster.name: tribe2 -tribe.t1.node_id.seed: 1 -tribe.t2.node_id.seed: 2 +tribe.t1.node.id.seed: 1 +tribe.t2.node.id.seed: 2 diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash index 21882c0ad53..eac0a80e4f4 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash @@ -228,6 +228,10 @@ fi install_and_check_plugin ingest geoip geoip2-*.jar jackson-annotations-*.jar jackson-databind-*.jar maxmind-db-*.jar } +@test "[$GROUP] install ingest-useragent plugin" { + install_and_check_plugin ingest useragent +} + @test "[$GROUP] check ingest-common module" { check_module ingest-common jcodings-*.jar joni-*.jar } @@ -357,6 +361,10 @@ fi remove_plugin ingest-geoip } +@test "[$GROUP] remove ingest-useragent plugin" { + remove_plugin ingest-useragent +} + @test "[$GROUP] remove javascript plugin" { remove_plugin lang-javascript } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/template.msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json similarity index 97% rename from rest-api-spec/src/main/resources/rest-api-spec/api/template.msearch.json rename to rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json index 379b7bdf362..39aa53b2572 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/template.msearch.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json @@ -1,5 +1,5 @@ { - "template.msearch": { + "msearch_template": { "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html", "methods": ["GET", "POST"], "url": { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index d2b9b8cf9b4..21fda8dc805 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -38,13 +38,17 @@ "type" : "boolean", "description" : "Specify whether to return detailed information about score computation as part of a hit" }, - "fields": { + "stored_fields": { "type" : "list", - "description" : "A comma-separated list of fields to return as part of a hit" + "description" : "A comma-separated list of stored fields to return as part of a hit" + }, + "docvalue_fields": { + "type" : "list", + "description" : "A comma-separated list of fields to return as the docvalue representation of a field for each hit" }, "fielddata_fields": { "type" : "list", - "description" : "A comma-separated list of fields to return as the field data representation of a field for each hit" + "description" : "A comma-separated list of fields to return as the docvalue representation of a field for each hit" }, "from": { "type" : "number", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json deleted file mode 100644 index 6a2a8c1d7e2..00000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "template.search": { - "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html", - "methods": ["GET", "POST"], - "url": { - "path": "/_search/template", - "paths": ["/_search/template", "/{index}/_search/template", "/{index}/{type}/_search/template"], - "parts": { - "index": { - "type" : "list", - "description" : "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices" - }, - "type": { - "type" : "list", - "description" : "A comma-separated list of document types to search; leave empty to perform the operation on all types" - } - }, - "params" : { - "ignore_unavailable": { - "type" : "boolean", - "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)" - }, - "allow_no_indices": { - "type" : "boolean", - "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)" - }, - "expand_wildcards": { - "type" : "enum", - "options" : ["open","closed","none","all"], - "default" : "open", - "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both." - }, - "preference": { - "type" : "string", - "description" : "Specify the node or shard the operation should be performed on (default: random)" - }, - "routing": { - "type" : "list", - "description" : "A comma-separated list of specific routing values" - }, - "scroll": { - "type" : "duration", - "description" : "Specify how long a consistent view of the index should be maintained for scrolled search" - }, - "search_type": { - "type" : "enum", - "options" : ["query_then_fetch", "query_and_fetch", "dfs_query_then_fetch", "dfs_query_and_fetch"], - "description" : "Search operation type" - } - } - }, - "body": { - "description": "The search definition template and its params" - } - } -} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml index 51f8fe9ed4c..1e485fc6d30 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml @@ -26,6 +26,7 @@ /^(green \s+ open \s+ index1 \s+ + ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+ 1 \s+ 0 \s+ 0 \s+ @@ -62,6 +63,7 @@ /^( \s+ close \s+ index1 \s+ + ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+ \s+ \s+ \s+ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml index 84bf44f7392..424153aa573 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml @@ -95,7 +95,7 @@ setup: - do: search: body: - fields: [ include.field2 ] + stored_fields: [ include.field2 ] query: { match_all: {} } - is_false: hits.hits.0._source @@ -104,7 +104,7 @@ setup: - do: search: body: - fields: [ include.field2, _source ] + stored_fields: [ include.field2, _source ] query: { match_all: {} } - match: { hits.hits.0._source.include.field2: v2 } - is_true: hits.hits.0._source @@ -113,5 +113,5 @@ setup: "fielddata_fields": - do: search: - fielddata_fields: [ "count" ] + docvalue_fields: [ "count" ] - match: { hits.hits.0.fields.count: [1] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml index df7322f12c8..993cbed2647 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml @@ -31,6 +31,6 @@ setup: term: data: some preference: _local - fields: [user,amount] + stored_fields: [user,amount] diff --git a/settings.gradle b/settings.gradle index 982fc004a18..48e695d00ec 100644 --- a/settings.gradle +++ b/settings.gradle @@ -37,6 +37,7 @@ List projects = [ 'plugins:discovery-gce', 'plugins:ingest-geoip', 'plugins:ingest-attachment', + 'plugins:ingest-useragent', 'plugins:lang-javascript', 'plugins:lang-python', 'plugins:mapper-attachments', diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java index aa327ae2546..f08f15f236b 100644 --- a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java @@ -53,12 +53,12 @@ abstract class ESElasticsearchCliTestCase extends ESTestCase { assertThat(status, equalTo(expectedStatus)); assertThat(init.get(), equalTo(expectedInit)); outputConsumer.accept(terminal.getOutput()); - } catch (Throwable t) { + } catch (Exception e) { // if an unexpected exception is thrown, we log // terminal output to aid debugging logger.info(terminal.getOutput()); // rethrow so the test fails - throw t; + throw e; } } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java index 128b0d0e315..45edbd8bcb2 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java @@ -32,7 +32,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.plugins.Plugin; @@ -67,7 +67,7 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService { usage.getTotalBytes(), usage.getFreeBytes(), usage.getFreeBytes()); paths[0] = path; FsInfo fsInfo = new FsInfo(System.currentTimeMillis(), null, paths); - return new NodeStats(new DiscoveryNode(nodeName, DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), + return new NodeStats(new DiscoveryNode(nodeName, LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), System.currentTimeMillis(), null, null, null, null, null, fsInfo, diff --git a/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java b/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java deleted file mode 100644 index a1c9da18470..00000000000 --- a/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.bytes; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; -import org.elasticsearch.common.io.stream.StreamInput; -import org.jboss.netty.util.CharsetUtil; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.CharacterCodingException; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CoderResult; -import java.nio.charset.StandardCharsets; - -/** - * Note: this is only used by one lone test method. - */ -public class ByteBufferBytesReference implements BytesReference { - - private final ByteBuffer buffer; - - public ByteBufferBytesReference(ByteBuffer buffer) { - this.buffer = buffer; - } - - @Override - public byte get(int index) { - return buffer.get(buffer.position() + index); - } - - @Override - public int length() { - return buffer.remaining(); - } - - @Override - public BytesReference slice(int from, int length) { - ByteBuffer dup = buffer.duplicate(); - dup.position(buffer.position() + from); - dup.limit(buffer.position() + from + length); - return new ByteBufferBytesReference(dup); - } - - @Override - public StreamInput streamInput() { - return new ByteBufferStreamInput(buffer); - } - - @Override - public void writeTo(OutputStream os) throws IOException { - if (buffer.hasArray()) { - os.write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); - } else { - byte[] tmp = new byte[8192]; - ByteBuffer buf = buffer.duplicate(); - while (buf.hasRemaining()) { - buf.get(tmp, 0, Math.min(tmp.length, buf.remaining())); - os.write(tmp); - } - } - } - - @Override - public byte[] toBytes() { - if (!buffer.hasRemaining()) { - return BytesRef.EMPTY_BYTES; - } - byte[] tmp = new byte[buffer.remaining()]; - buffer.duplicate().get(tmp); - return tmp; - } - - @Override - public BytesArray toBytesArray() { - if (buffer.hasArray()) { - return new BytesArray(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); - } - return new BytesArray(toBytes()); - } - - @Override - public BytesArray copyBytesArray() { - return new BytesArray(toBytes()); - } - - @Override - public boolean hasArray() { - return buffer.hasArray(); - } - - @Override - public byte[] array() { - return buffer.array(); - } - - @Override - public int arrayOffset() { - return buffer.arrayOffset() + buffer.position(); - } - - @Override - public int hashCode() { - return Helper.bytesHashCode(this); - } - - @Override - public boolean equals(Object obj) { - return Helper.bytesEqual(this, (BytesReference) obj); - } - - @Override - public String toUtf8() { - if (!buffer.hasRemaining()) { - return ""; - } - final CharsetDecoder decoder = CharsetUtil.getDecoder(StandardCharsets.UTF_8); - final CharBuffer dst = CharBuffer.allocate( - (int) ((double) buffer.remaining() * decoder.maxCharsPerByte())); - try { - CoderResult cr = decoder.decode(buffer, dst, true); - if (!cr.isUnderflow()) { - cr.throwException(); - } - cr = decoder.flush(dst); - if (!cr.isUnderflow()) { - cr.throwException(); - } - } catch (CharacterCodingException x) { - throw new IllegalStateException(x); - } - return dst.flip().toString(); - } - - @Override - public BytesRef toBytesRef() { - if (buffer.hasArray()) { - return new BytesRef(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); - } - return new BytesRef(toBytes()); - } - - @Override - public BytesRef copyBytesRef() { - return new BytesRef(toBytes()); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java index 613445c2271..22a1e2660b6 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java +++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java @@ -246,7 +246,7 @@ public class MockBigArrays extends BigArrays { return arr; } - private static abstract class AbstractArrayWrapper { + private abstract static class AbstractArrayWrapper { final BigArray in; boolean clearOnResize; diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index e86854b34c4..0d9ed196838 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -326,11 +326,11 @@ public abstract class AbstractQueryTestCase> /** * Parses the query provided as string argument and compares it with the expected result provided as argument as a {@link QueryBuilder} */ - protected final static void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException { + protected static final void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException { assertParsedQuery(queryAsString, expectedQuery, ParseFieldMatcher.STRICT); } - protected final static void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException { + protected static final void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException { QueryBuilder newQuery = parseQuery(queryAsString, matcher); assertNotSame(newQuery, expectedQuery); assertEquals(expectedQuery, newQuery); @@ -340,31 +340,31 @@ public abstract class AbstractQueryTestCase> /** * Parses the query provided as bytes argument and compares it with the expected result provided as argument as a {@link QueryBuilder} */ - protected final static void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException { + protected static final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException { assertParsedQuery(queryAsBytes, expectedQuery, ParseFieldMatcher.STRICT); } - protected final static void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException { + protected static final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException { QueryBuilder newQuery = parseQuery(queryAsBytes, matcher); assertNotSame(newQuery, expectedQuery); assertEquals(expectedQuery, newQuery); assertEquals(expectedQuery.hashCode(), newQuery.hashCode()); } - protected final static QueryBuilder parseQuery(String queryAsString) throws IOException { + protected static final QueryBuilder parseQuery(String queryAsString) throws IOException { return parseQuery(queryAsString, ParseFieldMatcher.STRICT); } - protected final static QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException { + protected static final QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException { XContentParser parser = XContentFactory.xContent(queryAsString).createParser(queryAsString); return parseQuery(parser, matcher); } - protected final static QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException { + protected static final QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException { return parseQuery(queryAsBytes, ParseFieldMatcher.STRICT); } - protected final static QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException { + protected static final QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException { XContentParser parser = XContentFactory.xContent(queryAsBytes).createParser(queryAsBytes); return parseQuery(parser, matcher); } @@ -513,7 +513,7 @@ public abstract class AbstractQueryTestCase> protected QueryBuilder assertSerialization(QueryBuilder testQuery) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { output.writeNamedWriteable(testQuery); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), serviceHolder.namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), serviceHolder.namedWriteableRegistry)) { QueryBuilder deserializedQuery = in.readNamedWriteable(QueryBuilder.class); assertEquals(testQuery, deserializedQuery); assertEquals(testQuery.hashCode(), deserializedQuery.hashCode()); @@ -562,7 +562,7 @@ public abstract class AbstractQueryTestCase> protected QB copyQuery(QB query) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { output.writeNamedWriteable(query); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), serviceHolder.namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), serviceHolder.namedWriteableRegistry)) { return (QB) in.readNamedWriteable(QueryBuilder.class); } } @@ -690,7 +690,7 @@ public abstract class AbstractQueryTestCase> } public static class GeohashGenerator extends CodepointSetGenerator { - private final static char[] ASCII_SET = "0123456789bcdefghjkmnpqrstuvwxyz".toCharArray(); + private static final char[] ASCII_SET = "0123456789bcdefghjkmnpqrstuvwxyz".toCharArray(); public GeohashGenerator() { super(ASCII_SET); diff --git a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java index 933f26e6e81..4440fbe117d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java +++ b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java @@ -49,7 +49,7 @@ public class BackgroundIndexer implements AutoCloseable { final Thread[] writers; final CountDownLatch stopLatch; - final CopyOnWriteArrayList failures; + final CopyOnWriteArrayList failures; final AtomicBoolean stop = new AtomicBoolean(false); final AtomicLong idGenerator = new AtomicLong(); final AtomicLong indexCounter = new AtomicLong(); @@ -169,7 +169,7 @@ public class BackgroundIndexer implements AutoCloseable { } } logger.info("**** done indexing thread {} stop: {} numDocsIndexed: {}", indexerId, stop.get(), indexCounter.get()); - } catch (Throwable e) { + } catch (Exception e) { failures.add(e); logger.warn("**** failed indexing thread {} on doc id {}", e, indexerId, id); } finally { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java index fe7ba74a327..a6d35930e6b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java @@ -21,7 +21,6 @@ package org.elasticsearch.test; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.NodeConnectionsService; @@ -30,7 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.threadpool.ThreadPool; import java.util.Arrays; @@ -46,7 +45,7 @@ public class ClusterServiceUtils { ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "ClusterServiceTests").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool); - clusterService.setLocalNode(new DiscoveryNode("node", DummyTransportAddress.INSTANCE, Collections.emptyMap(), + clusterService.setLocalNode(new DiscoveryNode("node", LocalTransportAddress.buildUnique(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())),Version.CURRENT)); clusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) { @Override @@ -98,8 +97,8 @@ public class ClusterServiceUtils { } @Override - public void onFailure(String source, Throwable t) { - fail("unexpected exception" + t); + public void onFailure(String source, Exception e) { + fail("unexpected exception" + e); } }); try { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java index 5704a178f48..1aa0428454e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java @@ -42,7 +42,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationD import org.elasticsearch.common.Randomness; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.gateway.AsyncShardFetch; import org.elasticsearch.gateway.GatewayAllocator; import org.elasticsearch.gateway.ReplicaShardAllocator; @@ -133,19 +133,19 @@ public abstract class ESAllocationTestCase extends ESTestCase { } protected static DiscoveryNode newNode(String nodeName, String nodeId, Map attributes) { - return new DiscoveryNode(nodeName, nodeId, DummyTransportAddress.INSTANCE, attributes, MASTER_DATA_ROLES, Version.CURRENT); + return new DiscoveryNode(nodeName, nodeId, LocalTransportAddress.buildUnique(), attributes, MASTER_DATA_ROLES, Version.CURRENT); } protected static DiscoveryNode newNode(String nodeId, Map attributes) { - return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, attributes, MASTER_DATA_ROLES, Version.CURRENT); + return new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), attributes, MASTER_DATA_ROLES, Version.CURRENT); } protected static DiscoveryNode newNode(String nodeId, Set roles) { - return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(), roles, Version.CURRENT); + return new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), emptyMap(), roles, Version.CURRENT); } protected static DiscoveryNode newNode(String nodeId, Version version) { - return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(), MASTER_DATA_ROLES, version); + return new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, version); } protected static ClusterState startRandomInitializingShard(ClusterState clusterState, AllocationService strategy) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index a49c33f5be8..6d07a85e50b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -106,7 +106,6 @@ import org.elasticsearch.index.MergeSchedulerConfig; import org.elasticsearch.index.MockEngineFactoryPlugin; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesRequestCache; @@ -405,7 +404,7 @@ public abstract class ESIntegTestCase extends ESTestCase { .setOrder(0) .setSettings(randomSettingsBuilder); if (mappings != null) { - logger.info("test using _default_ mappings: [{}]", mappings.bytes().toUtf8()); + logger.info("test using _default_ mappings: [{}]", mappings.bytes().utf8ToString()); putTemplate.addMapping("_default_", mappings); } assertAcked(putTemplate.execute().actionGet()); @@ -925,7 +924,7 @@ public abstract class ESIntegTestCase extends ESTestCase { * This saves on unneeded searches. * @return the actual number of docs seen. */ - public long waitForDocs(final long numDocs, final @Nullable BackgroundIndexer indexer) throws InterruptedException { + public long waitForDocs(final long numDocs, @Nullable final BackgroundIndexer indexer) throws InterruptedException { // indexing threads can wait for up to ~1m before retrying when they first try to index into a shard which is not STARTED. return waitForDocs(numDocs, 90, TimeUnit.SECONDS, indexer); } @@ -940,7 +939,7 @@ public abstract class ESIntegTestCase extends ESTestCase { * This saves on unneeded searches. * @return the actual number of docs seen. */ - public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTimeUnit, final @Nullable BackgroundIndexer indexer) + public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTimeUnit, @Nullable final BackgroundIndexer indexer) throws InterruptedException { final AtomicLong lastKnownCount = new AtomicLong(-1); long lastStartCount = -1; @@ -956,7 +955,7 @@ public abstract class ESIntegTestCase extends ESTestCase { client().admin().indices().prepareRefresh().get(); } lastKnownCount.set(count); - } catch (Throwable e) { // count now acts like search and barfs if all shards failed... + } catch (Exception e) { // count now acts like search and barfs if all shards failed... logger.debug("failed to executed count", e); return false; } @@ -1334,7 +1333,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } final String[] indices = indicesSet.toArray(new String[indicesSet.size()]); Collections.shuffle(builders, random()); - final CopyOnWriteArrayList> errors = new CopyOnWriteArrayList<>(); + final CopyOnWriteArrayList> errors = new CopyOnWriteArrayList<>(); List inFlightAsyncOperations = new ArrayList<>(); // If you are indexing just a few documents then frequently do it one at a time. If many then frequently in bulk. if (builders.size() < FREQUENT_BULK_THRESHOLD ? frequently() : builders.size() < ALWAYS_BULK_THRESHOLD ? rarely() : false) { @@ -1367,8 +1366,8 @@ public abstract class ESIntegTestCase extends ESTestCase { for (CountDownLatch operation : inFlightAsyncOperations) { operation.await(); } - final List actualErrors = new ArrayList<>(); - for (Tuple tuple : errors) { + final List actualErrors = new ArrayList<>(); + for (Tuple tuple : errors) { if (ExceptionsHelper.unwrapCause(tuple.v2()) instanceof EsRejectedExecutionException) { tuple.v1().execute().actionGet(); // re-index if rejected } else { @@ -1526,7 +1525,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } @Override - public final void onFailure(Throwable t) { + public final void onFailure(Exception t) { try { logger.info("Action Failed", t); addError(t); @@ -1535,24 +1534,24 @@ public abstract class ESIntegTestCase extends ESTestCase { } } - protected void addError(Throwable t) { + protected void addError(Exception e) { } } private class PayloadLatchedActionListener extends LatchedActionListener { - private final CopyOnWriteArrayList> errors; + private final CopyOnWriteArrayList> errors; private final T builder; - public PayloadLatchedActionListener(T builder, CountDownLatch latch, CopyOnWriteArrayList> errors) { + public PayloadLatchedActionListener(T builder, CountDownLatch latch, CopyOnWriteArrayList> errors) { super(latch); this.errors = errors; this.builder = builder; } @Override - protected void addError(Throwable t) { - errors.add(new Tuple<>(builder, t)); + protected void addError(Exception e) { + errors.add(new Tuple<>(builder, e)); } } @@ -2035,7 +2034,7 @@ public abstract class ESIntegTestCase extends ESTestCase { * The returned client gets automatically closed when needed, it shouldn't be closed as part of tests otherwise * it cannot be reused by other tests anymore. */ - protected synchronized static RestClient getRestClient() { + protected static synchronized RestClient getRestClient() { if (restClient == null) { restClient = createRestClient(null); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index b6b75b1ec64..620727e255d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -40,7 +40,6 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNode.Role; -import org.elasticsearch.cluster.node.DiscoveryNodeService; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.cluster.routing.ShardRouting; @@ -606,7 +605,7 @@ public final class InternalTestCluster extends TestCluster { .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home .put(settings) .put("node.name", name) - .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), seed) + .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed) .build(); MockNode node = new MockNode(finalSettings, plugins); return new NodeAndClient(name, node, nodeId); @@ -898,8 +897,8 @@ public final class InternalTestCluster extends TestCluster { } private void createNewNode(final Settings newSettings) { - final long newIdSeed = DiscoveryNodeService.NODE_ID_SEED_SETTING.get(node.settings()) + 1; // use a new seed to make sure we have new node id - Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), newIdSeed).build(); + final long newIdSeed = NodeEnvironment.NODE_ID_SEED_SETTING.get(node.settings()) + 1; // use a new seed to make sure we have new node id + Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), newIdSeed).build(); Collection> plugins = node.getPlugins(); node = new MockNode(finalSettings, plugins); markNodeDataDirsAsNotEligableForWipe(node); @@ -1337,7 +1336,7 @@ public final class InternalTestCluster extends TestCluster { /** * Restarts a node and calls the callback during restart. */ - synchronized public void restartNode(String nodeName, RestartCallback callback) throws Exception { + public synchronized void restartNode(String nodeName, RestartCallback callback) throws Exception { ensureOpen(); NodeAndClient nodeAndClient = nodes.get(nodeName); if (nodeAndClient != null) { @@ -1346,7 +1345,7 @@ public final class InternalTestCluster extends TestCluster { } } - synchronized private void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception { + private synchronized void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception { ensureOpen(); List toRemove = new ArrayList<>(); try { @@ -1393,7 +1392,6 @@ public final class InternalTestCluster extends TestCluster { // delete data folders now, before we start other nodes that may claim it nodeAndClient.clearDataIfNeeded(callback); - DiscoveryNode discoveryNode = getInstanceFromNode(ClusterService.class, nodeAndClient.node()).localNode(); nodesRoleOrder[nodeAndClient.nodeAndClientId()] = discoveryNode.getRoles(); nodesByRoles.computeIfAbsent(discoveryNode.getRoles(), k -> new ArrayList<>()).add(nodeAndClient); @@ -1481,7 +1479,7 @@ public final class InternalTestCluster extends TestCluster { Client client = viaNode != null ? client(viaNode) : client(); ClusterState state = client.admin().cluster().prepareState().execute().actionGet().getState(); return state.nodes().getMasterNode().getName(); - } catch (Throwable e) { + } catch (Exception e) { logger.warn("Can't fetch cluster state", e); throw new RuntimeException("Can't get master node " + e.getMessage(), e); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java b/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java index 10469286e1a..1d0eaa7ce51 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java @@ -19,6 +19,7 @@ package org.elasticsearch.test; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -53,7 +54,7 @@ public class StreamsUtils { } try (BytesStreamOutput out = new BytesStreamOutput()) { Streams.copy(is, out); - return out.bytes().toBytes(); + return BytesReference.toBytes(out.bytes()); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java index 5525baf4206..d09c763322c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java @@ -44,7 +44,7 @@ public class VersionUtils { try { Version object = (Version) field.get(null); ids.add(object.id); - } catch (Throwable e) { + } catch (IllegalAccessException e) { throw new RuntimeException(e); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java b/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java index 6ff45608700..1d91b0980e4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java @@ -50,8 +50,8 @@ public class NoOpClient extends AbstractClient { public void close() { try { ThreadPool.terminate(threadPool(), 10, TimeUnit.SECONDS); - } catch (Throwable t) { - throw new ElasticsearchException(t.getMessage(), t); + } catch (Exception e) { + throw new ElasticsearchException(e.getMessage(), e); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java index 7c832118ca0..48b726dc3cc 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java +++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java @@ -126,7 +126,7 @@ public class ClusterDiscoveryConfiguration extends NodeConfigurationSource { } @SuppressForbidden(reason = "we know we pass a IP address") - protected synchronized static int[] unicastHostPorts(int numHosts) { + protected static synchronized int[] unicastHostPorts(int numHosts) { int[] unicastHostPorts = new int[numHosts]; final int basePort = calcBasePort(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java index cbcb9766943..956088f0fd1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java @@ -76,8 +76,8 @@ public class BlockClusterStateProcessing extends SingleNodeDisruption { } @Override - public void onFailure(String source, Throwable t) { - logger.error("unexpected error during disruption", t); + public void onFailure(String source, Exception e) { + logger.error("unexpected error during disruption", e); } }); try { diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java index d957220c6df..caab35e4b42 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java @@ -61,7 +61,7 @@ public class IntermittentLongGCDisruption extends LongGCDisruption { this.delayDurationMax = delayDurationMax; } - final static AtomicInteger thread_ids = new AtomicInteger(); + static final AtomicInteger thread_ids = new AtomicInteger(); @Override public void startDisrupting() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java index 591540e72b9..b210a20cf70 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java @@ -33,7 +33,7 @@ import java.util.regex.Pattern; */ public class LongGCDisruption extends SingleNodeDisruption { - private final static Pattern[] unsafeClasses = new Pattern[]{ + private static final Pattern[] unsafeClasses = new Pattern[]{ // logging has shared JVM locks - we may suspend a thread and block other nodes from doing their thing Pattern.compile("Logger") }; diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java index be0b69a8e8b..f69c0a3085d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java @@ -124,7 +124,7 @@ public class SlowClusterStateProcessing extends SingleNodeDisruption { } @Override - public void onFailure(String source, Throwable t) { + public void onFailure(String source, Exception e) { countDownLatch.countDown(); } }); diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java index bf32b6b8575..304e3047496 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java @@ -172,7 +172,7 @@ public final class MockEngineSupport { return reader; } - public static abstract class DirectoryReaderWrapper extends FilterDirectoryReader { + public abstract static class DirectoryReaderWrapper extends FilterDirectoryReader { protected final SubReaderWrapper subReaderWrapper; public DirectoryReaderWrapper(DirectoryReader in, SubReaderWrapper subReaderWrapper) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java index 73281b3f6ea..37ed43b9450 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java +++ b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java @@ -62,18 +62,18 @@ public class ThrowingLeafReaderWrapper extends FilterLeafReader { * A callback interface that allows to throw certain exceptions for * methods called on the IndexReader that is wrapped by {@link ThrowingLeafReaderWrapper} */ - public static interface Thrower { + public interface Thrower { /** * Maybe throws an exception ;) */ - public void maybeThrow(Flags flag) throws IOException; + void maybeThrow(Flags flag) throws IOException; /** * If this method returns true the {@link Terms} instance for the given field * is wrapped with Thrower support otherwise no exception will be thrown for * the current {@link Terms} instance or any other instance obtained from it. */ - public boolean wrapTerms(String field); + boolean wrapTerms(String field); } public ThrowingLeafReaderWrapper(LeafReader in, Thrower thrower) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index baad3ef6f04..988d4b9bb87 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -95,9 +95,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -/** - * - */ public class ElasticsearchAssertions { public static void assertAcked(AcknowledgedRequestBuilder builder) { @@ -555,7 +552,6 @@ public class ElasticsearchAssertions { extraInfo += " with status [" + status + "]"; } - try { future.actionGet(); fail = true; @@ -565,7 +561,7 @@ public class ElasticsearchAssertions { if (status != null) { assertThat(extraInfo, ExceptionsHelper.status(esException), equalTo(status)); } - } catch (Throwable e) { + } catch (Exception e) { assertThat(extraInfo, e, instanceOf(exceptionClass)); if (status != null) { assertThat(extraInfo, ExceptionsHelper.status(e), equalTo(status)); @@ -597,7 +593,7 @@ public class ElasticsearchAssertions { try { future.actionGet(); fail = true; - } catch (Throwable e) { + } catch (Exception e) { assertThat(extraInfo, ExceptionsHelper.status(e), equalTo(status)); } // has to be outside catch clause to get a proper message @@ -647,7 +643,7 @@ public class ElasticsearchAssertions { ((ActionRequest) streamable).validate(); } BytesReference orig = serialize(version, streamable); - StreamInput input = StreamInput.wrap(orig); + StreamInput input = orig.streamInput(); if (namedWriteableRegistry != null) { input = new NamedWriteableAwareStreamInput(input, namedWriteableRegistry); } @@ -657,35 +653,38 @@ public class ElasticsearchAssertions { equalTo(0)); assertThat("Serialization failed with version [" + version + "] bytes should be equal for streamable [" + streamable + "]", serialize(version, streamable), equalTo(orig)); - } catch (Throwable ex) { + } catch (Exception ex) { throw new RuntimeException("failed to check serialization - version [" + version + "] for streamable [" + streamable + "]", ex); } } - public static void assertVersionSerializable(Version version, final Throwable t) { - ElasticsearchAssertions.assertVersionSerializable(version, new ThrowableWrapper(t)); + public static void assertVersionSerializable(Version version, final Exception e) { + ElasticsearchAssertions.assertVersionSerializable(version, new ExceptionWrapper(e)); } - public static final class ThrowableWrapper implements Streamable { - Throwable throwable; - public ThrowableWrapper(Throwable t) { - throwable = t; + public static final class ExceptionWrapper implements Streamable { + + private Exception exception; + + public ExceptionWrapper(Exception e) { + exception = e; } - public ThrowableWrapper() { - throwable = null; + public ExceptionWrapper() { + exception = null; } @Override public void readFrom(StreamInput in) throws IOException { - throwable = in.readThrowable(); + exception = in.readException(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeThrowable(throwable); + out.writeThrowable(exception); } + } @@ -697,7 +696,7 @@ public class ElasticsearchAssertions { assertThat(constructor, Matchers.notNullValue()); Streamable newInstance = constructor.newInstance(); return newInstance; - } catch (Throwable e) { + } catch (Exception e) { return null; } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java index 77de0f30f0c..8c492d279b0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java @@ -70,7 +70,7 @@ public class ObjectPath { @SuppressWarnings("unchecked") private Object evaluate(String key, Object object, Stash stash) throws IOException { - if (stash.isStashedValue(key)) { + if (stash.containsStashedValue(key)) { key = stash.getValue(key).toString(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java index 6d95c7e893f..d7295e1dca7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java @@ -66,7 +66,7 @@ public class RestTestExecutionContext implements Closeable { //makes a copy of the parameters before modifying them for this specific request HashMap requestParams = new HashMap<>(params); for (Map.Entry entry : requestParams.entrySet()) { - if (stash.isStashedValue(entry.getValue())) { + if (stash.containsStashedValue(entry.getValue())) { entry.setValue(stash.getValue(entry.getValue()).toString()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java index 13c5b9ff563..a4218f83340 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java @@ -29,12 +29,15 @@ import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Allows to cache the last obtained test response and or part of it within variables * that can be used as input values in following requests and assertions. */ public class Stash implements ToXContent { + private static final Pattern EXTENDED_KEY = Pattern.compile("\\$\\{([^}]+)\\}"); private static final ESLogger logger = Loggers.getLogger(Stash.class); @@ -67,12 +70,18 @@ public class Stash implements ToXContent { * The stash contains fields eventually extracted from previous responses that can be reused * as arguments for following requests (e.g. scroll_id) */ - public boolean isStashedValue(Object key) { + public boolean containsStashedValue(Object key) { if (key == null) { return false; } String stashKey = key.toString(); - return Strings.hasLength(stashKey) && stashKey.startsWith("$"); + if (false == Strings.hasLength(stashKey)) { + return false; + } + if (stashKey.startsWith("$")) { + return true; + } + return EXTENDED_KEY.matcher(stashKey).find(); } /** @@ -81,7 +90,27 @@ public class Stash implements ToXContent { * as arguments for following requests (e.g. scroll_id) */ public Object getValue(String key) throws IOException { - Object stashedValue = stashObjectPath.evaluate(key.substring(1)); + if (key.charAt(0) == '$' && key.charAt(1) != '{') { + return unstash(key.substring(1)); + } + Matcher matcher = EXTENDED_KEY.matcher(key); + /* + * String*Buffer* because that is what the Matcher API takes. In modern versions of java the uncontended synchronization is very, + * very cheap so that should not be a problem. + */ + StringBuffer result = new StringBuffer(key.length()); + if (false == matcher.find()) { + throw new IllegalArgumentException("Doesn't contain any stash keys [" + key + "]"); + } + do { + matcher.appendReplacement(result, Matcher.quoteReplacement(unstash(matcher.group(1)).toString())); + } while (matcher.find()); + matcher.appendTail(result); + return result.toString(); + } + + private Object unstash(String key) throws IOException { + Object stashedValue = stashObjectPath.evaluate(key); if (stashedValue == null) { throw new IllegalArgumentException("stashed value not found for key [" + key + "]"); } @@ -104,7 +133,7 @@ public class Stash implements ToXContent { List list = (List) obj; for (int i = 0; i < list.size(); i++) { Object o = list.get(i); - if (isStashedValue(o)) { + if (containsStashedValue(o)) { list.set(i, getValue(o.toString())); } else { unstashObject(o); @@ -114,7 +143,7 @@ public class Stash implements ToXContent { if (obj instanceof Map) { Map map = (Map) obj; for (Map.Entry entry : map.entrySet()) { - if (isStashedValue(entry.getValue())) { + if (containsStashedValue(entry.getValue())) { entry.setValue(getValue(entry.getValue().toString())); } else { unstashObject(entry.getValue()); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java index de38940169f..fbba9de163b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java @@ -51,14 +51,14 @@ public abstract class Assertion implements ExecutableSection { return executionContext.stash().replaceStashedValues(map); } - if (executionContext.stash().isStashedValue(expectedValue)) { + if (executionContext.stash().containsStashedValue(expectedValue)) { return executionContext.stash().getValue(expectedValue.toString()); } return expectedValue; } protected final Object getActualValue(RestTestExecutionContext executionContext) throws IOException { - if (executionContext.stash().isStashedValue(field)) { + if (executionContext.stash().containsStashedValue(field)) { return executionContext.stash().getValue(field); } return executionContext.response(field); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java b/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java index 106ff5176c7..c6ea48fd6ef 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java @@ -75,7 +75,7 @@ public class RestSpec { } restSpec.addApi(restApi); } - } catch (Throwable ex) { + } catch (Exception ex) { throw new IOException("Can't parse rest spec file: [" + jsonFile + "]", ex); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java index 0f51f72e8e5..cf33570ef28 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java @@ -34,7 +34,7 @@ import java.util.List; */ public final class Features { - private static final List SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers"); + private static final List SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers", "embedded_stash_key"); private Features() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java index 7b1c3fd936b..1b99d2f32cb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java @@ -131,11 +131,11 @@ public class MockFSDirectoryService extends FsDirectoryService { ESTestCase.checkIndexFailed = true; logger.warn("check index [failure] index files={}\n{}", Arrays.toString(dir.listAll()), - new String(os.bytes().toBytes(), StandardCharsets.UTF_8)); + os.bytes().utf8ToString()); throw new IOException("index check failure"); } else { if (logger.isDebugEnabled()) { - logger.debug("check index [success]\n{}", new String(os.bytes().toBytes(), StandardCharsets.UTF_8)); + logger.debug("check index [success]\n{}", os.bytes().utf8ToString()); } } } catch (LockObtainFailedException e) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java index b0d16d10c49..ec695e8bd41 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java +++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java @@ -50,8 +50,8 @@ public class MockTaskManager extends TaskManager { for (MockTaskManagerListener listener : listeners) { try { listener.onTaskRegistered(task); - } catch (Throwable t) { - logger.warn("failed to notify task manager listener about unregistering the task with id {}", t, task.getId()); + } catch (Exception e) { + logger.warn("failed to notify task manager listener about unregistering the task with id {}", e, task.getId()); } } } @@ -65,8 +65,8 @@ public class MockTaskManager extends TaskManager { for (MockTaskManagerListener listener : listeners) { try { listener.onTaskUnregistered(task); - } catch (Throwable t) { - logger.warn("failed to notify task manager listener about unregistering the task with id {}", t, task.getId()); + } catch (Exception e) { + logger.warn("failed to notify task manager listener about unregistering the task with id {}", e, task.getId()); } } } else { @@ -80,8 +80,8 @@ public class MockTaskManager extends TaskManager { for (MockTaskManagerListener listener : listeners) { try { listener.waitForTaskCompletion(task); - } catch (Throwable t) { - logger.warn("failed to notify task manager listener about waitForTaskCompletion the task with id {}", t, task.getId()); + } catch (Exception e) { + logger.warn("failed to notify task manager listener about waitForTaskCompletion the task with id {}", e, task.getId()); } } super.waitForTaskCompletion(task, untilInNanos); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java index 654a1c971ca..f0f85b24f72 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java @@ -51,11 +51,11 @@ public class CapturingTransport implements Transport { private TransportServiceAdapter adapter; - static public class CapturedRequest { - final public DiscoveryNode node; - final public long requestId; - final public String action; - final public TransportRequest request; + public static class CapturedRequest { + public final DiscoveryNode node; + public final long requestId; + public final String action; + public final TransportRequest request; public CapturedRequest(DiscoveryNode node, long requestId, String action, TransportRequest request) { this.node = node; @@ -242,19 +242,13 @@ public class CapturingTransport implements Transport { } @Override - public Transport start() { - return null; - } + public void start() {} @Override - public Transport stop() { - return null; - } + public void stop() {} @Override - public void close() { - - } + public void close() {} @Override public List getLocalAddresses() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 7688e2842e6..bc371ca02d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -20,7 +20,6 @@ package org.elasticsearch.test.transport; import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.transport.TransportService; @@ -34,7 +33,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; @@ -52,7 +50,6 @@ import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; -import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportServiceAdapter; import org.elasticsearch.transport.local.LocalTransport; import org.elasticsearch.transport.netty.NettyTransport; @@ -383,11 +380,11 @@ public class MockTransportService extends TransportService { BytesStreamOutput bStream = new BytesStreamOutput(); request.writeTo(bStream); final TransportRequest clonedRequest = reg.newRequest(); - clonedRequest.readFrom(StreamInput.wrap(bStream.bytes())); + clonedRequest.readFrom(bStream.bytes().streamInput()); threadPool.schedule(delay, ThreadPool.Names.GENERIC, new AbstractRunnable() { @Override - public void onFailure(Throwable e) { + public void onFailure(Exception e) { logger.debug("failed to send delayed request", e); } @@ -558,15 +555,13 @@ public class MockTransportService extends TransportService { } @Override - public Transport start() { + public void start() { transport.start(); - return this; } @Override - public Transport stop() { + public void stop() { transport.stop(); - return this; } @Override @@ -641,10 +636,10 @@ public class MockTransportService extends TransportService { } @Override - protected void traceResponseSent(long requestId, String action, Throwable t) { - super.traceResponseSent(requestId, action, t); + protected void traceResponseSent(long requestId, String action, Exception e) { + super.traceResponseSent(requestId, action, e); for (Tracer tracer : activeTracers) { - tracer.responseSent(requestId, action, t); + tracer.responseSent(requestId, action, e); } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java index 4a21fefe803..1d99a73c767 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java @@ -190,21 +190,49 @@ public class ObjectPathTests extends ESTestCase { objectPath.evaluate("field1.$placeholder.element1"); fail("evaluate should have failed due to unresolved placeholder"); } catch(IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("stashed value not found for key [$placeholder]")); + assertThat(e.getMessage(), containsString("stashed value not found for key [placeholder]")); } + // Stashed value is whole property name Stash stash = new Stash(); stash.stashValue("placeholder", "elements"); Object object = objectPath.evaluate("field1.$placeholder.element1", stash); assertThat(object, notNullValue()); assertThat(object.toString(), equalTo("value1")); + // Stash key has dots Map stashedObject = new HashMap<>(); stashedObject.put("subobject", "elements"); stash.stashValue("object", stashedObject); object = objectPath.evaluate("field1.$object\\.subobject.element1", stash); assertThat(object, notNullValue()); assertThat(object.toString(), equalTo("value1")); + + // Stashed value is part of property name + stash.stashValue("placeholder", "ele"); + object = objectPath.evaluate("field1.${placeholder}ments.element1", stash); + assertThat(object, notNullValue()); + assertThat(object.toString(), equalTo("value1")); + + // Stashed value is inside of property name + stash.stashValue("placeholder", "le"); + object = objectPath.evaluate("field1.e${placeholder}ments.element1", stash); + assertThat(object, notNullValue()); + assertThat(object.toString(), equalTo("value1")); + + // Multiple stashed values in property name + stash.stashValue("placeholder", "le"); + stash.stashValue("placeholder2", "nts"); + object = objectPath.evaluate("field1.e${placeholder}me${placeholder2}.element1", stash); + assertThat(object, notNullValue()); + assertThat(object.toString(), equalTo("value1")); + + // Stashed value is part of property name and has dots + stashedObject.put("subobject", "ele"); + stash.stashValue("object", stashedObject); + object = objectPath.evaluate("field1.${object\\.subobject}ments.element1", stash); + assertThat(object, notNullValue()); + assertThat(object.toString(), equalTo("value1")); } @SuppressWarnings("unchecked") diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 63c5eb01fb0..5bf11e4dc98 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -90,7 +90,7 @@ public class InternalTestClusterTests extends ESTestCase { * a set of settings that are expected to have different values betweem clusters, even they have been initialized with the same * base settins. */ - final static Set clusterUniqueSettings = new HashSet<>(); + static final Set clusterUniqueSettings = new HashSet<>(); static { clusterUniqueSettings.add(ClusterName.CLUSTER_NAME_SETTING.getKey()); @@ -178,7 +178,7 @@ public class InternalTestClusterTests extends ESTestCase { int maxNumDataNodes = 2; final String clusterName1 = "shared1"; NodeConfigurationSource nodeConfigurationSource = NodeConfigurationSource.EMPTY; - int numClientNodes = 0; + int numClientNodes = randomIntBetween(0, 2); boolean enableHttpPipelining = randomBoolean(); String nodePrefix = "test"; Path baseDir = createTempDir(); @@ -218,8 +218,7 @@ public class InternalTestClusterTests extends ESTestCase { assertFileNotExists(testMarker); // a new unknown node used this path, it should be cleaned assertFileExists(stableTestMarker); // but leaving the structure of existing, reused nodes for (String name: cluster.getNodeNames()) { - assertThat("data paths for " + name + " changed", getNodePaths(cluster, name), - equalTo(shardNodePaths.get(name))); + assertThat("data paths for " + name + " changed", getNodePaths(cluster, name), equalTo(shardNodePaths.get(name))); } cluster.beforeTest(random(), 0.0);